Compare commits

..
Author SHA1 Message Date
jschoubben 24f024dd74 The proxy is told its routes and the mesh on the bus, and serves internal names to the mesh only
The proxy answered every routed name to any request carrying it, so an
internal-only route would have been public under its internal name. Each
membership now carries what its module receives, from the same
composition as its received file, and every machine's private-network
address, the list the packet filter's "from the mesh" is. The proxy
follows its membership, serves internal names only to those machines and
itself, and keeps the file until the bus has spoken (novox/hq ADR 0167,
issue 191).
2026-10-02 01:46:30 +02:00
jschoubben 9acb5f1292 route-proxy: serve a route that names only its internal host
Since ADR 0138 an endpoint that reaches only the private network gets an
internal-name and no name, and the proxy skipped it as naming nothing, so
every internal-only module was unreachable by name (novox/hq issue 191).
2026-10-01 23:31:57 +02:00
13 changed files with 682 additions and 74 deletions
+2 -1
View File
@@ -404,7 +404,8 @@ func declarationWith(ctx context.Context, open *stores, node string,
if err != nil { if err != nil {
return sendable{}, err return sendable{}, err
} }
return sendable{Resources: composed.Resources, Adoption: adoption}, nil return sendable{Resources: composed.Resources, Adoption: adoption,
Received: composed.Received, Mesh: with.Mesh}, nil
} }
// renderingFor is everything a node's declaration is composed with, and the node's record. // renderingFor is everything a node's declaration is composed with, and the node's record.
+23 -10
View File
@@ -389,11 +389,7 @@ func pushCommand(ctx context.Context, args []string) error {
fmt.Printf("\n%d node(s) told\n", len(sending)) fmt.Printf("\n%d node(s) told\n", len(sending))
// And each machine's memberships, as every other send does (ADR 0160): a push is the one most // And each machine's memberships, as every other send does (ADR 0160): a push is the one most
// operators run, and on 2026-10-01 it was the one path that issued none. // operators run, and on 2026-10-01 it was the one path that issued none.
var told []string if err := issueMemberships(ctx, open, server, sending); err != nil {
for _, s := range sending {
told = append(told, s.node)
}
if err := issueMemberships(ctx, open, server, told); err != nil {
return err return err
} }
@@ -706,11 +702,15 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
// And every assignment on those machines its membership (novox/hq ADR 0160): composed from the // And every assignment on those machines its membership (novox/hq ADR 0160): composed from the
// same records the bus's accounts are, so what a runtime serves and what its account may are one // same records the bus's accounts are, so what a runtime serves and what its account may are one
// composition. Issued after the declaration, because the runtime it is for arrives with it. // composition. Issued after the declaration, because the runtime it is for arrives with it.
return issueMemberships(ctx, open, server, names) return issueMemberships(ctx, open, server, sending)
} }
// issueMemberships publishes the membership of every module on the named machines. // issueMemberships publishes the membership of every module on the machines just sent.
func issueMemberships(ctx context.Context, open *stores, server *link.Server, names []string) error { //
// Each carries what its module receives and the private network's addresses, from the same
// composition as the declaration it was sent (novox/hq ADR 0167): a provider reads what it is
// given on the bus, and the file written beside it says the same thing.
func issueMemberships(ctx context.Context, open *stores, server *link.Server, sent []readyNode) error {
records, err := open.inventory.BusRecords(ctx) records, err := open.inventory.BusRecords(ctx)
if err != nil { if err != nil {
return err return err
@@ -725,9 +725,22 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, na
// the push stands, the first failure is named once, and the next push tries again. // the push stands, the first failure is named once, and the next push tries again.
issued, failed := 0, 0 issued, failed := 0, 0
var first error var first error
for _, node := range names { for _, s := range sent {
node := s.node
for _, d := range records.Assigned[node] { for _, d := range records.Assigned[node] {
body, err := json.Marshal(broker.MembershipFor(node, d, where)) membership := broker.MembershipFor(node, d, where)
membership.Mesh = s.declared.Mesh
for requirement, given := range s.declared.Received[d.Module] {
raw, err := json.Marshal(given)
if err != nil {
return err
}
if membership.Receives == nil {
membership.Receives = map[string]json.RawMessage{}
}
membership.Receives[requirement] = raw
}
body, err := json.Marshal(membership)
if err != nil { if err != nil {
return err return err
} }
-8
View File
@@ -129,14 +129,6 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
// Half of either shape: the command says its usage, which names both shapes, and that is // Half of either shape: the command says its usage, which names both shapes, and that is
// the answer the caller needs. // the answer the caller needs.
return []string{"rotate"}, nil return []string{"rotate"}, nil
case "issue":
// The same act as `module issue` at a shell (novox/hq design 25 §4): the account is minted
// into the mesh's records and delivered at the machine's next push, which is the caller's to
// ask for — so the mesh is never pushed as a side effect of a credential.
if err := need("node", "module"); err != nil {
return nil, err
}
return []string{"module", "issue", str("module"), "--node", str("node")}, nil
case "build": case "build":
if err := need("repository"); err != nil { if err := need("repository"); err != nil {
return nil, err return nil, err
-16
View File
@@ -73,22 +73,6 @@ func TestRotateTakesAProvisionOrAnOwnSecret(t *testing.T) {
} }
} }
// `issue` is `module issue` at a shell: the module and the machine, and nothing that would push. A
// module's bus account was mintable only from the controller's command line, so an agent working
// through the tools could not finish a rollout that gave a module one (novox/hq issue 191).
func TestIssueGivesAModuleItsAccountOnAMachine(t *testing.T) {
argv, err := argvFor("issue", map[string]any{"node": "ace", "module": "route-proxy"})
if err != nil {
t.Fatal(err)
}
if strings.Join(argv, " ") != "module issue route-proxy --node ace" {
t.Fatalf("issue runs %v", argv)
}
if _, err := argvFor("issue", map[string]any{"module": "route-proxy"}); err == nil {
t.Error("an account was issued without saying which machine reads it")
}
}
// A required argument missing is refused in the verb's own words, before anything runs. // A required argument missing is refused in the verb's own words, before anything runs.
func TestAVerbMissingWhatItNeedsIsRefused(t *testing.T) { func TestAVerbMissingWhatItNeedsIsRefused(t *testing.T) {
if _, err := argvFor("node", map[string]any{}); err == nil || !strings.Contains(err.Error(), `node needs "node"`) { if _, err := argvFor("node", map[string]any{}); err == nil || !strings.Contains(err.Error(), `node needs "node"`) {
+6
View File
@@ -25,6 +25,12 @@ type sendable struct {
// Adoption is nil for a converged node, and then the body is byte for byte what it was before // Adoption is nil for a converged node, and then the body is byte for byte what it was before
// adoption existed: an older host parses the envelope strictly and would refuse the key. // adoption existed: an older host parses the envelope strictly and would refuse the key.
Adoption *adoptionEnvelope Adoption *adoptionEnvelope
// Received and Mesh are not sent in the declaration. They are what this machine's memberships
// are issued with on the bus (novox/hq ADR 0167): each module's received contributions, from
// the same composition as its received files, and every machine's private-network address.
Received map[string]map[string][]catalogue.Contribution
Mesh []string
} }
// adoptionEnvelope is what an adopted node is told about its mode. Taken is every module taken on // adoptionEnvelope is what an adopted node is told about its mode. Taken is every module taken on
+132
View File
@@ -0,0 +1,132 @@
package main
import (
"encoding/json"
"fmt"
"log"
"os"
"strings"
"sync/atomic"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
)
// What the mesh issued this proxy, read on the bus (novox/hq ADR 0160, ADR 0167).
//
// **The proxy is told, not left to work it out.** Its membership carries the routes it is given —
// the same contributions its file is written from — and every machine's address on the private
// network, which is who may be served an internal name. Read once at connect and followed, so a
// route added or a machine joining reaches a running proxy without a restart.
// credential is the bus account the mesh delivered as this module's own secret named broker.
type credential struct {
URL string `json:"url"`
Fingerprint string `json:"fingerprint"`
Node string `json:"node"`
Module string `json:"module"`
User string `json:"user"`
Password string `json:"password"`
}
// followMembership connects with the credential in path and applies every membership the mesh
// issues this proxy. It retries the first connection for as long as it takes: a proxy that started
// before the bus keeps serving the file, and takes the bus when it answers.
func followMembership(path string, held *table, fromBus *atomic.Bool) {
for {
err := followOnce(path, held, fromBus)
if err == nil {
return
}
log.Printf("cannot follow this proxy's membership, serving the file meanwhile: %v", err)
time.Sleep(30 * time.Second)
}
}
func followOnce(path string, held *table, fromBus *atomic.Bool) error {
raw, err := os.ReadFile(path)
if err != nil {
return err
}
var cred credential
if err := json.Unmarshal(raw, &cred); err != nil {
return fmt.Errorf("the broker credential is not one: %w", err)
}
if cred.Node == "" || cred.Module == "" {
return fmt.Errorf("the broker credential names no node or module, so it has no membership")
}
opts := []nats.Option{
nats.Name(cred.Node + "." + cred.Module),
nats.UserInfo(cred.User, cred.Password),
// Its own inbox, and nothing wider: every principal is granted `_INBOX.<its user>.>` alone.
nats.CustomInboxPrefix("_INBOX." + cred.User),
// The bus being restarted is an upgrade, not a reason to stop following.
nats.MaxReconnects(-1),
}
if strings.TrimSpace(cred.Fingerprint) != "" {
opts = append(opts, nats.Secure(broker.PinnedToFingerprint(cred.Fingerprint)))
}
conn, err := nats.Connect(cred.URL, opts...)
if err != nil {
return fmt.Errorf("connecting to the bus at %s: %w", broker.BareAddress(cred.URL), err)
}
subject := broker.MembershipSubject(cred.Node, cred.Module)
apply := func(body []byte) {
var issued broker.Membership
if err := json.Unmarshal(body, &issued); err != nil {
log.Printf("a membership arrived that is not one: %v", err)
return
}
if took := applyMembership(issued, held); took && !fromBus.Swap(true) {
log.Printf("routes now come from this proxy's membership on %s", subject)
}
}
// Followed first, read second: an issue landing between the two is applied, not missed.
if _, err := conn.Subscribe(subject, func(m *nats.Msg) { apply(m.Data) }); err != nil {
conn.Close()
return fmt.Errorf("cannot follow %s: %w", subject, err)
}
// The subject-addressed direct get: the one request this account may make of the stream.
got, err := conn.Request("$JS.API.DIRECT.GET."+broker.AssignmentsStream+"."+subject, nil, 5*time.Second)
switch {
case err != nil:
log.Printf("cannot read the membership issued on %s yet (%v); following it", subject, err)
case got.Header.Get("Status") != "" || len(got.Data) == 0:
log.Printf("no membership issued on %s yet; serving the file until one is", subject)
default:
apply(got.Data)
}
return nil
}
// applyMembership serves what a membership says, and says whether it said anything about routes.
//
// A membership with no routes in it is one from a controller older than ADR 0167, and the file stays
// the source rather than every route being withdrawn because a field was absent.
func applyMembership(issued broker.Membership, held *table) bool {
raw, carries := issued.Receives["route"]
if !carries {
return false
}
var contributions []contribution
if err := json.Unmarshal(raw, &contributions); err != nil {
log.Printf("the routes in this proxy's membership are not contributions, keeping what is served: %v", err)
return false
}
inside, err := sourcesOf(issued.Mesh)
if err != nil {
log.Printf("the mesh in this proxy's membership is unreadable, keeping what is served: %v", err)
return false
}
routes, public := routesOf(contributions)
held.set(routes, public)
held.setInside(inside)
log.Printf("serving %d route(s) from the membership, internal names to %d machine(s): %s",
len(routes), len(inside), strings.Join(held.names(), ", "))
return true
}
+174 -28
View File
@@ -54,12 +54,14 @@ import (
"net" "net"
"net/http" "net/http"
"net/http/httputil" "net/http/httputil"
"net/netip"
"net/url" "net/url"
"os" "os"
"path/filepath" "path/filepath"
"sort" "sort"
"strings" "strings"
"sync" "sync"
"sync/atomic"
"time" "time"
"golang.org/x/crypto/acme" "golang.org/x/crypto/acme"
@@ -196,6 +198,63 @@ type table struct {
// pass ACME's own validation (it has no public DNS to prove it against), so asking for it is // pass ACME's own validation (it has no public DNS to prove it against), so asking for it is
// not merely pointless but the failing order onlyWhatTheMeshSaid exists to prevent. // not merely pointless but the failing order onlyWhatTheMeshSaid exists to prevent.
public map[string]bool public map[string]bool
// inside is where a request must come from to be served a name that is only internal: every
// machine's address on the private network, as the mesh issued it in this proxy's membership
// (novox/hq ADR 0167). Empty until it is issued, and then only the machine itself is inside.
inside sources
}
// sources is who may be served an internal name: the private network's addresses as the mesh
// issued them. The machine itself is always inside — anything on a machine may call anything on it
// (novox/hq ADR 0144) — so loopback needs no entry.
type sources []netip.Prefix
// sourcesOf reads the addresses the mesh issued, each a single address or a range. One that does
// not parse is an error, not an entry skipped: the proxy would otherwise serve internal names to
// fewer machines than the mesh said, and say nothing.
func sourcesOf(mesh []string) (sources, error) {
var out sources
for _, entry := range mesh {
entry = strings.TrimSpace(entry)
if prefix, err := netip.ParsePrefix(entry); err == nil {
out = append(out, prefix.Masked())
continue
}
addr, err := netip.ParseAddr(entry)
if err != nil {
return nil, fmt.Errorf("%q is not an address on the private network", entry)
}
addr = addr.Unmap()
out = append(out, netip.PrefixFrom(addr, addr.BitLen()))
}
return out, nil
}
// holds says whether a request from this remote address came from the mesh or the machine itself.
//
// **By source, which the mesh's guard deliberately is not** — it names interfaces, because a source
// address can be claimed by whoever sends the packet. The proxy cannot see the interface a request
// arrived on, and here the claim does not carry: a connection needs its replies, and replies to a
// mesh address leave by the tunnel, never back to the claimant.
func (s sources) holds(remote string) bool {
host := remote
if h, _, err := net.SplitHostPort(remote); err == nil {
host = h
}
addr, err := netip.ParseAddr(host)
if err != nil {
return false
}
addr = addr.Unmap()
if addr.IsLoopback() {
return true
}
for _, prefix := range s {
if prefix.Contains(addr) {
return true
}
}
return false
} }
func (t *table) set(routes map[string][]rule, public map[string]bool) { func (t *table) set(routes map[string][]rule, public map[string]bool) {
@@ -314,6 +373,41 @@ func bareHost(host string) string {
return strings.ToLower(host) return strings.ToLower(host)
} }
// hiddenFrom says whether this host must look unrouted to a request from this address: it is
// only an internal name, and the request did not come from the private network.
//
// **The proxy is the only way in to a routed endpoint, so it is what makes `internal` true**
// (novox/hq ADR 0138, issue 191). It answers public names on the same listeners, so a request from
// anywhere can carry any Host header; a name being internal keeps nobody out unless this check does.
// Answered exactly as a name that was never routed, so an outsider learns nothing from asking.
func (t *table) hiddenFrom(host, remote string) bool {
if !t.eligibleForInternalACME(host) {
return false
}
t.mu.RLock()
defer t.mu.RUnlock()
return !t.inside.holds(remote)
}
// setInside replaces who the mesh is, as the membership said.
func (t *table) setInside(inside sources) {
t.mu.Lock()
t.inside = inside
t.mu.Unlock()
}
// namesSeenFrom is what this proxy says it serves to a request from this address — every routed
// name, less the internal-only ones when the request came from outside.
func (t *table) namesSeenFrom(remote string) []string {
out := []string{}
for _, name := range t.names() {
if !t.hiddenFrom(name, remote) {
out = append(out, name)
}
}
return out
}
func (t *table) names() []string { func (t *table) names() []string {
t.mu.RLock() t.mu.RLock()
defer t.mu.RUnlock() defer t.mu.RUnlock()
@@ -343,7 +437,20 @@ func run() error {
} }
held := newTable() held := newTable()
// **The bus first, the file until it has spoken** (novox/hq ADR 0167). The membership carries
// the routes and who the mesh is; the file carries the routes alone, so while the proxy reads
// it an internal name is served to this machine and to nobody else — refused, never opened.
fromBus := &atomic.Bool{}
if credential := strings.TrimSpace(os.Getenv("MESH_BROKER_FILE")); credential != "" {
go followMembership(credential, held, fromBus)
} else {
log.Printf("MESH_BROKER_FILE is not set: routes come from %s alone, and a name that is only "+
"internal is served to this machine alone", path)
}
read := func() { read := func() {
if fromBus.Load() {
return
}
routes, public, err := routesFrom(path) routes, public, err := routesFrom(path)
if err != nil { if err != nil {
// Kept serving what it had. A file being rewritten is momentarily unreadable, and // Kept serving what it had. A file being rewritten is momentarily unreadable, and
@@ -422,19 +529,7 @@ func run() error {
}() }()
tlsConfig := publicManager.TLSConfig() tlsConfig := publicManager.TLSConfig()
if internalManager != nil { tlsConfig.GetCertificate = certificateFor(held, tlsConfig.GetCertificate, internalManager)
// Dispatched by which authority may certify this name at all — the same question
// eligibleForInternalACME already answers, asked once more at handshake time rather than
// only when an order is placed, since a cached certificate is served here on every request
// and never goes through HostPolicy again.
fromPublic, fromInternal := tlsConfig.GetCertificate, internalManager.TLSConfig().GetCertificate
tlsConfig.GetCertificate = func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
if held.eligibleForInternalACME(hello.ServerName) {
return fromInternal(hello)
}
return fromPublic(hello)
}
}
server := &http.Server{ server := &http.Server{
Addr: secure, Addr: secure,
@@ -592,6 +687,33 @@ func forThisAuthority(cache, directory string, root []byte) string {
return filepath.Join(cache, hex.EncodeToString(sum[:])[:16]) return filepath.Join(cache, hex.EncodeToString(sum[:])[:16])
} }
// certificateFor picks the certificate a handshake is answered with.
//
// Dispatched by which authority may certify this name at all — the same question
// eligibleForInternalACME already answers, asked once more at handshake time rather than only when
// an order is placed, since a cached certificate is served here on every request and never goes
// through HostPolicy again. And refused, exactly as an unrouted name is, to a client outside the
// private network asking for a name that is only internal: the certificate would name it.
func certificateFor(held *table, fromPublic func(*tls.ClientHelloInfo) (*tls.Certificate, error),
internalManager *autocert.Manager) func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
var fromInternal func(*tls.ClientHelloInfo) (*tls.Certificate, error)
if internalManager != nil {
fromInternal = internalManager.TLSConfig().GetCertificate
}
return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
if held.eligibleForInternalACME(hello.ServerName) {
if hello.Conn != nil && held.hiddenFrom(hello.ServerName, hello.Conn.RemoteAddr().String()) {
return nil, fmt.Errorf("no public route for %q in this mesh, so no certificate is asked for",
hello.ServerName)
}
if fromInternal != nil {
return fromInternal(hello)
}
}
return fromPublic(hello)
}
}
// newTable is an empty routing table. // newTable is an empty routing table.
func newTable() *table { func newTable() *table {
return &table{to: map[string][]rule{}} return &table{to: map[string][]rule{}}
@@ -600,8 +722,9 @@ func newTable() *table {
// handler is the proxy itself, separated so it can be driven by a test without a listener. // handler is the proxy itself, separated so it can be driven by a test without a listener.
func handler(held *table) http.Handler { func handler(held *table) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hidden := held.hiddenFrom(r.Host, r.RemoteAddr)
matched, known := held.find(r.Host, r.URL.Path) matched, known := held.find(r.Host, r.URL.Path)
if !known { if hidden || !known {
// **Named, not a bare 404.** A route that was withdrawn and a name that never existed // **Named, not a bare 404.** A route that was withdrawn and a name that never existed
// are different things, and a proxy that says only "not found" makes an operator go // are different things, and a proxy that says only "not found" makes an operator go
// and read the mesh to tell them apart. What it is serving is the answer to both. // and read the mesh to tell them apart. What it is serving is the answer to both.
@@ -611,13 +734,13 @@ func handler(held *table) http.Handler {
// contradiction an operator would have to disbelieve the proxy to get past. // contradiction an operator would have to disbelieve the proxy to get past.
w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.WriteHeader(http.StatusNotFound) w.WriteHeader(http.StatusNotFound)
if held.routed(r.Host) { if !hidden && held.routed(r.Host) {
fmt.Fprintf(w, "%s is served here, but no route covers %q.\n", fmt.Fprintf(w, "%s is served here, but no route covers %q.\n",
bareHost(r.Host), r.URL.Path) bareHost(r.Host), r.URL.Path)
return return
} }
fmt.Fprintf(w, "no route for %q in this mesh.\nserving: %s\n", fmt.Fprintf(w, "no route for %q in this mesh.\nserving: %s\n",
r.Host, strings.Join(held.names(), ", ")) r.Host, strings.Join(held.namesSeenFrom(r.RemoteAddr), ", "))
return return
} }
@@ -716,6 +839,12 @@ func boolByte(b bool) byte {
// routesFrom reads what the mesh wrote and turns it into host → the rules for that host, and // routesFrom reads what the mesh wrote and turns it into host → the rules for that host, and
// which of those hosts is a public name — the second is `name`, ACME-eligible; a host reached // which of those hosts is a public name — the second is `name`, ACME-eligible; a host reached
// only through `internal-name` never appears there. // only through `internal-name` never appears there.
//
// **A route may carry either name, or both** (novox/hq ADR 0138). How far an endpoint reaches
// decides which names the mesh composes, so an endpoint that reaches only the private network
// arrives with an `internal-name` and no `name`. That is a whole route, not a malformed one: it is
// served under its internal name and certified by the internal authority. Only a route with
// neither name has nothing to be served under (novox/hq issue 191).
func routesFrom(path string) (map[string][]rule, map[string]bool, error) { func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
raw, err := os.ReadFile(path) raw, err := os.ReadFile(path)
if err != nil { if err != nil {
@@ -725,17 +854,29 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
if err := json.Unmarshal(raw, &said); err != nil { if err := json.Unmarshal(raw, &said); err != nil {
return nil, nil, err return nil, nil, err
} }
routes, public := routesOf(said.Given)
return routes, public, nil
}
// routesOf turns what the mesh gave into host → the rules for that host, and which hosts are public
// names — the same whether the contributions came in the file or in the membership.
func routesOf(contributions []contribution) (map[string][]rule, map[string]bool) {
out := map[string][]rule{} out := map[string][]rule{}
public := map[string]bool{} public := map[string]bool{}
for _, c := range said.Given { for _, c := range contributions {
name, _ := c.Values["name"].(string) name, _ := c.Values["name"].(string)
if name == "" { name = strings.TrimSpace(name)
internal, _ := c.Values["internal-name"].(string)
internal = strings.TrimSpace(internal)
if name == "" && internal == "" {
log.Printf("%s on %s asked for a route and named nothing; skipped", c.From, c.Node) log.Printf("%s on %s asked for a route and named nothing; skipped", c.From, c.Node)
continue continue
} }
host := strings.ToLower(name) // What the route is called in a log line: its public name when it has one.
public[host] = true called := name
if called == "" {
called = internal
}
made := rule{path: asPath(c.Values["path"])} made := rule{path: asPath(c.Values["path"])}
if p, ok := asWhole(c.Values["priority"]); ok { if p, ok := asWhole(c.Values["priority"]); ok {
@@ -752,7 +893,7 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
if looksLikeACredential(named) { if looksLikeACredential(named) {
log.Printf("%s on %s declared route %q with a credential in the declaration rather "+ log.Printf("%s on %s declared route %q with a credential in the declaration rather "+
"than the name of a secret; the whole route is refused (novox/hq ADR 0108)", "than the name of a secret; the whole route is refused (novox/hq ADR 0108)",
c.From, c.Node, name) c.From, c.Node, called)
continue continue
} }
users, err := usersFrom(named) users, err := usersFrom(named)
@@ -770,7 +911,7 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
port, ok := asPort(c.Values["port"]) port, ok := asPort(c.Values["port"])
if !ok { if !ok {
log.Printf("%s on %s asked for route %q and gave no usable port; skipped", log.Printf("%s on %s asked for route %q and gave no usable port; skipped",
c.From, c.Node, name) c.From, c.Node, called)
continue continue
} }
// Where the mesh says that machine is. Empty means it is this one — a workload beside // Where the mesh says that machine is. Empty means it is this one — a workload beside
@@ -791,7 +932,7 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
} }
if scheme != "http" && scheme != "https" { if scheme != "http" && scheme != "https" {
log.Printf("%s on %s asked for route %q with scheme %q, which is neither http "+ log.Printf("%s on %s asked for route %q with scheme %q, which is neither http "+
"nor https; skipped", c.From, c.Node, name, scheme) "nor https; skipped", c.From, c.Node, called, scheme)
continue continue
} }
made.insecure, _ = c.Values["insecure"].(bool) made.insecure, _ = c.Values["insecure"].(bool)
@@ -802,7 +943,7 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
bytes, whole := asWhole(asked) bytes, whole := asWhole(asked)
if !whole || bytes <= 0 { if !whole || bytes <= 0 {
log.Printf("%s on %s asked for route %q with a max-request-body of %v, which is "+ log.Printf("%s on %s asked for route %q with a max-request-body of %v, which is "+
"not a whole positive number of bytes; skipped", c.From, c.Node, name, asked) "not a whole positive number of bytes; skipped", c.From, c.Node, called, asked)
continue continue
} }
made.maxRequestBody = int64(bytes) made.maxRequestBody = int64(bytes)
@@ -810,19 +951,24 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
made.target = fmt.Sprintf("%s://%s:%d", scheme, at, port) made.target = fmt.Sprintf("%s://%s:%d", scheme, at, port)
} }
if name != "" {
host := strings.ToLower(name)
out[host] = append(out[host], made) out[host] = append(out[host], made)
public[host] = true
}
// The internal-network alias, the same rule under a second host — a predecessor proxy // The internal-network name, the same rule under a second host — a predecessor proxy
// answered both for one route, as a convenience (reaching a service over the VPN without a // answered both for one route, as a convenience (reaching a service over the VPN without a
// public TLS round trip), not as an access boundary; composing it here restores exactly // public TLS round trip), not as an access boundary; composing it here restores exactly
// that, nothing more. Absent whenever the node composed no internal name (novox/hq ADR // that, nothing more. Absent whenever the node composed no internal name (novox/hq ADR
// 0056's internalDomain half) — the same "nothing to join a label to" case the public name // 0056's internalDomain half) — the same "nothing to join a label to" case the public name
// already has. // already has. And the only name, when the endpoint reaches no further than the private
if internal, _ := c.Values["internal-name"].(string); strings.TrimSpace(internal) != "" { // network.
if internal != "" {
out[strings.ToLower(internal)] = append(out[strings.ToLower(internal)], made) out[strings.ToLower(internal)] = append(out[strings.ToLower(internal)], made)
} }
} }
return out, public, nil return out, public
} }
// asWhole is any whole number the mesh wrote, whatever its magnitude. // asWhole is any whole number the mesh wrote, whatever its magnitude.
+206
View File
@@ -0,0 +1,206 @@
package main
import (
"crypto/tls"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/broker"
)
// behind is a workload the proxy can send to, and a table routing one public name and one
// internal-only name to it, with the mesh's machines as the membership would issue them.
func behind(t *testing.T, mesh ...string) *table {
t.Helper()
workload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
io.WriteString(w, "the workload")
}))
t.Cleanup(workload.Close)
at, _ := url.Parse(workload.URL)
host, port, _ := net.SplitHostPort(at.Host)
routes, public, err := routesFrom(write(t, fmt.Sprintf(`{"given":[
{"from":"app","node":"anchor","at":%q,
"values":{"name":"app.example","internal-name":"app.anchor.internal","port":%s}},
{"from":"admin","node":"anchor","at":%q,
"values":{"internal-name":"admin.anchor.internal","port":%s}}
]}`, host, port, host, port)))
if err != nil {
t.Fatal(err)
}
held := newTable()
inside, err := sourcesOf(mesh)
if err != nil {
t.Fatal(err)
}
held.setInside(inside)
held.set(routes, public)
return held
}
// askFrom is what the proxy answers a request for host coming from remote.
func askFrom(held *table, host, remote string) (int, string) {
r := httptest.NewRequest(http.MethodGet, "http://"+host+"/", nil)
r.RemoteAddr = remote
w := httptest.NewRecorder()
handler(held).ServeHTTP(w, r)
return w.Code, w.Body.String()
}
// **An internal-only name is served to the private network and to nobody else** (novox/hq ADR
// 0138, issue 191). The proxy answers public names on the same listeners, so without this a name
// being internal kept nobody out: a request from the internet only had to carry it.
func TestAnInternalOnlyNameIsServedOnlyInsideThePrivateNetwork(t *testing.T) {
held := behind(t, "10.10.0.1", "10.10.0.7")
if code, body := askFrom(held, "admin.anchor.internal", "10.10.0.7:51000"); code != http.StatusOK ||
body != "the workload" {
t.Errorf("a request from the private network was not served: %d %q", code, body)
}
if code, body := askFrom(held, "admin.anchor.internal", "127.0.0.1:51000"); code != http.StatusOK {
t.Errorf("a request from the machine itself was not served: %d %q", code, body)
}
code, body := askFrom(held, "admin.anchor.internal", "203.0.113.9:51000")
if code != http.StatusNotFound {
t.Fatalf("a request from outside the private network reached an internal-only name: %d %q",
code, body)
}
// Answered as a name never routed, and the list of what is served does not name it either —
// otherwise the refusal would tell an outsider exactly what to ask for from inside.
if strings.Contains(strings.SplitN(body, "\n", 2)[1], "admin.anchor.internal") {
t.Errorf("the refusal names the internal-only route to an outsider: %q", body)
}
if !strings.Contains(body, "app.example") {
t.Errorf("the refusal stopped listing the public names: %q", body)
}
}
// The internal name of a route that also has a public one is internal too: served inside, and to
// an outsider only under the public name. Nothing is lost — the outsider has the public name — and a
// name stays one thing whichever route it came from.
func TestAnInternalAliasOfAPublicRouteIsServedInsideOnly(t *testing.T) {
held := behind(t, "10.10.0.1", "10.10.0.7")
if code, body := askFrom(held, "app.anchor.internal", "10.10.0.7:51000"); code != http.StatusOK {
t.Errorf("the internal alias stopped answering the private network: %d %q", code, body)
}
if code, _ := askFrom(held, "app.anchor.internal", "203.0.113.9:51000"); code != http.StatusNotFound {
t.Errorf("the internal alias was served to an outsider: %d", code)
}
if code, _ := askFrom(held, "app.example", "203.0.113.9:51000"); code != http.StatusOK {
t.Errorf("the public name was refused to an outsider: %d", code)
}
}
// Before a membership has said who the mesh is, only the machine itself is inside — refused to
// everyone else, never served to everyone.
func TestUntilTheMeshIsIssuedAnInternalOnlyNameIsServedToTheMachineAlone(t *testing.T) {
held := behind(t)
if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:51000"); code != http.StatusNotFound {
t.Errorf("an internal-only name was served with no private network said: %d", code)
}
if code, _ := askFrom(held, "admin.anchor.internal", "[::1]:51000"); code != http.StatusOK {
t.Errorf("an internal-only name was refused to the machine itself: %d", code)
}
}
type from struct {
net.Conn
remote net.Addr
}
func (c from) RemoteAddr() net.Addr { return c.remote }
// The handshake refuses an internal-only name to an outsider too: the certificate would name it,
// and serving it would answer the question the routing refuses to.
func TestTheHandshakeRefusesAnInternalOnlyNameToAnOutsider(t *testing.T) {
held := behind(t, "10.10.0.1", "10.10.0.7")
served := &tls.Certificate{}
pick := certificateFor(held, func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return served, nil }, nil)
hello := func(name, remote string) *tls.ClientHelloInfo {
addr, _ := net.ResolveTCPAddr("tcp", remote)
return &tls.ClientHelloInfo{ServerName: name, Conn: from{remote: addr}}
}
if _, err := pick(hello("admin.anchor.internal", "203.0.113.9:443")); err == nil {
t.Error("an outsider was handed a certificate for an internal-only name")
}
if got, err := pick(hello("admin.anchor.internal", "10.10.0.7:443")); err != nil || got != served {
t.Errorf("a client on the private network was refused: %v", err)
}
if got, err := pick(hello("app.example", "203.0.113.9:443")); err != nil || got != served {
t.Errorf("a public name was refused to an outsider: %v", err)
}
}
// The mesh is issued as machines' addresses; a range is read as well. One that does not parse is
// refused rather than skipped, so a typo never quietly narrows or widens who is inside.
func TestTheMeshIsReadAsAddressesAndRanges(t *testing.T) {
if _, err := sourcesOf([]string{"10.10.0.1", "not-an-address"}); err == nil {
t.Error("an entry that is not an address was accepted")
}
inside, err := sourcesOf([]string{"10.10.0.1", "fd00::1", "10.20.0.0/24"})
if err != nil {
t.Fatal(err)
}
for remote, want := range map[string]bool{
"10.10.0.1:1": true,
"[::ffff:10.10.0.1]:1": true,
"[fd00::1]:1": true,
"10.20.0.200:1": true,
"10.10.0.2:1": false,
"192.168.1.10:1": false,
"not-an-address": false,
} {
if inside.holds(remote) != want {
t.Errorf("%s inside the mesh: got %v, want %v", remote, !want, want)
}
}
}
// What the mesh issues is what is served: the routes in the membership, internal names to the
// machines it names (novox/hq ADR 0167).
func TestAMembershipIsServedAsIssued(t *testing.T) {
held := newTable()
took := applyMembership(broker.Membership{
Receives: map[string]json.RawMessage{"route": json.RawMessage(`[
{"from":"admin","node":"anchor","at":"anchor.internal",
"values":{"internal-name":"admin.anchor.internal","port":8080}}]`)},
Mesh: []string{"10.10.0.7"},
}, held)
if !took {
t.Fatal("a membership carrying routes was not applied")
}
if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:1"); code == http.StatusNotFound {
t.Error("a machine the membership names was refused the internal-only route")
}
if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.9:1"); code != http.StatusNotFound {
t.Errorf("a machine the membership does not name was served the internal-only route: %d", code)
}
}
// A membership that says nothing about routes is one from a controller that does not issue them,
// and changes nothing: the file stays the source rather than every route being withdrawn.
func TestAMembershipWithoutRoutesLeavesTheFileServing(t *testing.T) {
held := behind(t, "10.10.0.7")
before := held.names()
if applyMembership(broker.Membership{Mesh: []string{"10.10.0.7"}}, held) {
t.Error("a membership without routes was taken as the source of routes")
}
if got := held.names(); strings.Join(got, ",") != strings.Join(before, ",") {
t.Errorf("a membership without routes changed what is served: %v, was %v", got, before)
}
if applyMembership(broker.Membership{
Receives: map[string]json.RawMessage{"route": json.RawMessage(`[]`)},
Mesh: []string{"not-an-address"},
}, held) {
t.Error("a membership whose mesh cannot be read was applied")
}
}
+44
View File
@@ -90,6 +90,50 @@ func TestARouteWithAnInternalNameIsReachableUnderBoth(t *testing.T) {
} }
} }
// A route whose endpoint reaches only the private network carries an internal name and no public
// one (novox/hq ADR 0138), and is served under that name rather than skipped as naming nothing —
// skipping it left every internal-only module unreachable by name (novox/hq issue 191).
func TestARouteWithOnlyAnInternalNameIsServed(t *testing.T) {
routes, public, err := routesFrom(write(t, `{"given":[
{"from":"app","node":"anchor","at":"anchor.internal",
"values":{"internal-name":"App.Anchor.Internal","port":8443,"scheme":"https","insecure":true}}
]}`))
if err != nil {
t.Fatal(err)
}
if targetOf(routes, "app.anchor.internal") != "https://anchor.internal:8443" {
t.Fatalf("the internal-only route is not served: %v", routes)
}
if len(routes) != 1 {
t.Errorf("an internal-only route made hosts it never named: %v", routes)
}
if len(public) != 0 {
t.Errorf("an internal-only route made a name eligible for a public certificate: %v", public)
}
held := newTable()
held.set(routes, public)
if err := onlyInternalNamesTheMeshSaid(held)(context.Background(), "app.anchor.internal"); err != nil {
t.Errorf("the internal authority refused the internal-only route's name: %v", err)
}
if err := onlyWhatTheMeshSaid(held)(context.Background(), "app.anchor.internal"); err == nil {
t.Error("a public certificate was ordered for an internal-only name")
}
}
// A route with neither name has nothing to be served under, and is still skipped.
func TestARouteWithNeitherNameIsSkipped(t *testing.T) {
routes, public, err := routesFrom(write(t, `{"given":[
{"from":"app","node":"anchor","at":"anchor.internal","values":{"internal-name":" ","port":8080}}
]}`))
if err != nil {
t.Fatal(err)
}
if len(routes) != 0 || len(public) != 0 {
t.Errorf("a route that named nothing was served: %v %v", routes, public)
}
}
// A route with no internal-name composed gets no second host — the ordinary case, unchanged. // A route with no internal-name composed gets no second host — the ordinary case, unchanged.
func TestARouteWithNoInternalNameGetsNoAlias(t *testing.T) { func TestARouteWithNoInternalNameGetsNoAlias(t *testing.T) {
routes, _, err := routesFrom(write(t, `{"given":[ routes, _, err := routesFrom(write(t, `{"given":[
+12
View File
@@ -1,6 +1,7 @@
package broker package broker
import ( import (
"encoding/json"
"sort" "sort"
"strings" "strings"
) )
@@ -33,6 +34,17 @@ type Membership struct {
Reaches map[string][]string `json:"reaches,omitempty"` Reaches map[string][]string `json:"reaches,omitempty"`
// Tools is where this instance answers what it serves — the runtime's one verb of its own. // Tools is where this instance answers what it serves — the runtime's one verb of its own.
Tools string `json:"tools"` Tools string `json:"tools"`
// Receives is what this assignment is given for each requirement it receives, by requirement:
// the contributions of every module that asked for it, as the catalogue composed them (novox/hq
// ADR 0167). The same list its received file is written from, so the two cannot disagree; a
// requirement nobody contributed to is an empty list, never absent. Kept as JSON because the
// catalogue owns the shape of a contribution and the bus only carries it.
Receives map[string]json.RawMessage `json:"receives,omitempty"`
// Mesh is every machine's address on the private network — what a rule saying "from the mesh"
// resolves to in the packet filter, issued here from the same list (novox/hq ADR 0167). A
// module that must tell the mesh from the world, the route proxy serving an internal name, reads
// it here rather than keeping a definition of its own.
Mesh []string `json:"mesh,omitempty"`
} }
// Served is one address a tool is answered on. // Served is one address a tool is answered on.
+16 -3
View File
@@ -241,15 +241,21 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
// Owner is kept beside the resources because a resource id cannot be split back into its module: // Owner is kept beside the resources because a resource id cannot be split back into its module:
// a module's name may itself contain a dot. What the mesh adds of its own — an opening, the guard — // a module's name may itself contain a dot. What the mesh adds of its own — an opening, the guard —
// has no owner. // has no owner.
//
// Received is what each module on the machine is given for each requirement it receives — the same
// contributions its received file is written from, kept beside it so the mesh can also issue them
// on the bus in the module's membership (novox/hq ADR 0167). By module, then requirement.
type Composed struct { type Composed struct {
Resources []map[string]any Resources []map[string]any
Owner map[string]string Owner map[string]string
Received map[string]map[string][]Contribution
} }
// Compose is Declaration with the owner of every resource said. // Compose is Declaration with the owner of every resource said.
func (r Resolution) Compose(with Rendering) (Composed, error) { func (r Resolution) Compose(with Rendering) (Composed, error) {
owner := map[string]string{} owner := map[string]string{}
resources, err := r.compose(with, owner) received := map[string]map[string][]Contribution{}
resources, err := r.compose(with, owner, received)
if err != nil { if err != nil {
return Composed{}, err return Composed{}, err
} }
@@ -261,7 +267,7 @@ func (r Resolution) Compose(with Rendering) (Composed, error) {
"sealed": with.BusMembership, "mode": "0600", "sealed": with.BusMembership, "mode": "0600",
}) })
} }
return Composed{Resources: resources, Owner: owner}, nil return Composed{Resources: resources, Owner: owner, Received: received}, nil
} }
// BusMembershipID names the resource carrying a machine's membership for the new bus, and // BusMembershipID names the resource carrying a machine's membership for the new bus, and
@@ -270,7 +276,8 @@ func BusMembershipID() string { return "bus-membership" }
const BusMembershipPath = "/var/lib/mesh/membership-next.json" const BusMembershipPath = "/var/lib/mesh/membership-next.json"
func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[string]any, error) { func (r Resolution) compose(with Rendering, owner map[string]string,
received map[string]map[string][]Contribution) ([]map[string]any, error) {
// Every manifest is placed first (novox/hq ADR 0112): the maps naming where its bindings, // Every manifest is placed first (novox/hq ADR 0112): the maps naming where its bindings,
// credentials and contributions land are resolved against this node's directories, so every // credentials and contributions land are resolved against this node's directories, so every
// reader below — the binding files, the sealed secrets, the grant paths a contribution // reader below — the binding files, the sealed secrets, the grant paths a contribution
@@ -596,6 +603,12 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
return nil, err return nil, err
} }
first = append(first, file) first = append(first, file)
if received[m.Module] == nil {
received[m.Module] = map[string][]Contribution{}
}
// Empty rather than absent when nobody contributed, for the reason the file is
// written empty: "nothing asked" and "never told" want different responses.
received[m.Module][to] = append([]Contribution{}, given[to]...)
} }
if m.Keeps != "" && with.Kept != nil { if m.Keeps != "" && with.Kept != nil {
file, err := keptFile(m.Keeps, with.Kept) file, err := keptFile(m.Keeps, with.Kept)
@@ -0,0 +1,66 @@
package catalogue
import (
"encoding/json"
"reflect"
"testing"
)
// What a provider receives is composed once, and issued twice: as its received file, and in its
// membership on the bus (novox/hq ADR 0167). The two are the same list, so a proxy reading the bus
// and one reading the file serve the same routes — including the port the machine published, which
// is the same-node fix the file already carries.
func TestWhatAProviderReceivesIsTheSameOnTheBusAsInItsFile(t *testing.T) {
gitea := Manifest{
Module: "gitea", Version: "1",
Listens: []Listening{{Port: 3000, Protocol: "tcp", From: FromMesh}},
Contributes: map[string]map[string]any{"route": {"label": "git", "port": 3000}},
Resources: []map[string]any{{
"id": "server", "type": "container", "name": "gitea", "ports": []any{"3000"},
}},
}
r, err := Resolve(shelf(gitea, routeProxy(), stepCA()),
[]string{"gitea", "route-proxy", "step-ca"}, reachable(), World{})
if err != nil {
t.Fatal(err)
}
composed, err := r.Compose(Rendering{Ports: map[string]map[int]int{"gitea": {3000: 20000}}})
if err != nil {
t.Fatal(err)
}
file := fileNamed(composed.Resources, "route-proxy.received-route")
if file == nil {
t.Fatal("the proxy was given no routes file")
}
var written struct {
Given []Contribution `json:"given"`
}
if err := json.Unmarshal([]byte(file["content"].(string)), &written); err != nil {
t.Fatal(err)
}
issued, said := composed.Received["route-proxy"]["route"]
if !said {
t.Fatalf("nothing is issued for the proxy to receive on the bus: %v", composed.Received)
}
// Compared as JSON, which is what both are once they leave the controller.
a, _ := json.Marshal(written.Given)
b, _ := json.Marshal(issued)
var fromFile, fromBus any
_ = json.Unmarshal(a, &fromFile)
_ = json.Unmarshal(b, &fromBus)
if !reflect.DeepEqual(fromFile, fromBus) {
t.Errorf("the bus and the file disagree about the routes:\nfile %s\nbus %s", a, b)
}
if len(issued) != 1 {
t.Fatalf("expected one route on the bus, got %v", issued)
}
if port, ok := asPort(issued[0].Values["port"]); !ok || port != 20000 {
t.Errorf("the bus carries a port nothing listens on: %v", issued[0].Values["port"])
}
// A module that receives nothing is issued nothing to receive.
if _, any := composed.Received["gitea"]; any {
t.Errorf("a module that receives nothing was issued something: %v", composed.Received["gitea"])
}
}
-7
View File
@@ -129,13 +129,6 @@ var ControllerVerbs = []Verb{
"module": "an own secret: the module", "module": "an own secret: the module",
"secret": "an own secret: its name in the module's definition", "secret": "an own secret: its name in the module's definition",
}, nil)}, }, nil)},
{Name: "issue", Description: "Give a module on a machine its account on the bus: minted, and sealed to the " +
"machine as the module's own secret named broker, read at the next push of that machine. For a module " +
"whose definition declares that secret; refused with the reason otherwise. Issued again, it replaces the account.",
Input: schema(map[string]string{
"node": "the machine that runs the module",
"module": "the module's name",
}, []string{"node", "module"})},
{Name: "build", Description: "Have the build machine build a repository. Answers at once with the build's id: " + {Name: "build", Description: "Have the build machine build a repository. Answers at once with the build's id: " +
"`builds` with that id follows it line by line, and the module is registered when the outcome comes.", "`builds` with that id follows it line by line, and the module is registered when the outcome comes.",
Input: schema(map[string]string{ Input: schema(map[string]string{