The proxy is told its routes and the mesh on the bus, and serves internal names to the mesh only (hq issue 191, ADR 0167) #207

Merged
mesh-admin merged 2 commits from jschoubben/an-internal-only-route AGit into main 2026-10-02 07:42:46 +00:00
10 changed files with 682 additions and 43 deletions
+2 -1
View File
@@ -404,7 +404,8 @@ func declarationWith(ctx context.Context, open *stores, node string,
if err != nil { if err != nil {
return sendable{}, err return sendable{}, err
} }
return sendable{Resources: composed.Resources, Adoption: adoption}, nil return sendable{Resources: composed.Resources, Adoption: adoption,
Received: composed.Received, Mesh: with.Mesh}, nil
} }
// renderingFor is everything a node's declaration is composed with, and the node's record. // renderingFor is everything a node's declaration is composed with, and the node's record.
+23 -10
View File
@@ -389,11 +389,7 @@ func pushCommand(ctx context.Context, args []string) error {
fmt.Printf("\n%d node(s) told\n", len(sending)) fmt.Printf("\n%d node(s) told\n", len(sending))
// And each machine's memberships, as every other send does (ADR 0160): a push is the one most // And each machine's memberships, as every other send does (ADR 0160): a push is the one most
// operators run, and on 2026-10-01 it was the one path that issued none. // operators run, and on 2026-10-01 it was the one path that issued none.
var told []string if err := issueMemberships(ctx, open, server, sending); err != nil {
for _, s := range sending {
told = append(told, s.node)
}
if err := issueMemberships(ctx, open, server, told); err != nil {
return err return err
} }
@@ -706,11 +702,15 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
// And every assignment on those machines its membership (novox/hq ADR 0160): composed from the // And every assignment on those machines its membership (novox/hq ADR 0160): composed from the
// same records the bus's accounts are, so what a runtime serves and what its account may are one // same records the bus's accounts are, so what a runtime serves and what its account may are one
// composition. Issued after the declaration, because the runtime it is for arrives with it. // composition. Issued after the declaration, because the runtime it is for arrives with it.
return issueMemberships(ctx, open, server, names) return issueMemberships(ctx, open, server, sending)
} }
// issueMemberships publishes the membership of every module on the named machines. // issueMemberships publishes the membership of every module on the machines just sent.
func issueMemberships(ctx context.Context, open *stores, server *link.Server, names []string) error { //
// Each carries what its module receives and the private network's addresses, from the same
// composition as the declaration it was sent (novox/hq ADR 0167): a provider reads what it is
// given on the bus, and the file written beside it says the same thing.
func issueMemberships(ctx context.Context, open *stores, server *link.Server, sent []readyNode) error {
records, err := open.inventory.BusRecords(ctx) records, err := open.inventory.BusRecords(ctx)
if err != nil { if err != nil {
return err return err
@@ -725,9 +725,22 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, na
// the push stands, the first failure is named once, and the next push tries again. // the push stands, the first failure is named once, and the next push tries again.
issued, failed := 0, 0 issued, failed := 0, 0
var first error var first error
for _, node := range names { for _, s := range sent {
node := s.node
for _, d := range records.Assigned[node] { for _, d := range records.Assigned[node] {
body, err := json.Marshal(broker.MembershipFor(node, d, where)) membership := broker.MembershipFor(node, d, where)
membership.Mesh = s.declared.Mesh
for requirement, given := range s.declared.Received[d.Module] {
raw, err := json.Marshal(given)
if err != nil {
return err
}
if membership.Receives == nil {
membership.Receives = map[string]json.RawMessage{}
}
membership.Receives[requirement] = raw
}
body, err := json.Marshal(membership)
if err != nil { if err != nil {
return err return err
} }
+6
View File
@@ -25,6 +25,12 @@ type sendable struct {
// Adoption is nil for a converged node, and then the body is byte for byte what it was before // Adoption is nil for a converged node, and then the body is byte for byte what it was before
// adoption existed: an older host parses the envelope strictly and would refuse the key. // adoption existed: an older host parses the envelope strictly and would refuse the key.
Adoption *adoptionEnvelope Adoption *adoptionEnvelope
// Received and Mesh are not sent in the declaration. They are what this machine's memberships
// are issued with on the bus (novox/hq ADR 0167): each module's received contributions, from
// the same composition as its received files, and every machine's private-network address.
Received map[string]map[string][]catalogue.Contribution
Mesh []string
} }
// adoptionEnvelope is what an adopted node is told about its mode. Taken is every module taken on // adoptionEnvelope is what an adopted node is told about its mode. Taken is every module taken on
+132
View File
@@ -0,0 +1,132 @@
package main
import (
"encoding/json"
"fmt"
"log"
"os"
"strings"
"sync/atomic"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
)
// What the mesh issued this proxy, read on the bus (novox/hq ADR 0160, ADR 0167).
//
// **The proxy is told, not left to work it out.** Its membership carries the routes it is given —
// the same contributions its file is written from — and every machine's address on the private
// network, which is who may be served an internal name. Read once at connect and followed, so a
// route added or a machine joining reaches a running proxy without a restart.
// credential is the bus account the mesh delivered as this module's own secret named broker.
type credential struct {
URL string `json:"url"`
Fingerprint string `json:"fingerprint"`
Node string `json:"node"`
Module string `json:"module"`
User string `json:"user"`
Password string `json:"password"`
}
// followMembership connects with the credential in path and applies every membership the mesh
// issues this proxy. It retries the first connection for as long as it takes: a proxy that started
// before the bus keeps serving the file, and takes the bus when it answers.
func followMembership(path string, held *table, fromBus *atomic.Bool) {
for {
err := followOnce(path, held, fromBus)
if err == nil {
return
}
log.Printf("cannot follow this proxy's membership, serving the file meanwhile: %v", err)
time.Sleep(30 * time.Second)
}
}
func followOnce(path string, held *table, fromBus *atomic.Bool) error {
raw, err := os.ReadFile(path)
if err != nil {
return err
}
var cred credential
if err := json.Unmarshal(raw, &cred); err != nil {
return fmt.Errorf("the broker credential is not one: %w", err)
}
if cred.Node == "" || cred.Module == "" {
return fmt.Errorf("the broker credential names no node or module, so it has no membership")
}
opts := []nats.Option{
nats.Name(cred.Node + "." + cred.Module),
nats.UserInfo(cred.User, cred.Password),
// Its own inbox, and nothing wider: every principal is granted `_INBOX.<its user>.>` alone.
nats.CustomInboxPrefix("_INBOX." + cred.User),
// The bus being restarted is an upgrade, not a reason to stop following.
nats.MaxReconnects(-1),
}
if strings.TrimSpace(cred.Fingerprint) != "" {
opts = append(opts, nats.Secure(broker.PinnedToFingerprint(cred.Fingerprint)))
}
conn, err := nats.Connect(cred.URL, opts...)
if err != nil {
return fmt.Errorf("connecting to the bus at %s: %w", broker.BareAddress(cred.URL), err)
}
subject := broker.MembershipSubject(cred.Node, cred.Module)
apply := func(body []byte) {
var issued broker.Membership
if err := json.Unmarshal(body, &issued); err != nil {
log.Printf("a membership arrived that is not one: %v", err)
return
}
if took := applyMembership(issued, held); took && !fromBus.Swap(true) {
log.Printf("routes now come from this proxy's membership on %s", subject)
}
}
// Followed first, read second: an issue landing between the two is applied, not missed.
if _, err := conn.Subscribe(subject, func(m *nats.Msg) { apply(m.Data) }); err != nil {
conn.Close()
return fmt.Errorf("cannot follow %s: %w", subject, err)
}
// The subject-addressed direct get: the one request this account may make of the stream.
got, err := conn.Request("$JS.API.DIRECT.GET."+broker.AssignmentsStream+"."+subject, nil, 5*time.Second)
switch {
case err != nil:
log.Printf("cannot read the membership issued on %s yet (%v); following it", subject, err)
case got.Header.Get("Status") != "" || len(got.Data) == 0:
log.Printf("no membership issued on %s yet; serving the file until one is", subject)
default:
apply(got.Data)
}
return nil
}
// applyMembership serves what a membership says, and says whether it said anything about routes.
//
// A membership with no routes in it is one from a controller older than ADR 0167, and the file stays
// the source rather than every route being withdrawn because a field was absent.
func applyMembership(issued broker.Membership, held *table) bool {
raw, carries := issued.Receives["route"]
if !carries {
return false
}
var contributions []contribution
if err := json.Unmarshal(raw, &contributions); err != nil {
log.Printf("the routes in this proxy's membership are not contributions, keeping what is served: %v", err)
return false
}
inside, err := sourcesOf(issued.Mesh)
if err != nil {
log.Printf("the mesh in this proxy's membership is unreadable, keeping what is served: %v", err)
return false
}
routes, public := routesOf(contributions)
held.set(routes, public)
held.setInside(inside)
log.Printf("serving %d route(s) from the membership, internal names to %d machine(s): %s",
len(routes), len(inside), strings.Join(held.names(), ", "))
return true
}
+175 -29
View File
@@ -54,12 +54,14 @@ import (
"net" "net"
"net/http" "net/http"
"net/http/httputil" "net/http/httputil"
"net/netip"
"net/url" "net/url"
"os" "os"
"path/filepath" "path/filepath"
"sort" "sort"
"strings" "strings"
"sync" "sync"
"sync/atomic"
"time" "time"
"golang.org/x/crypto/acme" "golang.org/x/crypto/acme"
@@ -196,6 +198,63 @@ type table struct {
// pass ACME's own validation (it has no public DNS to prove it against), so asking for it is // pass ACME's own validation (it has no public DNS to prove it against), so asking for it is
// not merely pointless but the failing order onlyWhatTheMeshSaid exists to prevent. // not merely pointless but the failing order onlyWhatTheMeshSaid exists to prevent.
public map[string]bool public map[string]bool
// inside is where a request must come from to be served a name that is only internal: every
// machine's address on the private network, as the mesh issued it in this proxy's membership
// (novox/hq ADR 0167). Empty until it is issued, and then only the machine itself is inside.
inside sources
}
// sources is who may be served an internal name: the private network's addresses as the mesh
// issued them. The machine itself is always inside — anything on a machine may call anything on it
// (novox/hq ADR 0144) — so loopback needs no entry.
type sources []netip.Prefix
// sourcesOf reads the addresses the mesh issued, each a single address or a range. One that does
// not parse is an error, not an entry skipped: the proxy would otherwise serve internal names to
// fewer machines than the mesh said, and say nothing.
func sourcesOf(mesh []string) (sources, error) {
var out sources
for _, entry := range mesh {
entry = strings.TrimSpace(entry)
if prefix, err := netip.ParsePrefix(entry); err == nil {
out = append(out, prefix.Masked())
continue
}
addr, err := netip.ParseAddr(entry)
if err != nil {
return nil, fmt.Errorf("%q is not an address on the private network", entry)
}
addr = addr.Unmap()
out = append(out, netip.PrefixFrom(addr, addr.BitLen()))
}
return out, nil
}
// holds says whether a request from this remote address came from the mesh or the machine itself.
//
// **By source, which the mesh's guard deliberately is not** — it names interfaces, because a source
// address can be claimed by whoever sends the packet. The proxy cannot see the interface a request
// arrived on, and here the claim does not carry: a connection needs its replies, and replies to a
// mesh address leave by the tunnel, never back to the claimant.
func (s sources) holds(remote string) bool {
host := remote
if h, _, err := net.SplitHostPort(remote); err == nil {
host = h
}
addr, err := netip.ParseAddr(host)
if err != nil {
return false
}
addr = addr.Unmap()
if addr.IsLoopback() {
return true
}
for _, prefix := range s {
if prefix.Contains(addr) {
return true
}
}
return false
} }
func (t *table) set(routes map[string][]rule, public map[string]bool) { func (t *table) set(routes map[string][]rule, public map[string]bool) {
@@ -314,6 +373,41 @@ func bareHost(host string) string {
return strings.ToLower(host) return strings.ToLower(host)
} }
// hiddenFrom says whether this host must look unrouted to a request from this address: it is
// only an internal name, and the request did not come from the private network.
//
// **The proxy is the only way in to a routed endpoint, so it is what makes `internal` true**
// (novox/hq ADR 0138, issue 191). It answers public names on the same listeners, so a request from
// anywhere can carry any Host header; a name being internal keeps nobody out unless this check does.
// Answered exactly as a name that was never routed, so an outsider learns nothing from asking.
func (t *table) hiddenFrom(host, remote string) bool {
if !t.eligibleForInternalACME(host) {
return false
}
t.mu.RLock()
defer t.mu.RUnlock()
return !t.inside.holds(remote)
}
// setInside replaces who the mesh is, as the membership said.
func (t *table) setInside(inside sources) {
t.mu.Lock()
t.inside = inside
t.mu.Unlock()
}
// namesSeenFrom is what this proxy says it serves to a request from this address — every routed
// name, less the internal-only ones when the request came from outside.
func (t *table) namesSeenFrom(remote string) []string {
out := []string{}
for _, name := range t.names() {
if !t.hiddenFrom(name, remote) {
out = append(out, name)
}
}
return out
}
func (t *table) names() []string { func (t *table) names() []string {
t.mu.RLock() t.mu.RLock()
defer t.mu.RUnlock() defer t.mu.RUnlock()
@@ -343,7 +437,20 @@ func run() error {
} }
held := newTable() held := newTable()
// **The bus first, the file until it has spoken** (novox/hq ADR 0167). The membership carries
// the routes and who the mesh is; the file carries the routes alone, so while the proxy reads
// it an internal name is served to this machine and to nobody else — refused, never opened.
fromBus := &atomic.Bool{}
if credential := strings.TrimSpace(os.Getenv("MESH_BROKER_FILE")); credential != "" {
go followMembership(credential, held, fromBus)
} else {
log.Printf("MESH_BROKER_FILE is not set: routes come from %s alone, and a name that is only "+
"internal is served to this machine alone", path)
}
read := func() { read := func() {
if fromBus.Load() {
return
}
routes, public, err := routesFrom(path) routes, public, err := routesFrom(path)
if err != nil { if err != nil {
// Kept serving what it had. A file being rewritten is momentarily unreadable, and // Kept serving what it had. A file being rewritten is momentarily unreadable, and
@@ -422,19 +529,7 @@ func run() error {
}() }()
tlsConfig := publicManager.TLSConfig() tlsConfig := publicManager.TLSConfig()
if internalManager != nil { tlsConfig.GetCertificate = certificateFor(held, tlsConfig.GetCertificate, internalManager)
// Dispatched by which authority may certify this name at all — the same question
// eligibleForInternalACME already answers, asked once more at handshake time rather than
// only when an order is placed, since a cached certificate is served here on every request
// and never goes through HostPolicy again.
fromPublic, fromInternal := tlsConfig.GetCertificate, internalManager.TLSConfig().GetCertificate
tlsConfig.GetCertificate = func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
if held.eligibleForInternalACME(hello.ServerName) {
return fromInternal(hello)
}
return fromPublic(hello)
}
}
server := &http.Server{ server := &http.Server{
Addr: secure, Addr: secure,
@@ -592,6 +687,33 @@ func forThisAuthority(cache, directory string, root []byte) string {
return filepath.Join(cache, hex.EncodeToString(sum[:])[:16]) return filepath.Join(cache, hex.EncodeToString(sum[:])[:16])
} }
// certificateFor picks the certificate a handshake is answered with.
//
// Dispatched by which authority may certify this name at all — the same question
// eligibleForInternalACME already answers, asked once more at handshake time rather than only when
// an order is placed, since a cached certificate is served here on every request and never goes
// through HostPolicy again. And refused, exactly as an unrouted name is, to a client outside the
// private network asking for a name that is only internal: the certificate would name it.
func certificateFor(held *table, fromPublic func(*tls.ClientHelloInfo) (*tls.Certificate, error),
internalManager *autocert.Manager) func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
var fromInternal func(*tls.ClientHelloInfo) (*tls.Certificate, error)
if internalManager != nil {
fromInternal = internalManager.TLSConfig().GetCertificate
}
return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
if held.eligibleForInternalACME(hello.ServerName) {
if hello.Conn != nil && held.hiddenFrom(hello.ServerName, hello.Conn.RemoteAddr().String()) {
return nil, fmt.Errorf("no public route for %q in this mesh, so no certificate is asked for",
hello.ServerName)
}
if fromInternal != nil {
return fromInternal(hello)
}
}
return fromPublic(hello)
}
}
// newTable is an empty routing table. // newTable is an empty routing table.
func newTable() *table { func newTable() *table {
return &table{to: map[string][]rule{}} return &table{to: map[string][]rule{}}
@@ -600,8 +722,9 @@ func newTable() *table {
// handler is the proxy itself, separated so it can be driven by a test without a listener. // handler is the proxy itself, separated so it can be driven by a test without a listener.
func handler(held *table) http.Handler { func handler(held *table) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hidden := held.hiddenFrom(r.Host, r.RemoteAddr)
matched, known := held.find(r.Host, r.URL.Path) matched, known := held.find(r.Host, r.URL.Path)
if !known { if hidden || !known {
// **Named, not a bare 404.** A route that was withdrawn and a name that never existed // **Named, not a bare 404.** A route that was withdrawn and a name that never existed
// are different things, and a proxy that says only "not found" makes an operator go // are different things, and a proxy that says only "not found" makes an operator go
// and read the mesh to tell them apart. What it is serving is the answer to both. // and read the mesh to tell them apart. What it is serving is the answer to both.
@@ -611,13 +734,13 @@ func handler(held *table) http.Handler {
// contradiction an operator would have to disbelieve the proxy to get past. // contradiction an operator would have to disbelieve the proxy to get past.
w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.WriteHeader(http.StatusNotFound) w.WriteHeader(http.StatusNotFound)
if held.routed(r.Host) { if !hidden && held.routed(r.Host) {
fmt.Fprintf(w, "%s is served here, but no route covers %q.\n", fmt.Fprintf(w, "%s is served here, but no route covers %q.\n",
bareHost(r.Host), r.URL.Path) bareHost(r.Host), r.URL.Path)
return return
} }
fmt.Fprintf(w, "no route for %q in this mesh.\nserving: %s\n", fmt.Fprintf(w, "no route for %q in this mesh.\nserving: %s\n",
r.Host, strings.Join(held.names(), ", ")) r.Host, strings.Join(held.namesSeenFrom(r.RemoteAddr), ", "))
return return
} }
@@ -716,6 +839,12 @@ func boolByte(b bool) byte {
// routesFrom reads what the mesh wrote and turns it into host → the rules for that host, and // routesFrom reads what the mesh wrote and turns it into host → the rules for that host, and
// which of those hosts is a public name — the second is `name`, ACME-eligible; a host reached // which of those hosts is a public name — the second is `name`, ACME-eligible; a host reached
// only through `internal-name` never appears there. // only through `internal-name` never appears there.
//
// **A route may carry either name, or both** (novox/hq ADR 0138). How far an endpoint reaches
// decides which names the mesh composes, so an endpoint that reaches only the private network
// arrives with an `internal-name` and no `name`. That is a whole route, not a malformed one: it is
// served under its internal name and certified by the internal authority. Only a route with
// neither name has nothing to be served under (novox/hq issue 191).
func routesFrom(path string) (map[string][]rule, map[string]bool, error) { func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
raw, err := os.ReadFile(path) raw, err := os.ReadFile(path)
if err != nil { if err != nil {
@@ -725,17 +854,29 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
if err := json.Unmarshal(raw, &said); err != nil { if err := json.Unmarshal(raw, &said); err != nil {
return nil, nil, err return nil, nil, err
} }
routes, public := routesOf(said.Given)
return routes, public, nil
}
// routesOf turns what the mesh gave into host → the rules for that host, and which hosts are public
// names — the same whether the contributions came in the file or in the membership.
func routesOf(contributions []contribution) (map[string][]rule, map[string]bool) {
out := map[string][]rule{} out := map[string][]rule{}
public := map[string]bool{} public := map[string]bool{}
for _, c := range said.Given { for _, c := range contributions {
name, _ := c.Values["name"].(string) name, _ := c.Values["name"].(string)
if name == "" { name = strings.TrimSpace(name)
internal, _ := c.Values["internal-name"].(string)
internal = strings.TrimSpace(internal)
if name == "" && internal == "" {
log.Printf("%s on %s asked for a route and named nothing; skipped", c.From, c.Node) log.Printf("%s on %s asked for a route and named nothing; skipped", c.From, c.Node)
continue continue
} }
host := strings.ToLower(name) // What the route is called in a log line: its public name when it has one.
public[host] = true called := name
if called == "" {
called = internal
}
made := rule{path: asPath(c.Values["path"])} made := rule{path: asPath(c.Values["path"])}
if p, ok := asWhole(c.Values["priority"]); ok { if p, ok := asWhole(c.Values["priority"]); ok {
@@ -752,7 +893,7 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
if looksLikeACredential(named) { if looksLikeACredential(named) {
log.Printf("%s on %s declared route %q with a credential in the declaration rather "+ log.Printf("%s on %s declared route %q with a credential in the declaration rather "+
"than the name of a secret; the whole route is refused (novox/hq ADR 0108)", "than the name of a secret; the whole route is refused (novox/hq ADR 0108)",
c.From, c.Node, name) c.From, c.Node, called)
continue continue
} }
users, err := usersFrom(named) users, err := usersFrom(named)
@@ -770,7 +911,7 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
port, ok := asPort(c.Values["port"]) port, ok := asPort(c.Values["port"])
if !ok { if !ok {
log.Printf("%s on %s asked for route %q and gave no usable port; skipped", log.Printf("%s on %s asked for route %q and gave no usable port; skipped",
c.From, c.Node, name) c.From, c.Node, called)
continue continue
} }
// Where the mesh says that machine is. Empty means it is this one — a workload beside // Where the mesh says that machine is. Empty means it is this one — a workload beside
@@ -791,7 +932,7 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
} }
if scheme != "http" && scheme != "https" { if scheme != "http" && scheme != "https" {
log.Printf("%s on %s asked for route %q with scheme %q, which is neither http "+ log.Printf("%s on %s asked for route %q with scheme %q, which is neither http "+
"nor https; skipped", c.From, c.Node, name, scheme) "nor https; skipped", c.From, c.Node, called, scheme)
continue continue
} }
made.insecure, _ = c.Values["insecure"].(bool) made.insecure, _ = c.Values["insecure"].(bool)
@@ -802,7 +943,7 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
bytes, whole := asWhole(asked) bytes, whole := asWhole(asked)
if !whole || bytes <= 0 { if !whole || bytes <= 0 {
log.Printf("%s on %s asked for route %q with a max-request-body of %v, which is "+ log.Printf("%s on %s asked for route %q with a max-request-body of %v, which is "+
"not a whole positive number of bytes; skipped", c.From, c.Node, name, asked) "not a whole positive number of bytes; skipped", c.From, c.Node, called, asked)
continue continue
} }
made.maxRequestBody = int64(bytes) made.maxRequestBody = int64(bytes)
@@ -810,19 +951,24 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
made.target = fmt.Sprintf("%s://%s:%d", scheme, at, port) made.target = fmt.Sprintf("%s://%s:%d", scheme, at, port)
} }
out[host] = append(out[host], made) if name != "" {
host := strings.ToLower(name)
out[host] = append(out[host], made)
public[host] = true
}
// The internal-network alias, the same rule under a second host — a predecessor proxy // The internal-network name, the same rule under a second host — a predecessor proxy
// answered both for one route, as a convenience (reaching a service over the VPN without a // answered both for one route, as a convenience (reaching a service over the VPN without a
// public TLS round trip), not as an access boundary; composing it here restores exactly // public TLS round trip), not as an access boundary; composing it here restores exactly
// that, nothing more. Absent whenever the node composed no internal name (novox/hq ADR // that, nothing more. Absent whenever the node composed no internal name (novox/hq ADR
// 0056's internalDomain half) — the same "nothing to join a label to" case the public name // 0056's internalDomain half) — the same "nothing to join a label to" case the public name
// already has. // already has. And the only name, when the endpoint reaches no further than the private
if internal, _ := c.Values["internal-name"].(string); strings.TrimSpace(internal) != "" { // network.
if internal != "" {
out[strings.ToLower(internal)] = append(out[strings.ToLower(internal)], made) out[strings.ToLower(internal)] = append(out[strings.ToLower(internal)], made)
} }
} }
return out, public, nil return out, public
} }
// asWhole is any whole number the mesh wrote, whatever its magnitude. // asWhole is any whole number the mesh wrote, whatever its magnitude.
+206
View File
@@ -0,0 +1,206 @@
package main
import (
"crypto/tls"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/broker"
)
// behind is a workload the proxy can send to, and a table routing one public name and one
// internal-only name to it, with the mesh's machines as the membership would issue them.
func behind(t *testing.T, mesh ...string) *table {
t.Helper()
workload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
io.WriteString(w, "the workload")
}))
t.Cleanup(workload.Close)
at, _ := url.Parse(workload.URL)
host, port, _ := net.SplitHostPort(at.Host)
routes, public, err := routesFrom(write(t, fmt.Sprintf(`{"given":[
{"from":"app","node":"anchor","at":%q,
"values":{"name":"app.example","internal-name":"app.anchor.internal","port":%s}},
{"from":"admin","node":"anchor","at":%q,
"values":{"internal-name":"admin.anchor.internal","port":%s}}
]}`, host, port, host, port)))
if err != nil {
t.Fatal(err)
}
held := newTable()
inside, err := sourcesOf(mesh)
if err != nil {
t.Fatal(err)
}
held.setInside(inside)
held.set(routes, public)
return held
}
// askFrom is what the proxy answers a request for host coming from remote.
func askFrom(held *table, host, remote string) (int, string) {
r := httptest.NewRequest(http.MethodGet, "http://"+host+"/", nil)
r.RemoteAddr = remote
w := httptest.NewRecorder()
handler(held).ServeHTTP(w, r)
return w.Code, w.Body.String()
}
// **An internal-only name is served to the private network and to nobody else** (novox/hq ADR
// 0138, issue 191). The proxy answers public names on the same listeners, so without this a name
// being internal kept nobody out: a request from the internet only had to carry it.
func TestAnInternalOnlyNameIsServedOnlyInsideThePrivateNetwork(t *testing.T) {
held := behind(t, "10.10.0.1", "10.10.0.7")
if code, body := askFrom(held, "admin.anchor.internal", "10.10.0.7:51000"); code != http.StatusOK ||
body != "the workload" {
t.Errorf("a request from the private network was not served: %d %q", code, body)
}
if code, body := askFrom(held, "admin.anchor.internal", "127.0.0.1:51000"); code != http.StatusOK {
t.Errorf("a request from the machine itself was not served: %d %q", code, body)
}
code, body := askFrom(held, "admin.anchor.internal", "203.0.113.9:51000")
if code != http.StatusNotFound {
t.Fatalf("a request from outside the private network reached an internal-only name: %d %q",
code, body)
}
// Answered as a name never routed, and the list of what is served does not name it either —
// otherwise the refusal would tell an outsider exactly what to ask for from inside.
if strings.Contains(strings.SplitN(body, "\n", 2)[1], "admin.anchor.internal") {
t.Errorf("the refusal names the internal-only route to an outsider: %q", body)
}
if !strings.Contains(body, "app.example") {
t.Errorf("the refusal stopped listing the public names: %q", body)
}
}
// The internal name of a route that also has a public one is internal too: served inside, and to
// an outsider only under the public name. Nothing is lost — the outsider has the public name — and a
// name stays one thing whichever route it came from.
func TestAnInternalAliasOfAPublicRouteIsServedInsideOnly(t *testing.T) {
held := behind(t, "10.10.0.1", "10.10.0.7")
if code, body := askFrom(held, "app.anchor.internal", "10.10.0.7:51000"); code != http.StatusOK {
t.Errorf("the internal alias stopped answering the private network: %d %q", code, body)
}
if code, _ := askFrom(held, "app.anchor.internal", "203.0.113.9:51000"); code != http.StatusNotFound {
t.Errorf("the internal alias was served to an outsider: %d", code)
}
if code, _ := askFrom(held, "app.example", "203.0.113.9:51000"); code != http.StatusOK {
t.Errorf("the public name was refused to an outsider: %d", code)
}
}
// Before a membership has said who the mesh is, only the machine itself is inside — refused to
// everyone else, never served to everyone.
func TestUntilTheMeshIsIssuedAnInternalOnlyNameIsServedToTheMachineAlone(t *testing.T) {
held := behind(t)
if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:51000"); code != http.StatusNotFound {
t.Errorf("an internal-only name was served with no private network said: %d", code)
}
if code, _ := askFrom(held, "admin.anchor.internal", "[::1]:51000"); code != http.StatusOK {
t.Errorf("an internal-only name was refused to the machine itself: %d", code)
}
}
type from struct {
net.Conn
remote net.Addr
}
func (c from) RemoteAddr() net.Addr { return c.remote }
// The handshake refuses an internal-only name to an outsider too: the certificate would name it,
// and serving it would answer the question the routing refuses to.
func TestTheHandshakeRefusesAnInternalOnlyNameToAnOutsider(t *testing.T) {
held := behind(t, "10.10.0.1", "10.10.0.7")
served := &tls.Certificate{}
pick := certificateFor(held, func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return served, nil }, nil)
hello := func(name, remote string) *tls.ClientHelloInfo {
addr, _ := net.ResolveTCPAddr("tcp", remote)
return &tls.ClientHelloInfo{ServerName: name, Conn: from{remote: addr}}
}
if _, err := pick(hello("admin.anchor.internal", "203.0.113.9:443")); err == nil {
t.Error("an outsider was handed a certificate for an internal-only name")
}
if got, err := pick(hello("admin.anchor.internal", "10.10.0.7:443")); err != nil || got != served {
t.Errorf("a client on the private network was refused: %v", err)
}
if got, err := pick(hello("app.example", "203.0.113.9:443")); err != nil || got != served {
t.Errorf("a public name was refused to an outsider: %v", err)
}
}
// The mesh is issued as machines' addresses; a range is read as well. One that does not parse is
// refused rather than skipped, so a typo never quietly narrows or widens who is inside.
func TestTheMeshIsReadAsAddressesAndRanges(t *testing.T) {
if _, err := sourcesOf([]string{"10.10.0.1", "not-an-address"}); err == nil {
t.Error("an entry that is not an address was accepted")
}
inside, err := sourcesOf([]string{"10.10.0.1", "fd00::1", "10.20.0.0/24"})
if err != nil {
t.Fatal(err)
}
for remote, want := range map[string]bool{
"10.10.0.1:1": true,
"[::ffff:10.10.0.1]:1": true,
"[fd00::1]:1": true,
"10.20.0.200:1": true,
"10.10.0.2:1": false,
"192.168.1.10:1": false,
"not-an-address": false,
} {
if inside.holds(remote) != want {
t.Errorf("%s inside the mesh: got %v, want %v", remote, !want, want)
}
}
}
// What the mesh issues is what is served: the routes in the membership, internal names to the
// machines it names (novox/hq ADR 0167).
func TestAMembershipIsServedAsIssued(t *testing.T) {
held := newTable()
took := applyMembership(broker.Membership{
Receives: map[string]json.RawMessage{"route": json.RawMessage(`[
{"from":"admin","node":"anchor","at":"anchor.internal",
"values":{"internal-name":"admin.anchor.internal","port":8080}}]`)},
Mesh: []string{"10.10.0.7"},
}, held)
if !took {
t.Fatal("a membership carrying routes was not applied")
}
if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:1"); code == http.StatusNotFound {
t.Error("a machine the membership names was refused the internal-only route")
}
if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.9:1"); code != http.StatusNotFound {
t.Errorf("a machine the membership does not name was served the internal-only route: %d", code)
}
}
// A membership that says nothing about routes is one from a controller that does not issue them,
// and changes nothing: the file stays the source rather than every route being withdrawn.
func TestAMembershipWithoutRoutesLeavesTheFileServing(t *testing.T) {
held := behind(t, "10.10.0.7")
before := held.names()
if applyMembership(broker.Membership{Mesh: []string{"10.10.0.7"}}, held) {
t.Error("a membership without routes was taken as the source of routes")
}
if got := held.names(); strings.Join(got, ",") != strings.Join(before, ",") {
t.Errorf("a membership without routes changed what is served: %v, was %v", got, before)
}
if applyMembership(broker.Membership{
Receives: map[string]json.RawMessage{"route": json.RawMessage(`[]`)},
Mesh: []string{"not-an-address"},
}, held) {
t.Error("a membership whose mesh cannot be read was applied")
}
}
+44
View File
@@ -90,6 +90,50 @@ func TestARouteWithAnInternalNameIsReachableUnderBoth(t *testing.T) {
} }
} }
// A route whose endpoint reaches only the private network carries an internal name and no public
// one (novox/hq ADR 0138), and is served under that name rather than skipped as naming nothing —
// skipping it left every internal-only module unreachable by name (novox/hq issue 191).
func TestARouteWithOnlyAnInternalNameIsServed(t *testing.T) {
routes, public, err := routesFrom(write(t, `{"given":[
{"from":"app","node":"anchor","at":"anchor.internal",
"values":{"internal-name":"App.Anchor.Internal","port":8443,"scheme":"https","insecure":true}}
]}`))
if err != nil {
t.Fatal(err)
}
if targetOf(routes, "app.anchor.internal") != "https://anchor.internal:8443" {
t.Fatalf("the internal-only route is not served: %v", routes)
}
if len(routes) != 1 {
t.Errorf("an internal-only route made hosts it never named: %v", routes)
}
if len(public) != 0 {
t.Errorf("an internal-only route made a name eligible for a public certificate: %v", public)
}
held := newTable()
held.set(routes, public)
if err := onlyInternalNamesTheMeshSaid(held)(context.Background(), "app.anchor.internal"); err != nil {
t.Errorf("the internal authority refused the internal-only route's name: %v", err)
}
if err := onlyWhatTheMeshSaid(held)(context.Background(), "app.anchor.internal"); err == nil {
t.Error("a public certificate was ordered for an internal-only name")
}
}
// A route with neither name has nothing to be served under, and is still skipped.
func TestARouteWithNeitherNameIsSkipped(t *testing.T) {
routes, public, err := routesFrom(write(t, `{"given":[
{"from":"app","node":"anchor","at":"anchor.internal","values":{"internal-name":" ","port":8080}}
]}`))
if err != nil {
t.Fatal(err)
}
if len(routes) != 0 || len(public) != 0 {
t.Errorf("a route that named nothing was served: %v %v", routes, public)
}
}
// A route with no internal-name composed gets no second host — the ordinary case, unchanged. // A route with no internal-name composed gets no second host — the ordinary case, unchanged.
func TestARouteWithNoInternalNameGetsNoAlias(t *testing.T) { func TestARouteWithNoInternalNameGetsNoAlias(t *testing.T) {
routes, _, err := routesFrom(write(t, `{"given":[ routes, _, err := routesFrom(write(t, `{"given":[
+12
View File
@@ -1,6 +1,7 @@
package broker package broker
import ( import (
"encoding/json"
"sort" "sort"
"strings" "strings"
) )
@@ -33,6 +34,17 @@ type Membership struct {
Reaches map[string][]string `json:"reaches,omitempty"` Reaches map[string][]string `json:"reaches,omitempty"`
// Tools is where this instance answers what it serves — the runtime's one verb of its own. // Tools is where this instance answers what it serves — the runtime's one verb of its own.
Tools string `json:"tools"` Tools string `json:"tools"`
// Receives is what this assignment is given for each requirement it receives, by requirement:
// the contributions of every module that asked for it, as the catalogue composed them (novox/hq
// ADR 0167). The same list its received file is written from, so the two cannot disagree; a
// requirement nobody contributed to is an empty list, never absent. Kept as JSON because the
// catalogue owns the shape of a contribution and the bus only carries it.
Receives map[string]json.RawMessage `json:"receives,omitempty"`
// Mesh is every machine's address on the private network — what a rule saying "from the mesh"
// resolves to in the packet filter, issued here from the same list (novox/hq ADR 0167). A
// module that must tell the mesh from the world, the route proxy serving an internal name, reads
// it here rather than keeping a definition of its own.
Mesh []string `json:"mesh,omitempty"`
} }
// Served is one address a tool is answered on. // Served is one address a tool is answered on.
+16 -3
View File
@@ -241,15 +241,21 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
// Owner is kept beside the resources because a resource id cannot be split back into its module: // Owner is kept beside the resources because a resource id cannot be split back into its module:
// a module's name may itself contain a dot. What the mesh adds of its own — an opening, the guard — // a module's name may itself contain a dot. What the mesh adds of its own — an opening, the guard —
// has no owner. // has no owner.
//
// Received is what each module on the machine is given for each requirement it receives — the same
// contributions its received file is written from, kept beside it so the mesh can also issue them
// on the bus in the module's membership (novox/hq ADR 0167). By module, then requirement.
type Composed struct { type Composed struct {
Resources []map[string]any Resources []map[string]any
Owner map[string]string Owner map[string]string
Received map[string]map[string][]Contribution
} }
// Compose is Declaration with the owner of every resource said. // Compose is Declaration with the owner of every resource said.
func (r Resolution) Compose(with Rendering) (Composed, error) { func (r Resolution) Compose(with Rendering) (Composed, error) {
owner := map[string]string{} owner := map[string]string{}
resources, err := r.compose(with, owner) received := map[string]map[string][]Contribution{}
resources, err := r.compose(with, owner, received)
if err != nil { if err != nil {
return Composed{}, err return Composed{}, err
} }
@@ -261,7 +267,7 @@ func (r Resolution) Compose(with Rendering) (Composed, error) {
"sealed": with.BusMembership, "mode": "0600", "sealed": with.BusMembership, "mode": "0600",
}) })
} }
return Composed{Resources: resources, Owner: owner}, nil return Composed{Resources: resources, Owner: owner, Received: received}, nil
} }
// BusMembershipID names the resource carrying a machine's membership for the new bus, and // BusMembershipID names the resource carrying a machine's membership for the new bus, and
@@ -270,7 +276,8 @@ func BusMembershipID() string { return "bus-membership" }
const BusMembershipPath = "/var/lib/mesh/membership-next.json" const BusMembershipPath = "/var/lib/mesh/membership-next.json"
func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[string]any, error) { func (r Resolution) compose(with Rendering, owner map[string]string,
received map[string]map[string][]Contribution) ([]map[string]any, error) {
// Every manifest is placed first (novox/hq ADR 0112): the maps naming where its bindings, // Every manifest is placed first (novox/hq ADR 0112): the maps naming where its bindings,
// credentials and contributions land are resolved against this node's directories, so every // credentials and contributions land are resolved against this node's directories, so every
// reader below — the binding files, the sealed secrets, the grant paths a contribution // reader below — the binding files, the sealed secrets, the grant paths a contribution
@@ -596,6 +603,12 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
return nil, err return nil, err
} }
first = append(first, file) first = append(first, file)
if received[m.Module] == nil {
received[m.Module] = map[string][]Contribution{}
}
// Empty rather than absent when nobody contributed, for the reason the file is
// written empty: "nothing asked" and "never told" want different responses.
received[m.Module][to] = append([]Contribution{}, given[to]...)
} }
if m.Keeps != "" && with.Kept != nil { if m.Keeps != "" && with.Kept != nil {
file, err := keptFile(m.Keeps, with.Kept) file, err := keptFile(m.Keeps, with.Kept)
@@ -0,0 +1,66 @@
package catalogue
import (
"encoding/json"
"reflect"
"testing"
)
// What a provider receives is composed once, and issued twice: as its received file, and in its
// membership on the bus (novox/hq ADR 0167). The two are the same list, so a proxy reading the bus
// and one reading the file serve the same routes — including the port the machine published, which
// is the same-node fix the file already carries.
func TestWhatAProviderReceivesIsTheSameOnTheBusAsInItsFile(t *testing.T) {
gitea := Manifest{
Module: "gitea", Version: "1",
Listens: []Listening{{Port: 3000, Protocol: "tcp", From: FromMesh}},
Contributes: map[string]map[string]any{"route": {"label": "git", "port": 3000}},
Resources: []map[string]any{{
"id": "server", "type": "container", "name": "gitea", "ports": []any{"3000"},
}},
}
r, err := Resolve(shelf(gitea, routeProxy(), stepCA()),
[]string{"gitea", "route-proxy", "step-ca"}, reachable(), World{})
if err != nil {
t.Fatal(err)
}
composed, err := r.Compose(Rendering{Ports: map[string]map[int]int{"gitea": {3000: 20000}}})
if err != nil {
t.Fatal(err)
}
file := fileNamed(composed.Resources, "route-proxy.received-route")
if file == nil {
t.Fatal("the proxy was given no routes file")
}
var written struct {
Given []Contribution `json:"given"`
}
if err := json.Unmarshal([]byte(file["content"].(string)), &written); err != nil {
t.Fatal(err)
}
issued, said := composed.Received["route-proxy"]["route"]
if !said {
t.Fatalf("nothing is issued for the proxy to receive on the bus: %v", composed.Received)
}
// Compared as JSON, which is what both are once they leave the controller.
a, _ := json.Marshal(written.Given)
b, _ := json.Marshal(issued)
var fromFile, fromBus any
_ = json.Unmarshal(a, &fromFile)
_ = json.Unmarshal(b, &fromBus)
if !reflect.DeepEqual(fromFile, fromBus) {
t.Errorf("the bus and the file disagree about the routes:\nfile %s\nbus %s", a, b)
}
if len(issued) != 1 {
t.Fatalf("expected one route on the bus, got %v", issued)
}
if port, ok := asPort(issued[0].Values["port"]); !ok || port != 20000 {
t.Errorf("the bus carries a port nothing listens on: %v", issued[0].Values["port"])
}
// A module that receives nothing is issued nothing to receive.
if _, any := composed.Received["gitea"]; any {
t.Errorf("a module that receives nothing was issued something: %v", composed.Received["gitea"])
}
}