diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index 0e74502..4dea37e 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -404,7 +404,8 @@ func declarationWith(ctx context.Context, open *stores, node string, if err != nil { return sendable{}, err } - return sendable{Resources: composed.Resources, Adoption: adoption}, nil + return sendable{Resources: composed.Resources, Adoption: adoption, + Received: composed.Received, Mesh: with.Mesh}, nil } // renderingFor is everything a node's declaration is composed with, and the node's record. diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 27d00f9..0435863 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -389,11 +389,7 @@ func pushCommand(ctx context.Context, args []string) error { fmt.Printf("\n%d node(s) told\n", len(sending)) // And each machine's memberships, as every other send does (ADR 0160): a push is the one most // operators run, and on 2026-10-01 it was the one path that issued none. - var told []string - for _, s := range sending { - told = append(told, s.node) - } - if err := issueMemberships(ctx, open, server, told); err != nil { + if err := issueMemberships(ctx, open, server, sending); err != nil { return err } @@ -706,11 +702,15 @@ func sendTo(ctx context.Context, open *stores, names []string) error { // And every assignment on those machines its membership (novox/hq ADR 0160): composed from the // same records the bus's accounts are, so what a runtime serves and what its account may are one // composition. Issued after the declaration, because the runtime it is for arrives with it. - return issueMemberships(ctx, open, server, names) + return issueMemberships(ctx, open, server, sending) } -// issueMemberships publishes the membership of every module on the named machines. -func issueMemberships(ctx context.Context, open *stores, server *link.Server, names []string) error { +// issueMemberships publishes the membership of every module on the machines just sent. +// +// Each carries what its module receives and the private network's addresses, from the same +// composition as the declaration it was sent (novox/hq ADR 0167): a provider reads what it is +// given on the bus, and the file written beside it says the same thing. +func issueMemberships(ctx context.Context, open *stores, server *link.Server, sent []readyNode) error { records, err := open.inventory.BusRecords(ctx) if err != nil { return err @@ -725,9 +725,22 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, na // the push stands, the first failure is named once, and the next push tries again. issued, failed := 0, 0 var first error - for _, node := range names { + for _, s := range sent { + node := s.node for _, d := range records.Assigned[node] { - body, err := json.Marshal(broker.MembershipFor(node, d, where)) + membership := broker.MembershipFor(node, d, where) + membership.Mesh = s.declared.Mesh + for requirement, given := range s.declared.Received[d.Module] { + raw, err := json.Marshal(given) + if err != nil { + return err + } + if membership.Receives == nil { + membership.Receives = map[string]json.RawMessage{} + } + membership.Receives[requirement] = raw + } + body, err := json.Marshal(membership) if err != nil { return err } diff --git a/cmd/mesh-controller/sendable.go b/cmd/mesh-controller/sendable.go index f6a69d5..1ca2532 100644 --- a/cmd/mesh-controller/sendable.go +++ b/cmd/mesh-controller/sendable.go @@ -25,6 +25,12 @@ type sendable struct { // Adoption is nil for a converged node, and then the body is byte for byte what it was before // adoption existed: an older host parses the envelope strictly and would refuse the key. Adoption *adoptionEnvelope + + // Received and Mesh are not sent in the declaration. They are what this machine's memberships + // are issued with on the bus (novox/hq ADR 0167): each module's received contributions, from + // the same composition as its received files, and every machine's private-network address. + Received map[string]map[string][]catalogue.Contribution + Mesh []string } // adoptionEnvelope is what an adopted node is told about its mode. Taken is every module taken on diff --git a/examples/route-proxy/bus.go b/examples/route-proxy/bus.go new file mode 100644 index 0000000..51ed510 --- /dev/null +++ b/examples/route-proxy/bus.go @@ -0,0 +1,132 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "os" + "strings" + "sync/atomic" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" +) + +// What the mesh issued this proxy, read on the bus (novox/hq ADR 0160, ADR 0167). +// +// **The proxy is told, not left to work it out.** Its membership carries the routes it is given — +// the same contributions its file is written from — and every machine's address on the private +// network, which is who may be served an internal name. Read once at connect and followed, so a +// route added or a machine joining reaches a running proxy without a restart. + +// credential is the bus account the mesh delivered as this module's own secret named broker. +type credential struct { + URL string `json:"url"` + Fingerprint string `json:"fingerprint"` + Node string `json:"node"` + Module string `json:"module"` + User string `json:"user"` + Password string `json:"password"` +} + +// followMembership connects with the credential in path and applies every membership the mesh +// issues this proxy. It retries the first connection for as long as it takes: a proxy that started +// before the bus keeps serving the file, and takes the bus when it answers. +func followMembership(path string, held *table, fromBus *atomic.Bool) { + for { + err := followOnce(path, held, fromBus) + if err == nil { + return + } + log.Printf("cannot follow this proxy's membership, serving the file meanwhile: %v", err) + time.Sleep(30 * time.Second) + } +} + +func followOnce(path string, held *table, fromBus *atomic.Bool) error { + raw, err := os.ReadFile(path) + if err != nil { + return err + } + var cred credential + if err := json.Unmarshal(raw, &cred); err != nil { + return fmt.Errorf("the broker credential is not one: %w", err) + } + if cred.Node == "" || cred.Module == "" { + return fmt.Errorf("the broker credential names no node or module, so it has no membership") + } + + opts := []nats.Option{ + nats.Name(cred.Node + "." + cred.Module), + nats.UserInfo(cred.User, cred.Password), + // Its own inbox, and nothing wider: every principal is granted `_INBOX..>` alone. + nats.CustomInboxPrefix("_INBOX." + cred.User), + // The bus being restarted is an upgrade, not a reason to stop following. + nats.MaxReconnects(-1), + } + if strings.TrimSpace(cred.Fingerprint) != "" { + opts = append(opts, nats.Secure(broker.PinnedToFingerprint(cred.Fingerprint))) + } + conn, err := nats.Connect(cred.URL, opts...) + if err != nil { + return fmt.Errorf("connecting to the bus at %s: %w", broker.BareAddress(cred.URL), err) + } + + subject := broker.MembershipSubject(cred.Node, cred.Module) + apply := func(body []byte) { + var issued broker.Membership + if err := json.Unmarshal(body, &issued); err != nil { + log.Printf("a membership arrived that is not one: %v", err) + return + } + if took := applyMembership(issued, held); took && !fromBus.Swap(true) { + log.Printf("routes now come from this proxy's membership on %s", subject) + } + } + + // Followed first, read second: an issue landing between the two is applied, not missed. + if _, err := conn.Subscribe(subject, func(m *nats.Msg) { apply(m.Data) }); err != nil { + conn.Close() + return fmt.Errorf("cannot follow %s: %w", subject, err) + } + // The subject-addressed direct get: the one request this account may make of the stream. + got, err := conn.Request("$JS.API.DIRECT.GET."+broker.AssignmentsStream+"."+subject, nil, 5*time.Second) + switch { + case err != nil: + log.Printf("cannot read the membership issued on %s yet (%v); following it", subject, err) + case got.Header.Get("Status") != "" || len(got.Data) == 0: + log.Printf("no membership issued on %s yet; serving the file until one is", subject) + default: + apply(got.Data) + } + return nil +} + +// applyMembership serves what a membership says, and says whether it said anything about routes. +// +// A membership with no routes in it is one from a controller older than ADR 0167, and the file stays +// the source rather than every route being withdrawn because a field was absent. +func applyMembership(issued broker.Membership, held *table) bool { + raw, carries := issued.Receives["route"] + if !carries { + return false + } + var contributions []contribution + if err := json.Unmarshal(raw, &contributions); err != nil { + log.Printf("the routes in this proxy's membership are not contributions, keeping what is served: %v", err) + return false + } + inside, err := sourcesOf(issued.Mesh) + if err != nil { + log.Printf("the mesh in this proxy's membership is unreadable, keeping what is served: %v", err) + return false + } + routes, public := routesOf(contributions) + held.set(routes, public) + held.setInside(inside) + log.Printf("serving %d route(s) from the membership, internal names to %d machine(s): %s", + len(routes), len(inside), strings.Join(held.names(), ", ")) + return true +} diff --git a/examples/route-proxy/main.go b/examples/route-proxy/main.go index 5b9a39a..47d4f93 100644 --- a/examples/route-proxy/main.go +++ b/examples/route-proxy/main.go @@ -54,12 +54,14 @@ import ( "net" "net/http" "net/http/httputil" + "net/netip" "net/url" "os" "path/filepath" "sort" "strings" "sync" + "sync/atomic" "time" "golang.org/x/crypto/acme" @@ -196,6 +198,63 @@ type table struct { // pass ACME's own validation (it has no public DNS to prove it against), so asking for it is // not merely pointless but the failing order onlyWhatTheMeshSaid exists to prevent. public map[string]bool + // inside is where a request must come from to be served a name that is only internal: every + // machine's address on the private network, as the mesh issued it in this proxy's membership + // (novox/hq ADR 0167). Empty until it is issued, and then only the machine itself is inside. + inside sources +} + +// sources is who may be served an internal name: the private network's addresses as the mesh +// issued them. The machine itself is always inside — anything on a machine may call anything on it +// (novox/hq ADR 0144) — so loopback needs no entry. +type sources []netip.Prefix + +// sourcesOf reads the addresses the mesh issued, each a single address or a range. One that does +// not parse is an error, not an entry skipped: the proxy would otherwise serve internal names to +// fewer machines than the mesh said, and say nothing. +func sourcesOf(mesh []string) (sources, error) { + var out sources + for _, entry := range mesh { + entry = strings.TrimSpace(entry) + if prefix, err := netip.ParsePrefix(entry); err == nil { + out = append(out, prefix.Masked()) + continue + } + addr, err := netip.ParseAddr(entry) + if err != nil { + return nil, fmt.Errorf("%q is not an address on the private network", entry) + } + addr = addr.Unmap() + out = append(out, netip.PrefixFrom(addr, addr.BitLen())) + } + return out, nil +} + +// holds says whether a request from this remote address came from the mesh or the machine itself. +// +// **By source, which the mesh's guard deliberately is not** — it names interfaces, because a source +// address can be claimed by whoever sends the packet. The proxy cannot see the interface a request +// arrived on, and here the claim does not carry: a connection needs its replies, and replies to a +// mesh address leave by the tunnel, never back to the claimant. +func (s sources) holds(remote string) bool { + host := remote + if h, _, err := net.SplitHostPort(remote); err == nil { + host = h + } + addr, err := netip.ParseAddr(host) + if err != nil { + return false + } + addr = addr.Unmap() + if addr.IsLoopback() { + return true + } + for _, prefix := range s { + if prefix.Contains(addr) { + return true + } + } + return false } func (t *table) set(routes map[string][]rule, public map[string]bool) { @@ -314,6 +373,41 @@ func bareHost(host string) string { return strings.ToLower(host) } +// hiddenFrom says whether this host must look unrouted to a request from this address: it is +// only an internal name, and the request did not come from the private network. +// +// **The proxy is the only way in to a routed endpoint, so it is what makes `internal` true** +// (novox/hq ADR 0138, issue 191). It answers public names on the same listeners, so a request from +// anywhere can carry any Host header; a name being internal keeps nobody out unless this check does. +// Answered exactly as a name that was never routed, so an outsider learns nothing from asking. +func (t *table) hiddenFrom(host, remote string) bool { + if !t.eligibleForInternalACME(host) { + return false + } + t.mu.RLock() + defer t.mu.RUnlock() + return !t.inside.holds(remote) +} + +// setInside replaces who the mesh is, as the membership said. +func (t *table) setInside(inside sources) { + t.mu.Lock() + t.inside = inside + t.mu.Unlock() +} + +// namesSeenFrom is what this proxy says it serves to a request from this address — every routed +// name, less the internal-only ones when the request came from outside. +func (t *table) namesSeenFrom(remote string) []string { + out := []string{} + for _, name := range t.names() { + if !t.hiddenFrom(name, remote) { + out = append(out, name) + } + } + return out +} + func (t *table) names() []string { t.mu.RLock() defer t.mu.RUnlock() @@ -343,7 +437,20 @@ func run() error { } held := newTable() + // **The bus first, the file until it has spoken** (novox/hq ADR 0167). The membership carries + // the routes and who the mesh is; the file carries the routes alone, so while the proxy reads + // it an internal name is served to this machine and to nobody else — refused, never opened. + fromBus := &atomic.Bool{} + if credential := strings.TrimSpace(os.Getenv("MESH_BROKER_FILE")); credential != "" { + go followMembership(credential, held, fromBus) + } else { + log.Printf("MESH_BROKER_FILE is not set: routes come from %s alone, and a name that is only "+ + "internal is served to this machine alone", path) + } read := func() { + if fromBus.Load() { + return + } routes, public, err := routesFrom(path) if err != nil { // Kept serving what it had. A file being rewritten is momentarily unreadable, and @@ -422,19 +529,7 @@ func run() error { }() tlsConfig := publicManager.TLSConfig() - if internalManager != nil { - // Dispatched by which authority may certify this name at all — the same question - // eligibleForInternalACME already answers, asked once more at handshake time rather than - // only when an order is placed, since a cached certificate is served here on every request - // and never goes through HostPolicy again. - fromPublic, fromInternal := tlsConfig.GetCertificate, internalManager.TLSConfig().GetCertificate - tlsConfig.GetCertificate = func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { - if held.eligibleForInternalACME(hello.ServerName) { - return fromInternal(hello) - } - return fromPublic(hello) - } - } + tlsConfig.GetCertificate = certificateFor(held, tlsConfig.GetCertificate, internalManager) server := &http.Server{ Addr: secure, @@ -592,6 +687,33 @@ func forThisAuthority(cache, directory string, root []byte) string { return filepath.Join(cache, hex.EncodeToString(sum[:])[:16]) } +// certificateFor picks the certificate a handshake is answered with. +// +// Dispatched by which authority may certify this name at all — the same question +// eligibleForInternalACME already answers, asked once more at handshake time rather than only when +// an order is placed, since a cached certificate is served here on every request and never goes +// through HostPolicy again. And refused, exactly as an unrouted name is, to a client outside the +// private network asking for a name that is only internal: the certificate would name it. +func certificateFor(held *table, fromPublic func(*tls.ClientHelloInfo) (*tls.Certificate, error), + internalManager *autocert.Manager) func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + var fromInternal func(*tls.ClientHelloInfo) (*tls.Certificate, error) + if internalManager != nil { + fromInternal = internalManager.TLSConfig().GetCertificate + } + return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { + if held.eligibleForInternalACME(hello.ServerName) { + if hello.Conn != nil && held.hiddenFrom(hello.ServerName, hello.Conn.RemoteAddr().String()) { + return nil, fmt.Errorf("no public route for %q in this mesh, so no certificate is asked for", + hello.ServerName) + } + if fromInternal != nil { + return fromInternal(hello) + } + } + return fromPublic(hello) + } +} + // newTable is an empty routing table. func newTable() *table { return &table{to: map[string][]rule{}} @@ -600,8 +722,9 @@ func newTable() *table { // handler is the proxy itself, separated so it can be driven by a test without a listener. func handler(held *table) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hidden := held.hiddenFrom(r.Host, r.RemoteAddr) matched, known := held.find(r.Host, r.URL.Path) - if !known { + if hidden || !known { // **Named, not a bare 404.** A route that was withdrawn and a name that never existed // are different things, and a proxy that says only "not found" makes an operator go // and read the mesh to tell them apart. What it is serving is the answer to both. @@ -611,13 +734,13 @@ func handler(held *table) http.Handler { // contradiction an operator would have to disbelieve the proxy to get past. w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.WriteHeader(http.StatusNotFound) - if held.routed(r.Host) { + if !hidden && held.routed(r.Host) { fmt.Fprintf(w, "%s is served here, but no route covers %q.\n", bareHost(r.Host), r.URL.Path) return } fmt.Fprintf(w, "no route for %q in this mesh.\nserving: %s\n", - r.Host, strings.Join(held.names(), ", ")) + r.Host, strings.Join(held.namesSeenFrom(r.RemoteAddr), ", ")) return } @@ -716,6 +839,12 @@ func boolByte(b bool) byte { // routesFrom reads what the mesh wrote and turns it into host → the rules for that host, and // which of those hosts is a public name — the second is `name`, ACME-eligible; a host reached // only through `internal-name` never appears there. +// +// **A route may carry either name, or both** (novox/hq ADR 0138). How far an endpoint reaches +// decides which names the mesh composes, so an endpoint that reaches only the private network +// arrives with an `internal-name` and no `name`. That is a whole route, not a malformed one: it is +// served under its internal name and certified by the internal authority. Only a route with +// neither name has nothing to be served under (novox/hq issue 191). func routesFrom(path string) (map[string][]rule, map[string]bool, error) { raw, err := os.ReadFile(path) if err != nil { @@ -725,17 +854,29 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) { if err := json.Unmarshal(raw, &said); err != nil { return nil, nil, err } + routes, public := routesOf(said.Given) + return routes, public, nil +} +// routesOf turns what the mesh gave into host → the rules for that host, and which hosts are public +// names — the same whether the contributions came in the file or in the membership. +func routesOf(contributions []contribution) (map[string][]rule, map[string]bool) { out := map[string][]rule{} public := map[string]bool{} - for _, c := range said.Given { + for _, c := range contributions { name, _ := c.Values["name"].(string) - if name == "" { + name = strings.TrimSpace(name) + internal, _ := c.Values["internal-name"].(string) + internal = strings.TrimSpace(internal) + if name == "" && internal == "" { log.Printf("%s on %s asked for a route and named nothing; skipped", c.From, c.Node) continue } - host := strings.ToLower(name) - public[host] = true + // What the route is called in a log line: its public name when it has one. + called := name + if called == "" { + called = internal + } made := rule{path: asPath(c.Values["path"])} if p, ok := asWhole(c.Values["priority"]); ok { @@ -752,7 +893,7 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) { if looksLikeACredential(named) { log.Printf("%s on %s declared route %q with a credential in the declaration rather "+ "than the name of a secret; the whole route is refused (novox/hq ADR 0108)", - c.From, c.Node, name) + c.From, c.Node, called) continue } users, err := usersFrom(named) @@ -770,7 +911,7 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) { port, ok := asPort(c.Values["port"]) if !ok { log.Printf("%s on %s asked for route %q and gave no usable port; skipped", - c.From, c.Node, name) + c.From, c.Node, called) continue } // Where the mesh says that machine is. Empty means it is this one — a workload beside @@ -791,7 +932,7 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) { } if scheme != "http" && scheme != "https" { log.Printf("%s on %s asked for route %q with scheme %q, which is neither http "+ - "nor https; skipped", c.From, c.Node, name, scheme) + "nor https; skipped", c.From, c.Node, called, scheme) continue } made.insecure, _ = c.Values["insecure"].(bool) @@ -802,7 +943,7 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) { bytes, whole := asWhole(asked) if !whole || bytes <= 0 { log.Printf("%s on %s asked for route %q with a max-request-body of %v, which is "+ - "not a whole positive number of bytes; skipped", c.From, c.Node, name, asked) + "not a whole positive number of bytes; skipped", c.From, c.Node, called, asked) continue } made.maxRequestBody = int64(bytes) @@ -810,19 +951,24 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) { made.target = fmt.Sprintf("%s://%s:%d", scheme, at, port) } - out[host] = append(out[host], made) + if name != "" { + host := strings.ToLower(name) + out[host] = append(out[host], made) + public[host] = true + } - // The internal-network alias, the same rule under a second host — a predecessor proxy + // The internal-network name, the same rule under a second host — a predecessor proxy // answered both for one route, as a convenience (reaching a service over the VPN without a // public TLS round trip), not as an access boundary; composing it here restores exactly // that, nothing more. Absent whenever the node composed no internal name (novox/hq ADR // 0056's internalDomain half) — the same "nothing to join a label to" case the public name - // already has. - if internal, _ := c.Values["internal-name"].(string); strings.TrimSpace(internal) != "" { + // already has. And the only name, when the endpoint reaches no further than the private + // network. + if internal != "" { out[strings.ToLower(internal)] = append(out[strings.ToLower(internal)], made) } } - return out, public, nil + return out, public } // asWhole is any whole number the mesh wrote, whatever its magnitude. diff --git a/examples/route-proxy/reach_test.go b/examples/route-proxy/reach_test.go new file mode 100644 index 0000000..151a41a --- /dev/null +++ b/examples/route-proxy/reach_test.go @@ -0,0 +1,206 @@ +package main + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/broker" +) + +// behind is a workload the proxy can send to, and a table routing one public name and one +// internal-only name to it, with the mesh's machines as the membership would issue them. +func behind(t *testing.T, mesh ...string) *table { + t.Helper() + workload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, "the workload") + })) + t.Cleanup(workload.Close) + at, _ := url.Parse(workload.URL) + host, port, _ := net.SplitHostPort(at.Host) + + routes, public, err := routesFrom(write(t, fmt.Sprintf(`{"given":[ + {"from":"app","node":"anchor","at":%q, + "values":{"name":"app.example","internal-name":"app.anchor.internal","port":%s}}, + {"from":"admin","node":"anchor","at":%q, + "values":{"internal-name":"admin.anchor.internal","port":%s}} + ]}`, host, port, host, port))) + if err != nil { + t.Fatal(err) + } + held := newTable() + inside, err := sourcesOf(mesh) + if err != nil { + t.Fatal(err) + } + held.setInside(inside) + held.set(routes, public) + return held +} + +// askFrom is what the proxy answers a request for host coming from remote. +func askFrom(held *table, host, remote string) (int, string) { + r := httptest.NewRequest(http.MethodGet, "http://"+host+"/", nil) + r.RemoteAddr = remote + w := httptest.NewRecorder() + handler(held).ServeHTTP(w, r) + return w.Code, w.Body.String() +} + +// **An internal-only name is served to the private network and to nobody else** (novox/hq ADR +// 0138, issue 191). The proxy answers public names on the same listeners, so without this a name +// being internal kept nobody out: a request from the internet only had to carry it. +func TestAnInternalOnlyNameIsServedOnlyInsideThePrivateNetwork(t *testing.T) { + held := behind(t, "10.10.0.1", "10.10.0.7") + + if code, body := askFrom(held, "admin.anchor.internal", "10.10.0.7:51000"); code != http.StatusOK || + body != "the workload" { + t.Errorf("a request from the private network was not served: %d %q", code, body) + } + if code, body := askFrom(held, "admin.anchor.internal", "127.0.0.1:51000"); code != http.StatusOK { + t.Errorf("a request from the machine itself was not served: %d %q", code, body) + } + + code, body := askFrom(held, "admin.anchor.internal", "203.0.113.9:51000") + if code != http.StatusNotFound { + t.Fatalf("a request from outside the private network reached an internal-only name: %d %q", + code, body) + } + // Answered as a name never routed, and the list of what is served does not name it either — + // otherwise the refusal would tell an outsider exactly what to ask for from inside. + if strings.Contains(strings.SplitN(body, "\n", 2)[1], "admin.anchor.internal") { + t.Errorf("the refusal names the internal-only route to an outsider: %q", body) + } + if !strings.Contains(body, "app.example") { + t.Errorf("the refusal stopped listing the public names: %q", body) + } +} + +// The internal name of a route that also has a public one is internal too: served inside, and to +// an outsider only under the public name. Nothing is lost — the outsider has the public name — and a +// name stays one thing whichever route it came from. +func TestAnInternalAliasOfAPublicRouteIsServedInsideOnly(t *testing.T) { + held := behind(t, "10.10.0.1", "10.10.0.7") + if code, body := askFrom(held, "app.anchor.internal", "10.10.0.7:51000"); code != http.StatusOK { + t.Errorf("the internal alias stopped answering the private network: %d %q", code, body) + } + if code, _ := askFrom(held, "app.anchor.internal", "203.0.113.9:51000"); code != http.StatusNotFound { + t.Errorf("the internal alias was served to an outsider: %d", code) + } + if code, _ := askFrom(held, "app.example", "203.0.113.9:51000"); code != http.StatusOK { + t.Errorf("the public name was refused to an outsider: %d", code) + } +} + +// Before a membership has said who the mesh is, only the machine itself is inside — refused to +// everyone else, never served to everyone. +func TestUntilTheMeshIsIssuedAnInternalOnlyNameIsServedToTheMachineAlone(t *testing.T) { + held := behind(t) + if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:51000"); code != http.StatusNotFound { + t.Errorf("an internal-only name was served with no private network said: %d", code) + } + if code, _ := askFrom(held, "admin.anchor.internal", "[::1]:51000"); code != http.StatusOK { + t.Errorf("an internal-only name was refused to the machine itself: %d", code) + } +} + +type from struct { + net.Conn + remote net.Addr +} + +func (c from) RemoteAddr() net.Addr { return c.remote } + +// The handshake refuses an internal-only name to an outsider too: the certificate would name it, +// and serving it would answer the question the routing refuses to. +func TestTheHandshakeRefusesAnInternalOnlyNameToAnOutsider(t *testing.T) { + held := behind(t, "10.10.0.1", "10.10.0.7") + served := &tls.Certificate{} + pick := certificateFor(held, func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return served, nil }, nil) + hello := func(name, remote string) *tls.ClientHelloInfo { + addr, _ := net.ResolveTCPAddr("tcp", remote) + return &tls.ClientHelloInfo{ServerName: name, Conn: from{remote: addr}} + } + + if _, err := pick(hello("admin.anchor.internal", "203.0.113.9:443")); err == nil { + t.Error("an outsider was handed a certificate for an internal-only name") + } + if got, err := pick(hello("admin.anchor.internal", "10.10.0.7:443")); err != nil || got != served { + t.Errorf("a client on the private network was refused: %v", err) + } + if got, err := pick(hello("app.example", "203.0.113.9:443")); err != nil || got != served { + t.Errorf("a public name was refused to an outsider: %v", err) + } +} + +// The mesh is issued as machines' addresses; a range is read as well. One that does not parse is +// refused rather than skipped, so a typo never quietly narrows or widens who is inside. +func TestTheMeshIsReadAsAddressesAndRanges(t *testing.T) { + if _, err := sourcesOf([]string{"10.10.0.1", "not-an-address"}); err == nil { + t.Error("an entry that is not an address was accepted") + } + inside, err := sourcesOf([]string{"10.10.0.1", "fd00::1", "10.20.0.0/24"}) + if err != nil { + t.Fatal(err) + } + for remote, want := range map[string]bool{ + "10.10.0.1:1": true, + "[::ffff:10.10.0.1]:1": true, + "[fd00::1]:1": true, + "10.20.0.200:1": true, + "10.10.0.2:1": false, + "192.168.1.10:1": false, + "not-an-address": false, + } { + if inside.holds(remote) != want { + t.Errorf("%s inside the mesh: got %v, want %v", remote, !want, want) + } + } +} + +// What the mesh issues is what is served: the routes in the membership, internal names to the +// machines it names (novox/hq ADR 0167). +func TestAMembershipIsServedAsIssued(t *testing.T) { + held := newTable() + took := applyMembership(broker.Membership{ + Receives: map[string]json.RawMessage{"route": json.RawMessage(`[ + {"from":"admin","node":"anchor","at":"anchor.internal", + "values":{"internal-name":"admin.anchor.internal","port":8080}}]`)}, + Mesh: []string{"10.10.0.7"}, + }, held) + if !took { + t.Fatal("a membership carrying routes was not applied") + } + if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:1"); code == http.StatusNotFound { + t.Error("a machine the membership names was refused the internal-only route") + } + if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.9:1"); code != http.StatusNotFound { + t.Errorf("a machine the membership does not name was served the internal-only route: %d", code) + } +} + +// A membership that says nothing about routes is one from a controller that does not issue them, +// and changes nothing: the file stays the source rather than every route being withdrawn. +func TestAMembershipWithoutRoutesLeavesTheFileServing(t *testing.T) { + held := behind(t, "10.10.0.7") + before := held.names() + if applyMembership(broker.Membership{Mesh: []string{"10.10.0.7"}}, held) { + t.Error("a membership without routes was taken as the source of routes") + } + if got := held.names(); strings.Join(got, ",") != strings.Join(before, ",") { + t.Errorf("a membership without routes changed what is served: %v, was %v", got, before) + } + if applyMembership(broker.Membership{ + Receives: map[string]json.RawMessage{"route": json.RawMessage(`[]`)}, + Mesh: []string{"not-an-address"}, + }, held) { + t.Error("a membership whose mesh cannot be read was applied") + } +} diff --git a/examples/route-proxy/routes_test.go b/examples/route-proxy/routes_test.go index 3bb71a3..1ad9a26 100644 --- a/examples/route-proxy/routes_test.go +++ b/examples/route-proxy/routes_test.go @@ -90,6 +90,50 @@ func TestARouteWithAnInternalNameIsReachableUnderBoth(t *testing.T) { } } +// A route whose endpoint reaches only the private network carries an internal name and no public +// one (novox/hq ADR 0138), and is served under that name rather than skipped as naming nothing — +// skipping it left every internal-only module unreachable by name (novox/hq issue 191). +func TestARouteWithOnlyAnInternalNameIsServed(t *testing.T) { + routes, public, err := routesFrom(write(t, `{"given":[ + {"from":"app","node":"anchor","at":"anchor.internal", + "values":{"internal-name":"App.Anchor.Internal","port":8443,"scheme":"https","insecure":true}} + ]}`)) + if err != nil { + t.Fatal(err) + } + if targetOf(routes, "app.anchor.internal") != "https://anchor.internal:8443" { + t.Fatalf("the internal-only route is not served: %v", routes) + } + if len(routes) != 1 { + t.Errorf("an internal-only route made hosts it never named: %v", routes) + } + if len(public) != 0 { + t.Errorf("an internal-only route made a name eligible for a public certificate: %v", public) + } + + held := newTable() + held.set(routes, public) + if err := onlyInternalNamesTheMeshSaid(held)(context.Background(), "app.anchor.internal"); err != nil { + t.Errorf("the internal authority refused the internal-only route's name: %v", err) + } + if err := onlyWhatTheMeshSaid(held)(context.Background(), "app.anchor.internal"); err == nil { + t.Error("a public certificate was ordered for an internal-only name") + } +} + +// A route with neither name has nothing to be served under, and is still skipped. +func TestARouteWithNeitherNameIsSkipped(t *testing.T) { + routes, public, err := routesFrom(write(t, `{"given":[ + {"from":"app","node":"anchor","at":"anchor.internal","values":{"internal-name":" ","port":8080}} + ]}`)) + if err != nil { + t.Fatal(err) + } + if len(routes) != 0 || len(public) != 0 { + t.Errorf("a route that named nothing was served: %v %v", routes, public) + } +} + // A route with no internal-name composed gets no second host — the ordinary case, unchanged. func TestARouteWithNoInternalNameGetsNoAlias(t *testing.T) { routes, _, err := routesFrom(write(t, `{"given":[ diff --git a/internal/broker/membership.go b/internal/broker/membership.go index dbd99c5..8cf9b98 100644 --- a/internal/broker/membership.go +++ b/internal/broker/membership.go @@ -1,6 +1,7 @@ package broker import ( + "encoding/json" "sort" "strings" ) @@ -33,6 +34,17 @@ type Membership struct { Reaches map[string][]string `json:"reaches,omitempty"` // Tools is where this instance answers what it serves — the runtime's one verb of its own. Tools string `json:"tools"` + // Receives is what this assignment is given for each requirement it receives, by requirement: + // the contributions of every module that asked for it, as the catalogue composed them (novox/hq + // ADR 0167). The same list its received file is written from, so the two cannot disagree; a + // requirement nobody contributed to is an empty list, never absent. Kept as JSON because the + // catalogue owns the shape of a contribution and the bus only carries it. + Receives map[string]json.RawMessage `json:"receives,omitempty"` + // Mesh is every machine's address on the private network — what a rule saying "from the mesh" + // resolves to in the packet filter, issued here from the same list (novox/hq ADR 0167). A + // module that must tell the mesh from the world, the route proxy serving an internal name, reads + // it here rather than keeping a definition of its own. + Mesh []string `json:"mesh,omitempty"` } // Served is one address a tool is answered on. diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 41e77e5..527e0d2 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -241,15 +241,21 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) { // Owner is kept beside the resources because a resource id cannot be split back into its module: // a module's name may itself contain a dot. What the mesh adds of its own — an opening, the guard — // has no owner. +// +// Received is what each module on the machine is given for each requirement it receives — the same +// contributions its received file is written from, kept beside it so the mesh can also issue them +// on the bus in the module's membership (novox/hq ADR 0167). By module, then requirement. type Composed struct { Resources []map[string]any Owner map[string]string + Received map[string]map[string][]Contribution } // Compose is Declaration with the owner of every resource said. func (r Resolution) Compose(with Rendering) (Composed, error) { owner := map[string]string{} - resources, err := r.compose(with, owner) + received := map[string]map[string][]Contribution{} + resources, err := r.compose(with, owner, received) if err != nil { return Composed{}, err } @@ -261,7 +267,7 @@ func (r Resolution) Compose(with Rendering) (Composed, error) { "sealed": with.BusMembership, "mode": "0600", }) } - return Composed{Resources: resources, Owner: owner}, nil + return Composed{Resources: resources, Owner: owner, Received: received}, nil } // BusMembershipID names the resource carrying a machine's membership for the new bus, and @@ -270,7 +276,8 @@ func BusMembershipID() string { return "bus-membership" } const BusMembershipPath = "/var/lib/mesh/membership-next.json" -func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[string]any, error) { +func (r Resolution) compose(with Rendering, owner map[string]string, + received map[string]map[string][]Contribution) ([]map[string]any, error) { // Every manifest is placed first (novox/hq ADR 0112): the maps naming where its bindings, // credentials and contributions land are resolved against this node's directories, so every // reader below — the binding files, the sealed secrets, the grant paths a contribution @@ -596,6 +603,12 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri return nil, err } first = append(first, file) + if received[m.Module] == nil { + received[m.Module] = map[string][]Contribution{} + } + // Empty rather than absent when nobody contributed, for the reason the file is + // written empty: "nothing asked" and "never told" want different responses. + received[m.Module][to] = append([]Contribution{}, given[to]...) } if m.Keeps != "" && with.Kept != nil { file, err := keptFile(m.Keeps, with.Kept) diff --git a/internal/catalogue/received_on_the_bus_test.go b/internal/catalogue/received_on_the_bus_test.go new file mode 100644 index 0000000..185b10b --- /dev/null +++ b/internal/catalogue/received_on_the_bus_test.go @@ -0,0 +1,66 @@ +package catalogue + +import ( + "encoding/json" + "reflect" + "testing" +) + +// What a provider receives is composed once, and issued twice: as its received file, and in its +// membership on the bus (novox/hq ADR 0167). The two are the same list, so a proxy reading the bus +// and one reading the file serve the same routes — including the port the machine published, which +// is the same-node fix the file already carries. +func TestWhatAProviderReceivesIsTheSameOnTheBusAsInItsFile(t *testing.T) { + gitea := Manifest{ + Module: "gitea", Version: "1", + Listens: []Listening{{Port: 3000, Protocol: "tcp", From: FromMesh}}, + Contributes: map[string]map[string]any{"route": {"label": "git", "port": 3000}}, + Resources: []map[string]any{{ + "id": "server", "type": "container", "name": "gitea", "ports": []any{"3000"}, + }}, + } + r, err := Resolve(shelf(gitea, routeProxy(), stepCA()), + []string{"gitea", "route-proxy", "step-ca"}, reachable(), World{}) + if err != nil { + t.Fatal(err) + } + composed, err := r.Compose(Rendering{Ports: map[string]map[int]int{"gitea": {3000: 20000}}}) + if err != nil { + t.Fatal(err) + } + + file := fileNamed(composed.Resources, "route-proxy.received-route") + if file == nil { + t.Fatal("the proxy was given no routes file") + } + var written struct { + Given []Contribution `json:"given"` + } + if err := json.Unmarshal([]byte(file["content"].(string)), &written); err != nil { + t.Fatal(err) + } + issued, said := composed.Received["route-proxy"]["route"] + if !said { + t.Fatalf("nothing is issued for the proxy to receive on the bus: %v", composed.Received) + } + // Compared as JSON, which is what both are once they leave the controller. + a, _ := json.Marshal(written.Given) + b, _ := json.Marshal(issued) + var fromFile, fromBus any + _ = json.Unmarshal(a, &fromFile) + _ = json.Unmarshal(b, &fromBus) + if !reflect.DeepEqual(fromFile, fromBus) { + t.Errorf("the bus and the file disagree about the routes:\nfile %s\nbus %s", a, b) + } + if len(issued) != 1 { + t.Fatalf("expected one route on the bus, got %v", issued) + } + if port, ok := asPort(issued[0].Values["port"]); !ok || port != 20000 { + t.Errorf("the bus carries a port nothing listens on: %v", issued[0].Values["port"]) + } + + // A module that receives nothing is issued nothing to receive. + if _, any := composed.Received["gitea"]; any { + t.Errorf("a module that receives nothing was issued something: %v", composed.Received["gitea"]) + } +}