diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index 0e74502..4dea37e 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -404,7 +404,8 @@ func declarationWith(ctx context.Context, open *stores, node string, if err != nil { return sendable{}, err } - return sendable{Resources: composed.Resources, Adoption: adoption}, nil + return sendable{Resources: composed.Resources, Adoption: adoption, + Received: composed.Received, Mesh: with.Mesh}, nil } // renderingFor is everything a node's declaration is composed with, and the node's record. diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 27d00f9..0435863 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -389,11 +389,7 @@ func pushCommand(ctx context.Context, args []string) error { fmt.Printf("\n%d node(s) told\n", len(sending)) // And each machine's memberships, as every other send does (ADR 0160): a push is the one most // operators run, and on 2026-10-01 it was the one path that issued none. - var told []string - for _, s := range sending { - told = append(told, s.node) - } - if err := issueMemberships(ctx, open, server, told); err != nil { + if err := issueMemberships(ctx, open, server, sending); err != nil { return err } @@ -706,11 +702,15 @@ func sendTo(ctx context.Context, open *stores, names []string) error { // And every assignment on those machines its membership (novox/hq ADR 0160): composed from the // same records the bus's accounts are, so what a runtime serves and what its account may are one // composition. Issued after the declaration, because the runtime it is for arrives with it. - return issueMemberships(ctx, open, server, names) + return issueMemberships(ctx, open, server, sending) } -// issueMemberships publishes the membership of every module on the named machines. -func issueMemberships(ctx context.Context, open *stores, server *link.Server, names []string) error { +// issueMemberships publishes the membership of every module on the machines just sent. +// +// Each carries what its module receives and the private network's addresses, from the same +// composition as the declaration it was sent (novox/hq ADR 0167): a provider reads what it is +// given on the bus, and the file written beside it says the same thing. +func issueMemberships(ctx context.Context, open *stores, server *link.Server, sent []readyNode) error { records, err := open.inventory.BusRecords(ctx) if err != nil { return err @@ -725,9 +725,22 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, na // the push stands, the first failure is named once, and the next push tries again. issued, failed := 0, 0 var first error - for _, node := range names { + for _, s := range sent { + node := s.node for _, d := range records.Assigned[node] { - body, err := json.Marshal(broker.MembershipFor(node, d, where)) + membership := broker.MembershipFor(node, d, where) + membership.Mesh = s.declared.Mesh + for requirement, given := range s.declared.Received[d.Module] { + raw, err := json.Marshal(given) + if err != nil { + return err + } + if membership.Receives == nil { + membership.Receives = map[string]json.RawMessage{} + } + membership.Receives[requirement] = raw + } + body, err := json.Marshal(membership) if err != nil { return err } diff --git a/cmd/mesh-controller/sendable.go b/cmd/mesh-controller/sendable.go index f6a69d5..1ca2532 100644 --- a/cmd/mesh-controller/sendable.go +++ b/cmd/mesh-controller/sendable.go @@ -25,6 +25,12 @@ type sendable struct { // Adoption is nil for a converged node, and then the body is byte for byte what it was before // adoption existed: an older host parses the envelope strictly and would refuse the key. Adoption *adoptionEnvelope + + // Received and Mesh are not sent in the declaration. They are what this machine's memberships + // are issued with on the bus (novox/hq ADR 0167): each module's received contributions, from + // the same composition as its received files, and every machine's private-network address. + Received map[string]map[string][]catalogue.Contribution + Mesh []string } // adoptionEnvelope is what an adopted node is told about its mode. Taken is every module taken on diff --git a/examples/route-proxy/bus.go b/examples/route-proxy/bus.go new file mode 100644 index 0000000..51ed510 --- /dev/null +++ b/examples/route-proxy/bus.go @@ -0,0 +1,132 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "os" + "strings" + "sync/atomic" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" +) + +// What the mesh issued this proxy, read on the bus (novox/hq ADR 0160, ADR 0167). +// +// **The proxy is told, not left to work it out.** Its membership carries the routes it is given — +// the same contributions its file is written from — and every machine's address on the private +// network, which is who may be served an internal name. Read once at connect and followed, so a +// route added or a machine joining reaches a running proxy without a restart. + +// credential is the bus account the mesh delivered as this module's own secret named broker. +type credential struct { + URL string `json:"url"` + Fingerprint string `json:"fingerprint"` + Node string `json:"node"` + Module string `json:"module"` + User string `json:"user"` + Password string `json:"password"` +} + +// followMembership connects with the credential in path and applies every membership the mesh +// issues this proxy. It retries the first connection for as long as it takes: a proxy that started +// before the bus keeps serving the file, and takes the bus when it answers. +func followMembership(path string, held *table, fromBus *atomic.Bool) { + for { + err := followOnce(path, held, fromBus) + if err == nil { + return + } + log.Printf("cannot follow this proxy's membership, serving the file meanwhile: %v", err) + time.Sleep(30 * time.Second) + } +} + +func followOnce(path string, held *table, fromBus *atomic.Bool) error { + raw, err := os.ReadFile(path) + if err != nil { + return err + } + var cred credential + if err := json.Unmarshal(raw, &cred); err != nil { + return fmt.Errorf("the broker credential is not one: %w", err) + } + if cred.Node == "" || cred.Module == "" { + return fmt.Errorf("the broker credential names no node or module, so it has no membership") + } + + opts := []nats.Option{ + nats.Name(cred.Node + "." + cred.Module), + nats.UserInfo(cred.User, cred.Password), + // Its own inbox, and nothing wider: every principal is granted `_INBOX..>` alone. + nats.CustomInboxPrefix("_INBOX." + cred.User), + // The bus being restarted is an upgrade, not a reason to stop following. + nats.MaxReconnects(-1), + } + if strings.TrimSpace(cred.Fingerprint) != "" { + opts = append(opts, nats.Secure(broker.PinnedToFingerprint(cred.Fingerprint))) + } + conn, err := nats.Connect(cred.URL, opts...) + if err != nil { + return fmt.Errorf("connecting to the bus at %s: %w", broker.BareAddress(cred.URL), err) + } + + subject := broker.MembershipSubject(cred.Node, cred.Module) + apply := func(body []byte) { + var issued broker.Membership + if err := json.Unmarshal(body, &issued); err != nil { + log.Printf("a membership arrived that is not one: %v", err) + return + } + if took := applyMembership(issued, held); took && !fromBus.Swap(true) { + log.Printf("routes now come from this proxy's membership on %s", subject) + } + } + + // Followed first, read second: an issue landing between the two is applied, not missed. + if _, err := conn.Subscribe(subject, func(m *nats.Msg) { apply(m.Data) }); err != nil { + conn.Close() + return fmt.Errorf("cannot follow %s: %w", subject, err) + } + // The subject-addressed direct get: the one request this account may make of the stream. + got, err := conn.Request("$JS.API.DIRECT.GET."+broker.AssignmentsStream+"."+subject, nil, 5*time.Second) + switch { + case err != nil: + log.Printf("cannot read the membership issued on %s yet (%v); following it", subject, err) + case got.Header.Get("Status") != "" || len(got.Data) == 0: + log.Printf("no membership issued on %s yet; serving the file until one is", subject) + default: + apply(got.Data) + } + return nil +} + +// applyMembership serves what a membership says, and says whether it said anything about routes. +// +// A membership with no routes in it is one from a controller older than ADR 0167, and the file stays +// the source rather than every route being withdrawn because a field was absent. +func applyMembership(issued broker.Membership, held *table) bool { + raw, carries := issued.Receives["route"] + if !carries { + return false + } + var contributions []contribution + if err := json.Unmarshal(raw, &contributions); err != nil { + log.Printf("the routes in this proxy's membership are not contributions, keeping what is served: %v", err) + return false + } + inside, err := sourcesOf(issued.Mesh) + if err != nil { + log.Printf("the mesh in this proxy's membership is unreadable, keeping what is served: %v", err) + return false + } + routes, public := routesOf(contributions) + held.set(routes, public) + held.setInside(inside) + log.Printf("serving %d route(s) from the membership, internal names to %d machine(s): %s", + len(routes), len(inside), strings.Join(held.names(), ", ")) + return true +} diff --git a/examples/route-proxy/main.go b/examples/route-proxy/main.go index 484ce2a..47d4f93 100644 --- a/examples/route-proxy/main.go +++ b/examples/route-proxy/main.go @@ -54,12 +54,14 @@ import ( "net" "net/http" "net/http/httputil" + "net/netip" "net/url" "os" "path/filepath" "sort" "strings" "sync" + "sync/atomic" "time" "golang.org/x/crypto/acme" @@ -196,6 +198,63 @@ type table struct { // pass ACME's own validation (it has no public DNS to prove it against), so asking for it is // not merely pointless but the failing order onlyWhatTheMeshSaid exists to prevent. public map[string]bool + // inside is where a request must come from to be served a name that is only internal: every + // machine's address on the private network, as the mesh issued it in this proxy's membership + // (novox/hq ADR 0167). Empty until it is issued, and then only the machine itself is inside. + inside sources +} + +// sources is who may be served an internal name: the private network's addresses as the mesh +// issued them. The machine itself is always inside — anything on a machine may call anything on it +// (novox/hq ADR 0144) — so loopback needs no entry. +type sources []netip.Prefix + +// sourcesOf reads the addresses the mesh issued, each a single address or a range. One that does +// not parse is an error, not an entry skipped: the proxy would otherwise serve internal names to +// fewer machines than the mesh said, and say nothing. +func sourcesOf(mesh []string) (sources, error) { + var out sources + for _, entry := range mesh { + entry = strings.TrimSpace(entry) + if prefix, err := netip.ParsePrefix(entry); err == nil { + out = append(out, prefix.Masked()) + continue + } + addr, err := netip.ParseAddr(entry) + if err != nil { + return nil, fmt.Errorf("%q is not an address on the private network", entry) + } + addr = addr.Unmap() + out = append(out, netip.PrefixFrom(addr, addr.BitLen())) + } + return out, nil +} + +// holds says whether a request from this remote address came from the mesh or the machine itself. +// +// **By source, which the mesh's guard deliberately is not** — it names interfaces, because a source +// address can be claimed by whoever sends the packet. The proxy cannot see the interface a request +// arrived on, and here the claim does not carry: a connection needs its replies, and replies to a +// mesh address leave by the tunnel, never back to the claimant. +func (s sources) holds(remote string) bool { + host := remote + if h, _, err := net.SplitHostPort(remote); err == nil { + host = h + } + addr, err := netip.ParseAddr(host) + if err != nil { + return false + } + addr = addr.Unmap() + if addr.IsLoopback() { + return true + } + for _, prefix := range s { + if prefix.Contains(addr) { + return true + } + } + return false } func (t *table) set(routes map[string][]rule, public map[string]bool) { @@ -314,6 +373,41 @@ func bareHost(host string) string { return strings.ToLower(host) } +// hiddenFrom says whether this host must look unrouted to a request from this address: it is +// only an internal name, and the request did not come from the private network. +// +// **The proxy is the only way in to a routed endpoint, so it is what makes `internal` true** +// (novox/hq ADR 0138, issue 191). It answers public names on the same listeners, so a request from +// anywhere can carry any Host header; a name being internal keeps nobody out unless this check does. +// Answered exactly as a name that was never routed, so an outsider learns nothing from asking. +func (t *table) hiddenFrom(host, remote string) bool { + if !t.eligibleForInternalACME(host) { + return false + } + t.mu.RLock() + defer t.mu.RUnlock() + return !t.inside.holds(remote) +} + +// setInside replaces who the mesh is, as the membership said. +func (t *table) setInside(inside sources) { + t.mu.Lock() + t.inside = inside + t.mu.Unlock() +} + +// namesSeenFrom is what this proxy says it serves to a request from this address — every routed +// name, less the internal-only ones when the request came from outside. +func (t *table) namesSeenFrom(remote string) []string { + out := []string{} + for _, name := range t.names() { + if !t.hiddenFrom(name, remote) { + out = append(out, name) + } + } + return out +} + func (t *table) names() []string { t.mu.RLock() defer t.mu.RUnlock() @@ -343,7 +437,20 @@ func run() error { } held := newTable() + // **The bus first, the file until it has spoken** (novox/hq ADR 0167). The membership carries + // the routes and who the mesh is; the file carries the routes alone, so while the proxy reads + // it an internal name is served to this machine and to nobody else — refused, never opened. + fromBus := &atomic.Bool{} + if credential := strings.TrimSpace(os.Getenv("MESH_BROKER_FILE")); credential != "" { + go followMembership(credential, held, fromBus) + } else { + log.Printf("MESH_BROKER_FILE is not set: routes come from %s alone, and a name that is only "+ + "internal is served to this machine alone", path) + } read := func() { + if fromBus.Load() { + return + } routes, public, err := routesFrom(path) if err != nil { // Kept serving what it had. A file being rewritten is momentarily unreadable, and @@ -422,19 +529,7 @@ func run() error { }() tlsConfig := publicManager.TLSConfig() - if internalManager != nil { - // Dispatched by which authority may certify this name at all — the same question - // eligibleForInternalACME already answers, asked once more at handshake time rather than - // only when an order is placed, since a cached certificate is served here on every request - // and never goes through HostPolicy again. - fromPublic, fromInternal := tlsConfig.GetCertificate, internalManager.TLSConfig().GetCertificate - tlsConfig.GetCertificate = func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { - if held.eligibleForInternalACME(hello.ServerName) { - return fromInternal(hello) - } - return fromPublic(hello) - } - } + tlsConfig.GetCertificate = certificateFor(held, tlsConfig.GetCertificate, internalManager) server := &http.Server{ Addr: secure, @@ -592,6 +687,33 @@ func forThisAuthority(cache, directory string, root []byte) string { return filepath.Join(cache, hex.EncodeToString(sum[:])[:16]) } +// certificateFor picks the certificate a handshake is answered with. +// +// Dispatched by which authority may certify this name at all — the same question +// eligibleForInternalACME already answers, asked once more at handshake time rather than only when +// an order is placed, since a cached certificate is served here on every request and never goes +// through HostPolicy again. And refused, exactly as an unrouted name is, to a client outside the +// private network asking for a name that is only internal: the certificate would name it. +func certificateFor(held *table, fromPublic func(*tls.ClientHelloInfo) (*tls.Certificate, error), + internalManager *autocert.Manager) func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + var fromInternal func(*tls.ClientHelloInfo) (*tls.Certificate, error) + if internalManager != nil { + fromInternal = internalManager.TLSConfig().GetCertificate + } + return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { + if held.eligibleForInternalACME(hello.ServerName) { + if hello.Conn != nil && held.hiddenFrom(hello.ServerName, hello.Conn.RemoteAddr().String()) { + return nil, fmt.Errorf("no public route for %q in this mesh, so no certificate is asked for", + hello.ServerName) + } + if fromInternal != nil { + return fromInternal(hello) + } + } + return fromPublic(hello) + } +} + // newTable is an empty routing table. func newTable() *table { return &table{to: map[string][]rule{}} @@ -600,8 +722,9 @@ func newTable() *table { // handler is the proxy itself, separated so it can be driven by a test without a listener. func handler(held *table) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hidden := held.hiddenFrom(r.Host, r.RemoteAddr) matched, known := held.find(r.Host, r.URL.Path) - if !known { + if hidden || !known { // **Named, not a bare 404.** A route that was withdrawn and a name that never existed // are different things, and a proxy that says only "not found" makes an operator go // and read the mesh to tell them apart. What it is serving is the answer to both. @@ -611,13 +734,13 @@ func handler(held *table) http.Handler { // contradiction an operator would have to disbelieve the proxy to get past. w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.WriteHeader(http.StatusNotFound) - if held.routed(r.Host) { + if !hidden && held.routed(r.Host) { fmt.Fprintf(w, "%s is served here, but no route covers %q.\n", bareHost(r.Host), r.URL.Path) return } fmt.Fprintf(w, "no route for %q in this mesh.\nserving: %s\n", - r.Host, strings.Join(held.names(), ", ")) + r.Host, strings.Join(held.namesSeenFrom(r.RemoteAddr), ", ")) return } @@ -731,10 +854,16 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) { if err := json.Unmarshal(raw, &said); err != nil { return nil, nil, err } + routes, public := routesOf(said.Given) + return routes, public, nil +} +// routesOf turns what the mesh gave into host → the rules for that host, and which hosts are public +// names — the same whether the contributions came in the file or in the membership. +func routesOf(contributions []contribution) (map[string][]rule, map[string]bool) { out := map[string][]rule{} public := map[string]bool{} - for _, c := range said.Given { + for _, c := range contributions { name, _ := c.Values["name"].(string) name = strings.TrimSpace(name) internal, _ := c.Values["internal-name"].(string) @@ -839,7 +968,7 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) { out[strings.ToLower(internal)] = append(out[strings.ToLower(internal)], made) } } - return out, public, nil + return out, public } // asWhole is any whole number the mesh wrote, whatever its magnitude. diff --git a/examples/route-proxy/reach_test.go b/examples/route-proxy/reach_test.go new file mode 100644 index 0000000..151a41a --- /dev/null +++ b/examples/route-proxy/reach_test.go @@ -0,0 +1,206 @@ +package main + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/broker" +) + +// behind is a workload the proxy can send to, and a table routing one public name and one +// internal-only name to it, with the mesh's machines as the membership would issue them. +func behind(t *testing.T, mesh ...string) *table { + t.Helper() + workload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, "the workload") + })) + t.Cleanup(workload.Close) + at, _ := url.Parse(workload.URL) + host, port, _ := net.SplitHostPort(at.Host) + + routes, public, err := routesFrom(write(t, fmt.Sprintf(`{"given":[ + {"from":"app","node":"anchor","at":%q, + "values":{"name":"app.example","internal-name":"app.anchor.internal","port":%s}}, + {"from":"admin","node":"anchor","at":%q, + "values":{"internal-name":"admin.anchor.internal","port":%s}} + ]}`, host, port, host, port))) + if err != nil { + t.Fatal(err) + } + held := newTable() + inside, err := sourcesOf(mesh) + if err != nil { + t.Fatal(err) + } + held.setInside(inside) + held.set(routes, public) + return held +} + +// askFrom is what the proxy answers a request for host coming from remote. +func askFrom(held *table, host, remote string) (int, string) { + r := httptest.NewRequest(http.MethodGet, "http://"+host+"/", nil) + r.RemoteAddr = remote + w := httptest.NewRecorder() + handler(held).ServeHTTP(w, r) + return w.Code, w.Body.String() +} + +// **An internal-only name is served to the private network and to nobody else** (novox/hq ADR +// 0138, issue 191). The proxy answers public names on the same listeners, so without this a name +// being internal kept nobody out: a request from the internet only had to carry it. +func TestAnInternalOnlyNameIsServedOnlyInsideThePrivateNetwork(t *testing.T) { + held := behind(t, "10.10.0.1", "10.10.0.7") + + if code, body := askFrom(held, "admin.anchor.internal", "10.10.0.7:51000"); code != http.StatusOK || + body != "the workload" { + t.Errorf("a request from the private network was not served: %d %q", code, body) + } + if code, body := askFrom(held, "admin.anchor.internal", "127.0.0.1:51000"); code != http.StatusOK { + t.Errorf("a request from the machine itself was not served: %d %q", code, body) + } + + code, body := askFrom(held, "admin.anchor.internal", "203.0.113.9:51000") + if code != http.StatusNotFound { + t.Fatalf("a request from outside the private network reached an internal-only name: %d %q", + code, body) + } + // Answered as a name never routed, and the list of what is served does not name it either — + // otherwise the refusal would tell an outsider exactly what to ask for from inside. + if strings.Contains(strings.SplitN(body, "\n", 2)[1], "admin.anchor.internal") { + t.Errorf("the refusal names the internal-only route to an outsider: %q", body) + } + if !strings.Contains(body, "app.example") { + t.Errorf("the refusal stopped listing the public names: %q", body) + } +} + +// The internal name of a route that also has a public one is internal too: served inside, and to +// an outsider only under the public name. Nothing is lost — the outsider has the public name — and a +// name stays one thing whichever route it came from. +func TestAnInternalAliasOfAPublicRouteIsServedInsideOnly(t *testing.T) { + held := behind(t, "10.10.0.1", "10.10.0.7") + if code, body := askFrom(held, "app.anchor.internal", "10.10.0.7:51000"); code != http.StatusOK { + t.Errorf("the internal alias stopped answering the private network: %d %q", code, body) + } + if code, _ := askFrom(held, "app.anchor.internal", "203.0.113.9:51000"); code != http.StatusNotFound { + t.Errorf("the internal alias was served to an outsider: %d", code) + } + if code, _ := askFrom(held, "app.example", "203.0.113.9:51000"); code != http.StatusOK { + t.Errorf("the public name was refused to an outsider: %d", code) + } +} + +// Before a membership has said who the mesh is, only the machine itself is inside — refused to +// everyone else, never served to everyone. +func TestUntilTheMeshIsIssuedAnInternalOnlyNameIsServedToTheMachineAlone(t *testing.T) { + held := behind(t) + if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:51000"); code != http.StatusNotFound { + t.Errorf("an internal-only name was served with no private network said: %d", code) + } + if code, _ := askFrom(held, "admin.anchor.internal", "[::1]:51000"); code != http.StatusOK { + t.Errorf("an internal-only name was refused to the machine itself: %d", code) + } +} + +type from struct { + net.Conn + remote net.Addr +} + +func (c from) RemoteAddr() net.Addr { return c.remote } + +// The handshake refuses an internal-only name to an outsider too: the certificate would name it, +// and serving it would answer the question the routing refuses to. +func TestTheHandshakeRefusesAnInternalOnlyNameToAnOutsider(t *testing.T) { + held := behind(t, "10.10.0.1", "10.10.0.7") + served := &tls.Certificate{} + pick := certificateFor(held, func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return served, nil }, nil) + hello := func(name, remote string) *tls.ClientHelloInfo { + addr, _ := net.ResolveTCPAddr("tcp", remote) + return &tls.ClientHelloInfo{ServerName: name, Conn: from{remote: addr}} + } + + if _, err := pick(hello("admin.anchor.internal", "203.0.113.9:443")); err == nil { + t.Error("an outsider was handed a certificate for an internal-only name") + } + if got, err := pick(hello("admin.anchor.internal", "10.10.0.7:443")); err != nil || got != served { + t.Errorf("a client on the private network was refused: %v", err) + } + if got, err := pick(hello("app.example", "203.0.113.9:443")); err != nil || got != served { + t.Errorf("a public name was refused to an outsider: %v", err) + } +} + +// The mesh is issued as machines' addresses; a range is read as well. One that does not parse is +// refused rather than skipped, so a typo never quietly narrows or widens who is inside. +func TestTheMeshIsReadAsAddressesAndRanges(t *testing.T) { + if _, err := sourcesOf([]string{"10.10.0.1", "not-an-address"}); err == nil { + t.Error("an entry that is not an address was accepted") + } + inside, err := sourcesOf([]string{"10.10.0.1", "fd00::1", "10.20.0.0/24"}) + if err != nil { + t.Fatal(err) + } + for remote, want := range map[string]bool{ + "10.10.0.1:1": true, + "[::ffff:10.10.0.1]:1": true, + "[fd00::1]:1": true, + "10.20.0.200:1": true, + "10.10.0.2:1": false, + "192.168.1.10:1": false, + "not-an-address": false, + } { + if inside.holds(remote) != want { + t.Errorf("%s inside the mesh: got %v, want %v", remote, !want, want) + } + } +} + +// What the mesh issues is what is served: the routes in the membership, internal names to the +// machines it names (novox/hq ADR 0167). +func TestAMembershipIsServedAsIssued(t *testing.T) { + held := newTable() + took := applyMembership(broker.Membership{ + Receives: map[string]json.RawMessage{"route": json.RawMessage(`[ + {"from":"admin","node":"anchor","at":"anchor.internal", + "values":{"internal-name":"admin.anchor.internal","port":8080}}]`)}, + Mesh: []string{"10.10.0.7"}, + }, held) + if !took { + t.Fatal("a membership carrying routes was not applied") + } + if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:1"); code == http.StatusNotFound { + t.Error("a machine the membership names was refused the internal-only route") + } + if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.9:1"); code != http.StatusNotFound { + t.Errorf("a machine the membership does not name was served the internal-only route: %d", code) + } +} + +// A membership that says nothing about routes is one from a controller that does not issue them, +// and changes nothing: the file stays the source rather than every route being withdrawn. +func TestAMembershipWithoutRoutesLeavesTheFileServing(t *testing.T) { + held := behind(t, "10.10.0.7") + before := held.names() + if applyMembership(broker.Membership{Mesh: []string{"10.10.0.7"}}, held) { + t.Error("a membership without routes was taken as the source of routes") + } + if got := held.names(); strings.Join(got, ",") != strings.Join(before, ",") { + t.Errorf("a membership without routes changed what is served: %v, was %v", got, before) + } + if applyMembership(broker.Membership{ + Receives: map[string]json.RawMessage{"route": json.RawMessage(`[]`)}, + Mesh: []string{"not-an-address"}, + }, held) { + t.Error("a membership whose mesh cannot be read was applied") + } +} diff --git a/internal/broker/membership.go b/internal/broker/membership.go index dbd99c5..8cf9b98 100644 --- a/internal/broker/membership.go +++ b/internal/broker/membership.go @@ -1,6 +1,7 @@ package broker import ( + "encoding/json" "sort" "strings" ) @@ -33,6 +34,17 @@ type Membership struct { Reaches map[string][]string `json:"reaches,omitempty"` // Tools is where this instance answers what it serves — the runtime's one verb of its own. Tools string `json:"tools"` + // Receives is what this assignment is given for each requirement it receives, by requirement: + // the contributions of every module that asked for it, as the catalogue composed them (novox/hq + // ADR 0167). The same list its received file is written from, so the two cannot disagree; a + // requirement nobody contributed to is an empty list, never absent. Kept as JSON because the + // catalogue owns the shape of a contribution and the bus only carries it. + Receives map[string]json.RawMessage `json:"receives,omitempty"` + // Mesh is every machine's address on the private network — what a rule saying "from the mesh" + // resolves to in the packet filter, issued here from the same list (novox/hq ADR 0167). A + // module that must tell the mesh from the world, the route proxy serving an internal name, reads + // it here rather than keeping a definition of its own. + Mesh []string `json:"mesh,omitempty"` } // Served is one address a tool is answered on. diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 41e77e5..527e0d2 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -241,15 +241,21 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) { // Owner is kept beside the resources because a resource id cannot be split back into its module: // a module's name may itself contain a dot. What the mesh adds of its own — an opening, the guard — // has no owner. +// +// Received is what each module on the machine is given for each requirement it receives — the same +// contributions its received file is written from, kept beside it so the mesh can also issue them +// on the bus in the module's membership (novox/hq ADR 0167). By module, then requirement. type Composed struct { Resources []map[string]any Owner map[string]string + Received map[string]map[string][]Contribution } // Compose is Declaration with the owner of every resource said. func (r Resolution) Compose(with Rendering) (Composed, error) { owner := map[string]string{} - resources, err := r.compose(with, owner) + received := map[string]map[string][]Contribution{} + resources, err := r.compose(with, owner, received) if err != nil { return Composed{}, err } @@ -261,7 +267,7 @@ func (r Resolution) Compose(with Rendering) (Composed, error) { "sealed": with.BusMembership, "mode": "0600", }) } - return Composed{Resources: resources, Owner: owner}, nil + return Composed{Resources: resources, Owner: owner, Received: received}, nil } // BusMembershipID names the resource carrying a machine's membership for the new bus, and @@ -270,7 +276,8 @@ func BusMembershipID() string { return "bus-membership" } const BusMembershipPath = "/var/lib/mesh/membership-next.json" -func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[string]any, error) { +func (r Resolution) compose(with Rendering, owner map[string]string, + received map[string]map[string][]Contribution) ([]map[string]any, error) { // Every manifest is placed first (novox/hq ADR 0112): the maps naming where its bindings, // credentials and contributions land are resolved against this node's directories, so every // reader below — the binding files, the sealed secrets, the grant paths a contribution @@ -596,6 +603,12 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri return nil, err } first = append(first, file) + if received[m.Module] == nil { + received[m.Module] = map[string][]Contribution{} + } + // Empty rather than absent when nobody contributed, for the reason the file is + // written empty: "nothing asked" and "never told" want different responses. + received[m.Module][to] = append([]Contribution{}, given[to]...) } if m.Keeps != "" && with.Kept != nil { file, err := keptFile(m.Keeps, with.Kept) diff --git a/internal/catalogue/received_on_the_bus_test.go b/internal/catalogue/received_on_the_bus_test.go new file mode 100644 index 0000000..185b10b --- /dev/null +++ b/internal/catalogue/received_on_the_bus_test.go @@ -0,0 +1,66 @@ +package catalogue + +import ( + "encoding/json" + "reflect" + "testing" +) + +// What a provider receives is composed once, and issued twice: as its received file, and in its +// membership on the bus (novox/hq ADR 0167). The two are the same list, so a proxy reading the bus +// and one reading the file serve the same routes — including the port the machine published, which +// is the same-node fix the file already carries. +func TestWhatAProviderReceivesIsTheSameOnTheBusAsInItsFile(t *testing.T) { + gitea := Manifest{ + Module: "gitea", Version: "1", + Listens: []Listening{{Port: 3000, Protocol: "tcp", From: FromMesh}}, + Contributes: map[string]map[string]any{"route": {"label": "git", "port": 3000}}, + Resources: []map[string]any{{ + "id": "server", "type": "container", "name": "gitea", "ports": []any{"3000"}, + }}, + } + r, err := Resolve(shelf(gitea, routeProxy(), stepCA()), + []string{"gitea", "route-proxy", "step-ca"}, reachable(), World{}) + if err != nil { + t.Fatal(err) + } + composed, err := r.Compose(Rendering{Ports: map[string]map[int]int{"gitea": {3000: 20000}}}) + if err != nil { + t.Fatal(err) + } + + file := fileNamed(composed.Resources, "route-proxy.received-route") + if file == nil { + t.Fatal("the proxy was given no routes file") + } + var written struct { + Given []Contribution `json:"given"` + } + if err := json.Unmarshal([]byte(file["content"].(string)), &written); err != nil { + t.Fatal(err) + } + issued, said := composed.Received["route-proxy"]["route"] + if !said { + t.Fatalf("nothing is issued for the proxy to receive on the bus: %v", composed.Received) + } + // Compared as JSON, which is what both are once they leave the controller. + a, _ := json.Marshal(written.Given) + b, _ := json.Marshal(issued) + var fromFile, fromBus any + _ = json.Unmarshal(a, &fromFile) + _ = json.Unmarshal(b, &fromBus) + if !reflect.DeepEqual(fromFile, fromBus) { + t.Errorf("the bus and the file disagree about the routes:\nfile %s\nbus %s", a, b) + } + if len(issued) != 1 { + t.Fatalf("expected one route on the bus, got %v", issued) + } + if port, ok := asPort(issued[0].Values["port"]); !ok || port != 20000 { + t.Errorf("the bus carries a port nothing listens on: %v", issued[0].Values["port"]) + } + + // A module that receives nothing is issued nothing to receive. + if _, any := composed.Received["gitea"]; any { + t.Errorf("a module that receives nothing was issued something: %v", composed.Received["gitea"]) + } +}