Merge pull request 'The mesh makes the bus's certificate itself' (#145) from fix/the-mesh-makes-its-own-bus-certificate into main

novox/hq 04-ISSUES/146. The foundation made it by running openssl inside the
broker's image, which worked while the broker was one that carried it and
stopped the day the bus changed: the new one has a shell and no openssl, so
the step exited 127 and no mesh could be raised. No other image the bundle
names has it either, so there was nothing to substitute.

broker certificate --into <dir> writes the pair, --check is the step's verify.
Self-signed on purpose — a host pins this server's exact certificate (ADR
0004) and at genesis there is no authority to ask — and made once, because a
second certificate is one every host that pinned the first no longer believes.
The key is written before the certificate, so an interruption never leaves
something that looks finished.

cmd/mesh-controller/buscertificate.go

@@ -0,0 +1,171 @@
+package main
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"math/big"
+	"net"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+// The bus's own certificate, made by the mesh rather than borrowed from an image.
+//
+// **The foundation asked a third-party image for a tool it never said must be there** (novox/hq
+// 04-ISSUES/146). The bootstrap made this certificate by running `openssl` inside the broker's
+// image, which worked while the broker was one that happened to carry it and stopped the day the
+// bus changed: the new one has a shell and no openssl, so the step exited 127 and no mesh could be
+// raised. Substituting another image the bundle names does not help — none of them carry it
+// either.
+//
+// So the program that needs a certificate makes one. It is the mesh's own binary, already on the
+// machine at this point in the bootstrap (the schema step ran it), and it needs nothing from the
+// image it writes into but a mounted directory.
+//
+// **Self-signed, and that is the design** — a host pins this server's exact certificate and
+// authenticates with a password (novox/hq ADR 0004). There is no authority above it to ask, and at
+// this moment in a bootstrap there is no mesh to ask one of.
+//
+// Idempotent, because the step is applied again on every reconcile and a second certificate would
+// be one the hosts that pinned the first no longer believe.
+
+// busCertificateNames is what the bus is reached by: the container name on a mesh network, and the
+// loopback address the machine's own foundation dials.
+var busCertificateNames = []string{"mesh-broker"}
+
+const busCertificateLife = 10 * 365 * 24 * time.Hour
+
+// busCertificate makes the bus's certificate in a directory, or says whether one is there.
+//
+//	broker certificate --into /tls           make it if it is not there
+//	broker certificate --check --into /tls   exit non-zero unless a usable pair is
+func busCertificate(args []string) error {
+	into, check := "", false
+	for i := 0; i < len(args); i++ {
+		switch args[i] {
+		case "--check":
+			check = true
+		case "--into":
+			if i+1 >= len(args) {
+				return errors.New("--into needs a directory")
+			}
+			into = args[i+1]
+			i++
+		default:
+			return fmt.Errorf("broker certificate [--check] --into <directory>: %q", args[i])
+		}
+	}
+	if into == "" {
+		return errors.New("broker certificate [--check] --into <directory>")
+	}
+	crt, key := filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key")
+
+	if usable, err := busCertificateUsable(crt, key); err != nil {
+		return err
+	} else if usable {
+		fmt.Printf("the bus already has a certificate at %s, and it was left alone\n", crt)
+		return nil
+	}
+	if check {
+		// Said as a failure, because that is what the caller asked: a bootstrap's verify runs
+		// this and a false answer is what makes the step run.
+		return fmt.Errorf("no usable certificate and key at %s", into)
+	}
+	return writeBusCertificate(crt, key)
+}
+
+// busCertificateUsable says whether a certificate and its key are both there and parse.
+//
+// Both, and parsed rather than stat'ed: a half-written pair is the state a bootstrap interrupted
+// between the two files leaves behind, and a step that treated it as done would hand the server a
+// certificate with no key and report success.
+func busCertificateUsable(crt, key string) (bool, error) {
+	certPEM, err := os.ReadFile(crt)
+	if errors.Is(err, os.ErrNotExist) {
+		return false, nil
+	}
+	if err != nil {
+		return false, err
+	}
+	keyPEM, err := os.ReadFile(key)
+	if errors.Is(err, os.ErrNotExist) {
+		return false, nil
+	}
+	if err != nil {
+		return false, err
+	}
+	if _, err := tlsPairParses(certPEM, keyPEM); err != nil {
+		return false, nil
+	}
+	return true, nil
+}
+
+func tlsPairParses(certPEM, keyPEM []byte) (*x509.Certificate, error) {
+	block, _ := pem.Decode(certPEM)
+	if block == nil || block.Type != "CERTIFICATE" {
+		return nil, errors.New("not a certificate")
+	}
+	certificate, err := x509.ParseCertificate(block.Bytes)
+	if err != nil {
+		return nil, err
+	}
+	keyBlock, _ := pem.Decode(keyPEM)
+	if keyBlock == nil {
+		return nil, errors.New("not a key")
+	}
+	if _, err := x509.ParsePKCS8PrivateKey(keyBlock.Bytes); err != nil {
+		if _, err := x509.ParsePKCS1PrivateKey(keyBlock.Bytes); err != nil {
+			return nil, err
+		}
+	}
+	return certificate, nil
+}
+
+func writeBusCertificate(crt, key string) error {
+	private, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		return err
+	}
+	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
+	if err != nil {
+		return err
+	}
+	template := &x509.Certificate{
+		SerialNumber:          serial,
+		Subject:               pkix.Name{CommonName: busCertificateNames[0]},
+		DNSNames:              busCertificateNames,
+		IPAddresses:           []net.IP{net.ParseIP("127.0.0.1")},
+		NotBefore:             time.Now().Add(-time.Hour),
+		NotAfter:              time.Now().Add(busCertificateLife),
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, template, template, &private.PublicKey, private)
+	if err != nil {
+		return err
+	}
+	pkcs8, err := x509.MarshalPKCS8PrivateKey(private)
+	if err != nil {
+		return err
+	}
+
+	// **The key first, and only then the certificate**, so the pair a reader finds is never a
+	// certificate whose key has not been written yet — the one order in which an interruption
+	// leaves something that looks finished (novox/hq 04-ISSUES/014, a key present and unusable).
+	if err := os.WriteFile(key, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8}), 0o600); err != nil {
+		return err
+	}
+	if err := os.WriteFile(crt, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644); err != nil {
+		return err
+	}
+	fmt.Printf("made the bus a certificate for %v, valid until %s\n  %s\n  %s\n",
+		busCertificateNames, template.NotAfter.Format(time.RFC3339), crt, key)
+	return nil
+}

cmd/mesh-controller/buscertificate_test.go

@@ -0,0 +1,121 @@
+package main
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// novox/hq 04-ISSUES/146. The bootstrap could not make the bus a certificate: it asked an image for
+// `openssl` and the image it asks has none. What replaces it is this command, so what is checked is
+// what the bootstrap needs from it — a pair a TLS server can actually load, made once and only once.
+
+func TestTheBusCertificateLoadsAsAServersWould(t *testing.T) {
+	into := t.TempDir()
+	if err := busCertificate([]string{"--into", into}); err != nil {
+		t.Fatalf("the bus could not be given a certificate: %v", err)
+	}
+
+	// The check a cheaper test would not make. The key was present and valid and the server could
+	// not start, once, because nothing loaded the pair the way a server loads it
+	// (novox/hq 04-ISSUES/014).
+	pair, err := tls.LoadX509KeyPair(filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key"))
+	if err != nil {
+		t.Fatalf("a TLS server cannot load what was written: %v", err)
+	}
+	leaf := pair.Leaf
+	if leaf == nil {
+		if leaf, err = x509.ParseCertificate(pair.Certificate[0]); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := leaf.VerifyHostname("mesh-broker"); err != nil {
+		t.Errorf("the certificate is not for the name the bus is reached by: %v", err)
+	}
+	if len(leaf.IPAddresses) == 0 || leaf.IPAddresses[0].String() != "127.0.0.1" {
+		t.Errorf("the certificate does not cover the loopback address the foundation dials: %v", leaf.IPAddresses)
+	}
+
+	// The key is not readable by anything else on the machine; the certificate is public and is.
+	key, err := os.Stat(filepath.Join(into, "tls.key"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if key.Mode().Perm() != 0o600 {
+		t.Errorf("the key is %v", key.Mode().Perm())
+	}
+	crt, err := os.Stat(filepath.Join(into, "tls.crt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if crt.Mode().Perm() != 0o644 {
+		t.Errorf("the certificate is %v, which the server runs as another user cannot read", crt.Mode().Perm())
+	}
+}
+
+// **Made once.** The step is applied again on every reconcile, and a second certificate is one the
+// hosts that pinned the first no longer believe (novox/hq ADR 0004).
+func TestTheBusCertificateIsMadeOnce(t *testing.T) {
+	into := t.TempDir()
+	if err := busCertificate([]string{"--into", into}); err != nil {
+		t.Fatal(err)
+	}
+	first, err := os.ReadFile(filepath.Join(into, "tls.crt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := busCertificate([]string{"--into", into}); err != nil {
+		t.Fatal(err)
+	}
+	again, err := os.ReadFile(filepath.Join(into, "tls.crt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(first) != string(again) {
+		t.Fatal("running it twice replaced the certificate every host had pinned")
+	}
+}
+
+// The verify half: false before, true after, which is what makes the bootstrap run the step at all.
+func TestTheCheckIsFalseUntilThereIsAPair(t *testing.T) {
+	into := t.TempDir()
+	if err := busCertificate([]string{"--check", "--into", into}); err == nil {
+		t.Fatal("an empty directory reported a usable certificate")
+	}
+	if err := busCertificate([]string{"--into", into}); err != nil {
+		t.Fatal(err)
+	}
+	if err := busCertificate([]string{"--check", "--into", into}); err != nil {
+		t.Fatalf("the certificate it just made does not satisfy its own check: %v", err)
+	}
+}
+
+// A half-written pair is not a pair. An interrupted bootstrap leaves exactly this, and a step that
+// called it done would hand the server a certificate with no key and report success.
+func TestACertificateWithoutItsKeyIsNotUsable(t *testing.T) {
+	into := t.TempDir()
+	if err := busCertificate([]string{"--into", into}); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Remove(filepath.Join(into, "tls.key")); err != nil {
+		t.Fatal(err)
+	}
+	if err := busCertificate([]string{"--check", "--into", into}); err == nil {
+		t.Fatal("a certificate with no key passed the check")
+	}
+	if err := busCertificate([]string{"--into", into}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := tls.LoadX509KeyPair(filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key")); err != nil {
+		t.Fatalf("it did not replace the unusable pair: %v", err)
+	}
+}
+
+func TestWhereToWriteIsRequired(t *testing.T) {
+	if err := busCertificate(nil); err == nil || !strings.Contains(err.Error(), "--into") {
+		t.Fatalf("it did not ask where to write: %v", err)
+	}
+}

cmd/mesh-controller/nodes.go

@@ -364,8 +364,11 @@ func identityCommand(ctx context.Context, args []string) error {
 }
 
 func brokerCommand(args []string) error {
+	if len(args) > 0 && args[0] == "certificate" {
+		return busCertificate(args[1:])
+	}
 	if len(args) == 0 || args[0] != "show" {
-		return errors.New("broker show")
+		return errors.New("broker show | broker certificate [--check] --into <directory>")
 	}
 	known, err := broker.FromEnvironment()
 	if errors.Is(err, broker.ErrNotConfigured) {

internal/catalogue/catalogue_check_test.go

@@ -0,0 +1,47 @@
+package catalogue
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// TestEveryCatalogueManifestParses runs the real catalogue through the real gate.
+//
+// Not a fixture: the point is whether the manifests as written are accepted by the control plane that
+// will read them, and a copy of one manifest proves nothing about the other seventy-one.
+func TestEveryCatalogueManifestParses(t *testing.T) {
+	root := os.Getenv("MESH_CATALOGUE")
+	if root == "" {
+		t.Skip("set MESH_CATALOGUE to a catalogue checkout to run this")
+	}
+	found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
+	if err != nil || len(found) == 0 {
+		t.Fatalf("no manifests under %s: %v", root, err)
+	}
+	named, routed := 0, 0
+	for _, p := range found {
+		raw, err := os.ReadFile(p)
+		if err != nil {
+			t.Fatalf("%s: %v", p, err)
+		}
+		m, err := ParseManifest(raw)
+		if err != nil {
+			t.Errorf("%s: %v", filepath.Base(filepath.Dir(p)), err)
+			continue
+		}
+		for _, l := range m.Listens {
+			if l.Name != "" {
+				named++
+			}
+		}
+		for port := range RoutedPorts(m) {
+			_ = port
+			routed++
+		}
+	}
+	t.Logf("%d manifests, %d named endpoints, %d routed endpoints resolved", len(found), named, routed)
+	if named == 0 {
+		t.Fatal("no endpoint in the catalogue is named, so this proved nothing")
+	}
+}

internal/catalogue/catrust_manifest_test.go

@@ -0,0 +1,71 @@
+package catalogue
+
+import (
+	"os"
+	"strings"
+	"testing"
+)
+
+// **A machine trusts the mesh's authority because a module put its root there** (novox/hq ADR
+// 0147, issue 129). The module carries a shell script and a unit, and both are worthless unless
+// the mesh fills in where the authority is — which is the one thing about it the manifest cannot
+// state, because the authority's address is a fact about the mesh and not about the module.
+//
+// So what is checked here is the rendering, not the parsing: the script the machine will run
+// names the authority it was bound to, and the unit runs that script both ways. The verification
+// itself — a plain client trusting an internal name on a machine holding this, and failing on one
+// that does not — is the lab's, and cannot be had here.
+func TestCaTrustRendersTheAuthorityItWasBoundTo(t *testing.T) {
+	raw, err := os.ReadFile("../../../mesh-catalog/modules/ca-trust/module.json")
+	if err != nil {
+		t.Skipf("the catalogue is not beside this checkout: %v", err)
+	}
+	m, err := ParseManifest(raw)
+	if err != nil {
+		t.Fatalf("the trust module does not parse:\n%v", err)
+	}
+
+	r := Resolution{
+		Node:    "workstation",
+		Modules: []Manifest{m},
+		Needs: []Needed{{
+			Name: "internal-acme-ca", From: "anchor", At: "anchor.internal", For: "ca-trust",
+			Serves: map[string]any{
+				"port": float64(9000), "path": "/acme/acme/directory", "roots": "/roots.pem",
+			},
+		}},
+	}
+	out, err := r.Declaration(Rendering{})
+	if err != nil {
+		t.Fatalf("the trust module could not be composed for a machine: %v", err)
+	}
+
+	script := fileNamed(out, "ca-trust.anchor")
+	if script == nil {
+		t.Fatalf("nothing writes the script the unit runs: %v", out)
+	}
+	body, _ := script["content"].(string)
+	if !strings.Contains(body, "https://anchor.internal:9000/roots.pem") {
+		t.Errorf("the script does not fetch from the authority it was bound to:\n%s", body)
+	}
+	if script["mode"] != "0755" {
+		t.Errorf("the script is written %v, which systemd cannot execute", script["mode"])
+	}
+
+	unit := fileNamed(out, "ca-trust.unit")
+	if unit == nil {
+		t.Fatalf("no unit: %v", out)
+	}
+	text, _ := unit["content"].(string)
+	// Both halves. A unit that only installs the anchor leaves a machine trusting an authority
+	// nobody assigned it to any more, which is the half issue 129 asked for by name.
+	for _, want := range []string{
+		"ExecStart=" + script["path"].(string) + " install",
+		"ExecStop=" + script["path"].(string) + " remove",
+		"RemainAfterExit=yes",
+	} {
+		if !strings.Contains(text, want) {
+			t.Errorf("the unit does not say %q:\n%s", want, text)
+		}
+	}
+}

internal/catalogue/declaration.go

@@ -1098,7 +1098,12 @@ func (r Resolution) contributions(settings SettingsBy, grants []Grant,
 			if err != nil {
 				return nil, fmt.Errorf("%s contributing to %s: %w", m.Module, to, err)
 			}
-			composeName(values, r.PublicDomain, r.At, reaches, endpointPorts(m))
+			blocks, err := Endpoints(m, settings[m.Module])
+			if err != nil {
+				return nil, fmt.Errorf("%s contributing to %s: %w", m.Module, to, err)
+			}
+			portOfEndpoint(values, endpointPorts(m))
+			composeName(values, r.PublicDomain, r.At, reaches, endpointPorts(m), blocks)
 			out[to] = append(out[to], Contribution{From: m.Module, Values: values})
 		}
 		// Several contributions to one requirement (ADR 0094's sibling for `contributes`): an
@@ -1116,7 +1121,12 @@ func (r Resolution) contributions(settings SettingsBy, grants []Grant,
 				if err != nil {
 					return nil, fmt.Errorf("%s contributing %s to %s: %w", m.Module, local, to, err)
 				}
-				composeName(values, r.PublicDomain, r.At, reaches, endpointPorts(m))
+				blocks, err := Endpoints(m, settings[m.Module])
+				if err != nil {
+					return nil, fmt.Errorf("%s contributing %s to %s: %w", m.Module, local, to, err)
+				}
+				portOfEndpoint(values, endpointPorts(m))
+				composeName(values, r.PublicDomain, r.At, reaches, endpointPorts(m), blocks)
 				out[to] = append(out[to], Contribution{From: m.Module, Values: values})
 			}
 		}
@@ -1148,10 +1158,19 @@ func (r Resolution) contributions(settings SettingsBy, grants []Grant,
 // no public domain composes nothing — there is nothing to join it to — which reads downstream as a
 // route that named no host, the same as it would have before this existed.
 func composeName(values map[string]any, publicDomain, internalDomain string, reaches map[int]string,
-	ports map[string]int) {
+	ports map[string]int, blocks map[string]Endpoint) {
 	if values == nil {
 		return
 	}
+	// **The subdomain an assignment gave this endpoint**, before the name is joined (novox/hq ADR
+	// 0138). The module contributes a label because it names its own parts; an assignment may say a
+	// different one, because where a thing lives under a domain is the operator's to choose and used
+	// to require editing the module to change.
+	if name, ok := values[RouteEndpoint].(string); ok {
+		if ep, said := blocks[strings.TrimSpace(name)]; said && ep.Label != "" {
+			values["label"] = ep.Label
+		}
+	}
 	// **How far the endpoint this route serves reaches decides which names exist** (novox/hq ADR
 	// 0138). Both were composed whenever the node had both domains, so every routed module got a
 	// public name and an internal one whether anybody wanted them or not — and a certificate for
@@ -1820,3 +1839,32 @@ func endpointPorts(m Manifest) map[string]int {
 	}
 	return out
 }
+
+// portOfEndpoint fills in the port of the endpoint a contribution names, in place.
+//
+// **A contribution that names an endpoint must still carry that endpoint's port**, because everything
+// downstream reads the port: the provider is told where to reach the consumer, and the machine-side
+// redirection that turns a declared port into the number the machine published is keyed on it
+// (atMachinePort). A route that named only its endpoint left the proxy with no port at all, and a
+// proxy with no port has nothing to dial.
+//
+// Found before it shipped and after the catalogue had already been changed to name endpoints — the
+// manifests were merged and the mesh had not yet picked them up, so nothing was broken yet. The
+// declared port, not the machine one: the redirection happens later and is keyed on the declared
+// number, so filling in the machine port here would be redirected a second time or not at all.
+func portOfEndpoint(values map[string]any, ports map[string]int) {
+	if values == nil {
+		return
+	}
+	if _, already := values["port"]; already {
+		// A route that says both is its own answer; the older shape repeated the port and is still read.
+		return
+	}
+	name, ok := values[RouteEndpoint].(string)
+	if !ok {
+		return
+	}
+	if port, found := ports[strings.TrimSpace(name)]; found {
+		values["port"] = port
+	}
+}

internal/catalogue/endpoint_test.go

@@ -19,6 +19,12 @@ func aMediaServer() Manifest {
 		Contributes: map[string]map[string]any{
 			"route": {"label": "media", RouteEndpoint: "web"},
 		},
+		// Both endpoints are published by its container, which is what lets a machine port be given
+		// for either: the mesh moves a port the module publishes, never one it merely listens on.
+		Resources: []map[string]any{
+			{"id": "server", "type": "container", "name": "media",
+				"ports": []any{"80", "32400"}},
+		},
 	}
 }
 
@@ -139,3 +145,61 @@ func TestAnUnnamedEndpointIsStillValid(t *testing.T) {
 		t.Fatalf("a module with no route was refused: %v", got)
 	}
 }
+
+// **A route that names an endpoint still carries that endpoint's port.**
+//
+// Everything downstream reads the port: the provider is told where to reach the consumer, and the
+// redirection that turns a declared port into the number the machine published is keyed on it. A route
+// naming only its endpoint left the proxy with no port, and a proxy with no port has nothing to dial.
+//
+// Caught after the catalogue had already been changed to name endpoints, and before the mesh picked
+// those manifests up — which is the only reason nothing broke.
+func TestARouteNamingAnEndpointStillCarriesItsPort(t *testing.T) {
+	m := aMediaServer()
+	r := Resolution{Node: "anchor", Modules: []Manifest{m},
+		PublicDomain: "example.test", At: "anchor.internal"}
+	given, err := r.contributions(nil, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var saw bool
+	for _, c := range given["route"] {
+		saw = true
+		port, ok := asPort(c.Values["port"])
+		if !ok {
+			t.Fatalf("the route carries no port, so the proxy has nothing to dial: %v", c.Values)
+		}
+		if port != 80 {
+			t.Fatalf("the route carries port %d, want the web endpoint's 80", port)
+		}
+	}
+	if !saw {
+		t.Fatal("the module contributed no route")
+	}
+}
+
+// And the declared port, not the machine one: the redirection to where the machine published it
+// happens later and is keyed on the declared number, so filling the machine port in here would be
+// redirected twice or not at all.
+func TestTheEndpointsDeclaredPortIsFilledInNotTheMachineOne(t *testing.T) {
+	m := aMediaServer()
+	values := map[string]any{RouteEndpoint: "web", "label": "media"}
+	portOfEndpoint(values, endpointPorts(m))
+	if got, _ := asPort(values["port"]); got != 80 {
+		t.Fatalf("filled in port %d, want the declared 80", got)
+	}
+	// Then the ordinary redirection puts it where the machine published it.
+	moved := atMachinePort(values, m.Module, map[string]map[int]int{"media": {80: 20009}})
+	if got, _ := asPort(moved["port"]); got != 20009 {
+		t.Fatalf("after redirection the port is %d, want the machine's 20009", got)
+	}
+}
+
+// A route that repeats a port keeps it, because that is the older shape and still read.
+func TestARouteThatRepeatsItsPortKeepsIt(t *testing.T) {
+	values := map[string]any{RouteEndpoint: "web", "port": 8080}
+	portOfEndpoint(values, map[string]int{"web": 80})
+	if got, _ := asPort(values["port"]); got != 8080 {
+		t.Fatalf("the port it stated was overwritten with %d", got)
+	}
+}

internal/catalogue/endpoints_setting_test.go

@@ -0,0 +1,140 @@
+package catalogue
+
+import (
+	"strings"
+	"testing"
+)
+
+func configured(block map[string]any) SettingsBy {
+	return SettingsBy{"media": {{From: "node anchor",
+		Values: map[string]any{EndpointsSetting: block}}}}
+}
+
+// **One block per endpoint, saying all three things.** The machine port, the subdomain and the reach
+// were `ports`, the route's label and `reach`, each keyed by a port number, so configuring a module
+// with two endpoints of different shapes meant knowing which number was which.
+func TestAnEndpointsBlockSaysPortLabelAndReach(t *testing.T) {
+	m := aMediaServer()
+	// stream's reach NARROWS what the manifest says — the manifest has it from anywhere, the
+	// assignment says internal. Chosen deliberately: a reach that agrees with the manifest proves
+	// nothing about whether the block was read at all.
+	settings := configured(map[string]any{
+		"web":    map[string]any{"port": 20009, "label": "cinema", "reach": ReachBoth},
+		"stream": map[string]any{"reach": ReachInternal},
+	})
+
+	// The machine port, where the mapping is read.
+	given, err := GivenPorts(m, settings["media"])
+	if err != nil {
+		t.Fatal(err)
+	}
+	if given[80] != 20009 {
+		t.Fatalf("the web endpoint is on machine port %d, want 20009: %v", given[80], given)
+	}
+
+	// The reach, where the filter reads it.
+	r := Resolution{Node: "anchor", Modules: []Manifest{m},
+		PublicDomain: "example.test", At: "anchor.internal"}
+	rules, err := r.Rules(Rendering{Settings: settings})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, rule := range rules {
+		if rule.Port == 32400 && rule.From != FromMesh {
+			t.Fatalf("the directly-dialled endpoint is %q; the assignment narrowed it to the private "+
+				"network and the manifest's 'anywhere' should not win", rule.From)
+		}
+		if rule.Port == 80 && rule.From != FromMesh {
+			t.Fatalf("the routed endpoint's port opened to %q; the proxy is how it is reached", rule.From)
+		}
+	}
+
+	// And the subdomain, where the name is composed — the assignment's, not the module's.
+	nodes, err := r.contributions(settings, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, c := range nodes["route"] {
+		if got, _ := c.Values["name"].(string); got != "cinema.example.test" {
+			t.Fatalf("the public name is %q, want the label the assignment gave", got)
+		}
+		if got, _ := c.Values["internal-name"].(string); got != "cinema.anchor.internal" {
+			t.Fatalf("the internal name is %q, want the label the assignment gave", got)
+		}
+	}
+}
+
+// A block that says only a reach leaves the port to the mesh and the label to the module, which is the
+// ordinary case and must not require writing the other two.
+func TestABlockMaySayOnlyAReach(t *testing.T) {
+	m := aMediaServer()
+	settings := configured(map[string]any{"web": map[string]any{"reach": ReachInternal}})
+	r := Resolution{Node: "anchor", Modules: []Manifest{m},
+		PublicDomain: "example.test", At: "anchor.internal"}
+	nodes, err := r.contributions(settings, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, c := range nodes["route"] {
+		if got, _ := c.Values["name"].(string); got != "" {
+			t.Fatalf("an internal endpoint composed the public name %q", got)
+		}
+		// The module's own label, untouched.
+		if got, _ := c.Values["internal-name"].(string); got != "media.anchor.internal" {
+			t.Fatalf("the internal name is %q, want the module's own label", got)
+		}
+	}
+}
+
+// An endpoint the module does not declare reaches nothing, and the refusal says what it does declare.
+func TestConfiguringAnEndpointTheModuleLacksIsRefused(t *testing.T) {
+	_, err := Endpoints(aMediaServer(), configured(map[string]any{
+		"admin": map[string]any{"reach": ReachInternal}})["media"])
+	if err == nil || !strings.Contains(err.Error(), "does not declare") {
+		t.Fatalf("configuring an absent endpoint was accepted: %v", err)
+	}
+	if err != nil && !strings.Contains(err.Error(), "stream") {
+		t.Fatalf("the refusal does not name what the module declares: %v", err)
+	}
+}
+
+func TestAReachInABlockIsHeldToTheFourValues(t *testing.T) {
+	_, err := Endpoints(aMediaServer(), configured(map[string]any{
+		"web": map[string]any{"reach": "mesh"}})["media"])
+	if err == nil || !strings.Contains(err.Error(), "a reach is") {
+		t.Fatalf("a filter word was accepted as a reach: %v", err)
+	}
+}
+
+// **Two places giving one endpoint a machine port is the confusion this key exists to end.**
+func TestAnEndpointGivenAPortTwiceIsRefused(t *testing.T) {
+	m := aMediaServer()
+	_, err := GivenPorts(m, []Layer{{From: "node anchor", Values: map[string]any{
+		EndpointsSetting: map[string]any{"web": map[string]any{"port": 20009}},
+		PortsSetting:     map[string]any{"80": 30000},
+	}}})
+	if err == nil || !strings.Contains(err.Error(), "published once") {
+		t.Fatalf("an endpoint given two machine ports was accepted: %v", err)
+	}
+}
+
+// And the same for its reach, said once here and once through the older key.
+func TestAnEndpointWhoseReachIsAlsoExposedIsRefused(t *testing.T) {
+	_, err := Endpoints(aMediaServer(), []Layer{{From: "node anchor", Values: map[string]any{
+		EndpointsSetting: map[string]any{"web": map[string]any{"reach": ReachInternal}},
+		ExposeSetting:    map[string]any{"80": FromEverywhere},
+	}}})
+	if err == nil || !strings.Contains(err.Error(), "same thing in different words") {
+		t.Fatalf("a reach said two ways was accepted: %v", err)
+	}
+}
+
+// A module whose endpoints are unnamed cannot be configured this way, and is told so rather than
+// having a block silently reach nothing — which is every module in the catalogue today.
+func TestAModuleWithNoNamedEndpointsIsToldSo(t *testing.T) {
+	m := Manifest{Module: "media", Listens: []Listening{{Port: 80, From: FromMesh}}}
+	_, err := Endpoints(m, configured(map[string]any{"web": map[string]any{"reach": ReachBoth}})["media"])
+	if err == nil || !strings.Contains(err.Error(), "no endpoints by name") {
+		t.Fatalf("a module with no named endpoints accepted a block: %v", err)
+	}
+}

internal/catalogue/filtering.go

@@ -265,6 +265,26 @@ func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int,
 	b.WriteString("\t\tct state established,related accept\n")
 	b.WriteString("\t\tct state invalid drop\n")
 	b.WriteString("\t\tiif lo accept\n")
+	// **Anything on this machine may call anything on this machine.**
+	//
+	// Local is not a boundary this mesh draws. A service running here is callable by everything else
+	// running here, whatever form either takes — a package with a unit, a binary, a container. Whether
+	// a caller sits in a container was never meant to change the answer, and the only reason it did was
+	// that this chain asked about addresses: a caller on the machine carries the machine's address, a
+	// caller in one of its containers carries a bridge address, and a rule naming the former silently
+	// refused the latter.
+	//
+	// Measured: a module reaching its database on this machine's own name timed out for eleven hours
+	// while the machine itself could reach it, and the mesh called the machine healthy throughout
+	// (novox/hq 04-ISSUES/145).
+	//
+	// Asked by the link it arrives on rather than the address it comes from: anything that did not
+	// arrive from outside this machine, and did not arrive over the private network, is this machine's
+	// own. One rule for every service here, in place of a line per port that only ever covered the
+	// ports somebody remembered to think about.
+	if inward != "" {
+		b.WriteString(fmt.Sprintf("\t\tiifname != { %s } accept\n", inward))
+	}
 	b.WriteString("\t\ticmp type echo-request accept\n")
 	b.WriteString("\t\ticmpv6 type { echo-request, nd-neighbor-solicit, nd-neighbor-advert, nd-router-advert } accept\n")
 
@@ -536,6 +556,21 @@ const MeshWideLayer = "the mesh"
 // two mappings share a number, it is an entry one of them writes over the other's, and the reader
 // that finds the survivor disagrees with the reader that recomputes it.
 func GivenPorts(m Manifest, layers []Layer) (map[int]int, error) {
+	// An endpoint's own block may put it on a machine port, which is the same thing `ports` says about
+	// the number rather than about the endpoint (novox/hq ADR 0138). Collected first and then let the
+	// older key be read, which refuses a port said twice.
+	byName, err := Endpoints(m, layers)
+	if err != nil {
+		return nil, err
+	}
+	named := map[int]int{}
+	for name, ep := range byName {
+		if ep.Port == 0 {
+			continue
+		}
+		named[endpointPorts(m)[name]] = ep.Port
+	}
+
 	// Every name a setting may use, and the mapping it names.
 	names := map[int][]publishing{}
 	for _, p := range publishedPorts(m) {
@@ -634,6 +669,17 @@ func GivenPorts(m Manifest, layers []Layer) (map[int]int, error) {
 			out[key], by[key] = at, port
 		}
 	}
+	// And what the endpoints' own blocks put them on. Refused rather than merged where both keys name
+	// one endpoint: two places giving a port is the confusion this key exists to end.
+	for wanted, at := range named {
+		if was, twice := out[wanted]; twice && was != at {
+			return nil, fmt.Errorf(
+				"%s puts its port %d on %d through %s and on %d through %s — one endpoint, two "+
+					"machine ports, and it is published once. Keep the endpoint's own block",
+				m.Module, wanted, at, EndpointsSetting, was, PortsSetting)
+		}
+		out[wanted] = at
+	}
 	if len(out) == 0 {
 		return nil, nil
 	}
@@ -821,6 +867,20 @@ func Reaches(m Manifest, layers []Layer) (map[int]string, error) {
 	}
 
 	out := map[int]string{}
+	// What an endpoint's own block says, which is the same statement in the shape that names the
+	// endpoint rather than its port (novox/hq ADR 0138). Read first so the older key, which says less,
+	// cannot quietly win over the newer one that says more.
+	blocks, err := Endpoints(m, layers)
+	if err != nil {
+		return nil, err
+	}
+	declared := endpointPorts(m)
+	for name, ep := range blocks {
+		if ep.Reach == "" {
+			continue
+		}
+		out[declared[name]] = ep.Reach
+	}
 	for _, layer := range layers {
 		raw, ok := layer.Values[ReachSetting]
 		if !ok {
@@ -896,3 +956,123 @@ func RouteProblems(m Manifest) []string {
 	}
 	return problems
 }
+
+// EndpointsSetting is the settings key that configures a module's endpoints by name, per node
+// (novox/hq ADR 0138):
+//
+//	{"endpoints": {"web":    {"port": 20009, "label": "media", "reach": "both"},
+//	               "stream": {"reach": "public"}}}
+//
+// **One block per endpoint, instead of three keys joined by a number.** Which machine port it lands
+// on, the subdomain a proxy serves it under, and how far it reaches are the three things an operator
+// says when a module is assigned, and they were said in `ports`, in the route's label and in `reach`,
+// each keyed by the port. A module with two endpoints of different shapes — a web surface behind the
+// proxy and a protocol port clients dial directly — could only be configured by a reader who knew
+// which number was which.
+//
+// Every field is optional. A block that says only a reach leaves the port to the mesh and the label to
+// the module, which is the ordinary case.
+const EndpointsSetting = "endpoints"
+
+// Endpoint is what an assignment says about one of a module's endpoints.
+type Endpoint struct {
+	// Port is the machine-side port it is published on. Zero means the mesh assigns one, which it
+	// does anyway — a fixed port is the module's claim and is honoured without being said here.
+	Port int
+	// Label is the subdomain a proxy serves it under, overriding the one the module contributes.
+	Label string
+	// Reach is how far it reaches: machine, internal, public or both.
+	Reach string
+}
+
+// Endpoints reads a module's per-node endpoint configuration, by endpoint name.
+//
+// It refuses a name the module does not declare — the setting would reach nothing — and a reach that
+// is not one of the four. It also refuses an endpoint whose port or reach is said twice, once here and
+// once through the older key: two places saying the same thing is what this key exists to end, and
+// letting both stand would mean the mesh followed whichever it read last.
+func Endpoints(m Manifest, layers []Layer) (map[string]Endpoint, error) {
+	declared := endpointPorts(m)
+	exposed, err := Exposure(m, layers)
+	if err != nil {
+		return nil, err
+	}
+
+	out := map[string]Endpoint{}
+	for _, layer := range layers {
+		raw, ok := layer.Values[EndpointsSetting]
+		if !ok {
+			continue
+		}
+		blocks, ok := raw.(map[string]any)
+		if !ok {
+			return nil, fmt.Errorf("%s: %s is a { endpoint: { … } } map, and %q set it to something else",
+				m.Module, EndpointsSetting, layer.From)
+		}
+		for name, body := range blocks {
+			port, known := declared[name]
+			if !known {
+				return nil, fmt.Errorf(
+					"%s configures the endpoint %q, which it does not declare — the setting reaches "+
+						"nothing. It declares %s", m.Module, name, spokenEndpoints(m))
+			}
+			values, ok := body.(map[string]any)
+			if !ok {
+				return nil, fmt.Errorf("%s: the endpoint %q is configured with something that is not a "+
+					"block of settings", m.Module, name)
+			}
+			ep := out[name]
+			if reach, said := values["reach"]; said {
+				text, ok := reach.(string)
+				if !ok || !slices.Contains(reaches, text) {
+					return nil, fmt.Errorf("%s says the endpoint %q reaches %v; a reach is %s",
+						m.Module, name, reach, strings.Join(reaches, ", "))
+				}
+				if _, also := exposed[port]; also {
+					return nil, fmt.Errorf(
+						"%s says how far %q reaches and also exposes port %d. They say the same thing "+
+							"in different words; keep the endpoint's own block",
+						m.Module, name, port)
+				}
+				ep.Reach = text
+			}
+			if at, said := values["port"]; said {
+				machine, ok := asPort(at)
+				if !ok {
+					return nil, fmt.Errorf("%s puts the endpoint %q on %v, which is not a port",
+						m.Module, name, at)
+				}
+				ep.Port = machine
+			}
+			if label, said := values["label"]; said {
+				text, ok := label.(string)
+				if !ok || strings.TrimSpace(text) == "" {
+					return nil, fmt.Errorf("%s gives the endpoint %q a label that is not a name: %v",
+						m.Module, name, label)
+				}
+				ep.Label = strings.TrimSpace(text)
+			}
+			out[name] = ep
+		}
+	}
+	if len(out) == 0 {
+		return nil, nil
+	}
+	return out, nil
+}
+
+// spokenEndpoints is what a module's endpoints are called, as a refusal lists them — so a reader who
+// named one wrongly is one edit from right, and a module that has named none is told so.
+func spokenEndpoints(m Manifest) string {
+	names := make([]string, 0, len(m.Listens))
+	for _, l := range m.Listens {
+		if name := strings.TrimSpace(l.Name); name != "" {
+			names = append(names, name)
+		}
+	}
+	if len(names) == 0 {
+		return "no endpoints by name"
+	}
+	sort.Strings(names)
+	return strings.Join(names, ", ")
+}

internal/catalogue/filtering_test.go

@@ -804,3 +804,86 @@ func TestSSHIsNeverLeftWithoutARule(t *testing.T) {
 		t.Fatalf("a machine with no mesh addresses has no ssh rule, so adopting it locks it:\n%s", nft)
 	}
 }
+
+// chainBody is one chain's own lines, so an assertion cannot be satisfied by an identical line in
+// another chain.
+//
+// **Written because that happened.** The rule letting this machine's own callers through appears in the
+// input chain and, in the same words, in the forward chain. A test asserting on the whole rendered file
+// passed with the input chain's copy deleted — it was reading the forward chain's. ADR 0137's own tests
+// say to assert per chain body for exactly this reason, and this file was not doing it.
+func chainBody(t *testing.T, nft, chain string) string {
+	t.Helper()
+	open := "\tchain " + chain + " {"
+	i := strings.Index(nft, open)
+	if i < 0 {
+		t.Fatalf("no chain %q in:\n%s", chain, nft)
+	}
+	rest := nft[i+len(open):]
+	j := strings.Index(rest, "\n\t}")
+	if j < 0 {
+		t.Fatalf("chain %q does not close in:\n%s", chain, nft)
+	}
+	return rest[:j]
+}
+
+// **Anything on this machine may call anything on this machine.**
+//
+// Local is not a boundary this mesh draws, and whether a caller sits in a container was never meant to
+// change the answer. It did, because the chain asked about addresses: a caller on the machine carries
+// the machine's address and a caller in one of its containers carries a bridge address, so a rule
+// naming the machines' own addresses silently refused every container on them.
+//
+// Measured: a module reaching its database on its own machine's name timed out for eleven hours while
+// the machine itself could reach it (novox/hq 04-ISSUES/145).
+func TestAnythingOnThisMachineMayCallAnythingOnIt(t *testing.T) {
+	nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
+		{Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}},
+		{Module: "private", Listens: []Listening{{Port: 9999, From: FromMachine}}},
+	}}, nil), []string{"10.10.0.1", "10.10.0.2"}, false, nil, []string{"eth0"}, "mesh0")
+
+	// In the INPUT chain, which is where a call to a service on this machine arrives. The forward
+	// chain carries the same line in the same words, so asserting on the whole file proves nothing.
+	input := chainBody(t, nft, "input")
+	if !strings.Contains(input, `iifname != { "eth0", "mesh0" } accept`) {
+		t.Fatalf("a caller on this machine cannot reach a service on it:\n%s", input)
+	}
+	// One rule, for every service here — not a line per port that only covers the ports somebody
+	// remembered to think about.
+	if strings.Contains(input, `iifname != { "eth0", "mesh0" } tcp dport 5432`) {
+		t.Fatalf("the local allowance is still written per port:\n%s", input)
+	}
+	// And the private network still reaches what is exposed to it, which is a different question.
+	if !strings.Contains(input, "ip saddr { 10.10.0.1, 10.10.0.2 } tcp dport 5432 accept") {
+		t.Fatalf("the private network no longer reaches a service exposed to it:\n%s", input)
+	}
+}
+
+// The three reaches, as three lines. This is the whole of what the filter says about who may call what.
+func TestTheThreeReachesAreThreeLines(t *testing.T) {
+	nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
+		{Module: "internal-only", Listens: []Listening{{Port: 5432, From: FromMesh}}},
+		{Module: "public", Listens: []Listening{{Port: 443, From: FromEverywhere}}},
+	}}, nil), []string{"10.10.0.1"}, false, nil, []string{"eth0"}, "mesh0")
+
+	input := chainBody(t, nft, "input")
+	for what, want := range map[string]string{
+		"on this machine":          `iifname != { "eth0", "mesh0" } accept`,
+		"over the private network": "ip saddr { 10.10.0.1 } tcp dport 5432 accept",
+		"from anywhere":            "tcp dport 443 accept",
+	} {
+		if !strings.Contains(input, want) {
+			t.Fatalf("a caller %s cannot reach what is exposed to it (%q):\n%s", what, want, input)
+		}
+	}
+}
+
+// A port open to everything needs no such line — it is already open to a guest.
+func TestAPublicPortNeedsNoGuestLine(t *testing.T) {
+	nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
+		{Module: "web", Listens: []Listening{{Port: 443, From: FromEverywhere}}},
+	}}, nil), []string{"10.10.0.1"}, false, nil, []string{"eth0"}, "mesh0")
+	if strings.Count(nft, `iifname != { "eth0", "mesh0" } tcp dport 443`) != 0 {
+		t.Fatalf("a public port was given a guest line it does not need:\n%s", nft)
+	}
+}

internal/catalogue/settings.go

@@ -192,6 +192,11 @@ func UnusedSettings(m Manifest, layers []Layer) []string {
 			if key == ReachSetting && len(m.Listens) > 0 {
 				continue
 			}
+			// `endpoints` configures a module's endpoints by name — the machine port, the subdomain and
+			// the reach as one block each (novox/hq ADR 0138). Validated in Endpoints, so not stray.
+			if key == EndpointsSetting && len(m.Listens) > 0 {
+				continue
+			}
 			unused = append(unused, fmt.Sprintf(
 				"%s sets %q, and %s has no file or contribution to merge it into",
 				layer.From, key, m.Module))