A route's internal name says where the request arrives

novox/hq ADR 0151 (issues 139, 157). <label>.<node>.internal is answered
by every resolver as 'anything under that node goes to that node', so
the node in a route's internal name must be the one whose proxy answers
it; composed under the consumer's own name it sent a client to a machine
with nothing listening whenever the proxy ran elsewhere. Composed under
the serving node now — the same machine wherever the proxy runs beside
the module, so nothing changes on a mesh with one hub.

A routed public name gets no .internal alias any more: the roster
publishes it as itself, once. The alias resolved and nothing served it.

Makefile

@@ -27,8 +27,18 @@ build:
 IMAGE ?= mesh-controller:$(VERSION)
 DEV_TAG ?= mesh-controller:development
 
+# The base the module declares, read from the manifest rather than written here twice.
+#
+# **`make image` was broken and stayed broken**, because the Dockerfile's fallback base was a Go
+# older than go.mod asks for: every build died at `go mod download` with "go.mod requires go >=
+# 1.26.0", and the pipeline never saw it because the pipeline passes the declared base in. Anybody
+# building the image by hand hit it and had to find the digest themselves (novox/hq 04-ISSUES/146,
+# what it cost).
+GO_BASE ?= $(shell python3 -c "import json;print(next(o['image'] for o in json.load(open('module.json'))['build']['on'] if o['arg']=='GO_BASE'))" 2>/dev/null)
+
 image:
-	docker build --build-arg VERSION=$(VERSION) -t $(IMAGE) -t $(DEV_TAG) .
+	@test -n "$(GO_BASE)" || { echo "module.json declares no GO_BASE; pass GO_BASE=<image> or fix the manifest"; exit 1; }
+	docker build --build-arg GO_BASE=$(GO_BASE) --build-arg VERSION=$(VERSION) -t $(IMAGE) -t $(DEV_TAG) .
 	@echo
 	@docker image inspect $(IMAGE) --format 'built {{.RepoTags}}  {{.Size}} bytes'
 
@@ -38,7 +48,8 @@ BUILDER_IMAGE ?= mesh-builder:$(VERSION)
 BUILDER_DEV_TAG ?= mesh-builder:development
 
 builder-image:
-	docker build -f cmd/mesh-builder/Dockerfile -t $(BUILDER_IMAGE) -t $(BUILDER_DEV_TAG) .
+	@test -n "$(GO_BASE)" || { echo "module.json declares no GO_BASE; pass GO_BASE=<image> or fix the manifest"; exit 1; }
+	docker build --build-arg GO_BASE=$(GO_BASE) -f cmd/mesh-builder/Dockerfile -t $(BUILDER_IMAGE) -t $(BUILDER_DEV_TAG) .
 	@echo
 	@docker image inspect $(BUILDER_IMAGE) --format 'built {{.RepoTags}}  {{.Size}} bytes'
 

cmd/mesh-controller/acts.go

@@ -46,6 +46,13 @@ func assign(ctx context.Context, open *stores, node, module string) (string, err
 		return "", err
 	}
 	defer release()
+	// **Before the new assignment can unsettle a seat somebody holds only by being alone**
+	// (novox/hq 04-ISSUES/170): what the mesh derived so far is written down, and then the
+	// assignment resolves against a record rather than against a coincidence.
+	settled, err := recordDerivedHolders(ctx, open)
+	if err != nil {
+		return "", err
+	}
 	fresh, err := open.inventory.Assign(ctx, node, module)
 	if err != nil {
 		return "", err
@@ -57,6 +64,9 @@ func assign(ctx context.Context, open *stores, node, module string) (string, err
 			node, module), nil
 	}
 	said := fmt.Sprintf("%s is assigned %s", node, module)
+	for _, line := range settled {
+		said += "\n  " + line
+	}
 	plan, _, err := planFor(ctx, open, node)
 	if err != nil {
 		// Kept, and still refused. Both halves are the answer, and the rest of the mesh is still

cmd/mesh-controller/build.go

@@ -556,6 +556,16 @@ type answers struct {
 	// a consequence of the refusals above: a node that does not resolve is not on the network, and
 	// a mesh whose hub is that node has no hub.
 	network string
+	// untaken is, per machine, each assigned module whose resources the machine is holding as it
+	// found them, and how many — a module that was assigned, sent, and is running none of what it
+	// declares because nothing has taken it (novox/hq ADR 0100, 04-ISSUES/125).
+	//
+	// **Its absence cost an outage.** The module was assigned, the push reported success, this
+	// command said the machine was doing everything it was told, and the module's three containers
+	// did not exist. On the strength of those reports the predecessor's proxy was stopped and every
+	// public name on the machine went dark. The holds were correct; they were recorded only in the
+	// machine's own state file, and the one visible symptom was a count that did not add up.
+	untaken map[string]map[string]int
 }
 
 // heldBy is every artifact this mesh has built, for a build that may need one as its base.

cmd/mesh-controller/buscertificate.go

@@ -0,0 +1,258 @@
+package main
+
+import (
+	"context"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"math/big"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/broker"
+)
+
+// The bus's own certificate, made by the mesh rather than borrowed from an image.
+//
+// **The foundation asked a third-party image for a tool it never said must be there** (novox/hq
+// 04-ISSUES/146). The bootstrap made this certificate by running `openssl` inside the broker's
+// image, which worked while the broker was one that happened to carry it and stopped the day the
+// bus changed: the new one has a shell and no openssl, so the step exited 127 and no mesh could be
+// raised. Substituting another image the bundle names does not help — none of them carry it
+// either.
+//
+// So the program that needs a certificate makes one. It is the mesh's own binary, already on the
+// machine at this point in the bootstrap (the schema step ran it), and it needs nothing from the
+// image it writes into but a mounted directory.
+//
+// **Self-signed, and that is the design** — a host pins this server's exact certificate and
+// authenticates with a password (novox/hq ADR 0004). There is no authority above it to ask, and at
+// this moment in a bootstrap there is no mesh to ask one of.
+//
+// Idempotent, because the step is applied again on every reconcile and a second certificate would
+// be one the hosts that pinned the first no longer believe.
+
+// busCertificateNames is what the bus is reached by: the container name on a mesh network, and the
+// loopback address the machine's own foundation dials.
+var busCertificateNames = []string{"mesh-broker"}
+
+const busCertificateLife = 10 * 365 * 24 * time.Hour
+
+// busCertificate makes the bus's certificate in a directory, or says whether one is there.
+//
+//	broker certificate --into /tls           make it if it is not there
+//	broker certificate --check --into /tls   exit non-zero unless a usable pair is
+func busCertificate(args []string) error {
+	into, check := "", false
+	for i := 0; i < len(args); i++ {
+		switch args[i] {
+		case "--check":
+			check = true
+		case "--into":
+			if i+1 >= len(args) {
+				return errors.New("--into needs a directory")
+			}
+			into = args[i+1]
+			i++
+		default:
+			return fmt.Errorf("broker certificate [--check] --into <directory>: %q", args[i])
+		}
+	}
+	if into == "" {
+		return errors.New("broker certificate [--check] --into <directory>")
+	}
+	crt, key := filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key")
+
+	if usable, err := busCertificateUsable(crt, key); err != nil {
+		return err
+	} else if usable {
+		fmt.Printf("the bus already has a certificate at %s, and it was left alone\n", crt)
+		return nil
+	}
+	if check {
+		// Said as a failure, because that is what the caller asked: a bootstrap's verify runs
+		// this and a false answer is what makes the step run.
+		return fmt.Errorf("no usable certificate and key at %s", into)
+	}
+	return writeBusCertificate(crt, key)
+}
+
+// busCertificateUsable says whether a certificate and its key are both there and parse.
+//
+// Both, and parsed rather than stat'ed: a half-written pair is the state a bootstrap interrupted
+// between the two files leaves behind, and a step that treated it as done would hand the server a
+// certificate with no key and report success.
+func busCertificateUsable(crt, key string) (bool, error) {
+	certPEM, err := os.ReadFile(crt)
+	if errors.Is(err, os.ErrNotExist) {
+		return false, nil
+	}
+	if err != nil {
+		return false, err
+	}
+	keyPEM, err := os.ReadFile(key)
+	if errors.Is(err, os.ErrNotExist) {
+		return false, nil
+	}
+	if err != nil {
+		return false, err
+	}
+	if _, err := tlsPairParses(certPEM, keyPEM); err != nil {
+		return false, nil
+	}
+	return true, nil
+}
+
+func tlsPairParses(certPEM, keyPEM []byte) (*x509.Certificate, error) {
+	block, _ := pem.Decode(certPEM)
+	if block == nil || block.Type != "CERTIFICATE" {
+		return nil, errors.New("not a certificate")
+	}
+	certificate, err := x509.ParseCertificate(block.Bytes)
+	if err != nil {
+		return nil, err
+	}
+	keyBlock, _ := pem.Decode(keyPEM)
+	if keyBlock == nil {
+		return nil, errors.New("not a key")
+	}
+	if _, err := x509.ParsePKCS8PrivateKey(keyBlock.Bytes); err != nil {
+		if _, err := x509.ParsePKCS1PrivateKey(keyBlock.Bytes); err != nil {
+			return nil, err
+		}
+	}
+	return certificate, nil
+}
+
+func writeBusCertificate(crt, key string) error {
+	private, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		return err
+	}
+	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
+	if err != nil {
+		return err
+	}
+	template := &x509.Certificate{
+		SerialNumber:          serial,
+		Subject:               pkix.Name{CommonName: busCertificateNames[0]},
+		DNSNames:              busCertificateNames,
+		IPAddresses:           []net.IP{net.ParseIP("127.0.0.1")},
+		NotBefore:             time.Now().Add(-time.Hour),
+		NotAfter:              time.Now().Add(busCertificateLife),
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, template, template, &private.PublicKey, private)
+	if err != nil {
+		return err
+	}
+	pkcs8, err := x509.MarshalPKCS8PrivateKey(private)
+	if err != nil {
+		return err
+	}
+
+	// **The key first, and only then the certificate**, so the pair a reader finds is never a
+	// certificate whose key has not been written yet — the one order in which an interruption
+	// leaves something that looks finished (novox/hq 04-ISSUES/014, a key present and unusable).
+	if err := os.WriteFile(key, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8}), 0o600); err != nil {
+		return err
+	}
+	if err := os.WriteFile(crt, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644); err != nil {
+		return err
+	}
+	fmt.Printf("made the bus a certificate for %v, valid until %s\n  %s\n  %s\n",
+		busCertificateNames, template.NotAfter.Format(time.RFC3339), crt, key)
+	return nil
+}
+
+// busAccounts writes the mesh's composed user list to a file.
+//
+// **For genesis, where no declaration can deliver it** (novox/hq 04-ISSUES/146). Everywhere else
+// the list reaches the machine running the bus as a resource of the module that holds it — which
+// requires that machine to be an enrolled node, and at genesis it is not: the first node cannot
+// enrol because the account it would enrol with cannot be composed onto a bus it has no declaration
+// for. The installer breaks that circle by placing the file itself, once, and the module takes the
+// file over from its first push.
+//
+// The same composition, not a second one: this asks the store for the same records and renders them
+// with the same composer the declaration uses. A genesis that hand-wrote an account would be a
+// second statement of who may say what, able to disagree with the first.
+//
+// **It writes to standard output unless told a file**, and that is the point: the control plane
+// composes and says what it composed, and whoever is raising the machine puts it where that
+// machine's bus reads it. A control plane that wrote into the bus's own directory would have to
+// know where that is and how to make the server re-read it — which is the module's knowledge, and
+// the module is what takes this over on the first push.
+//
+//	broker accounts > /var/lib/mesh-bus-conf/accounts.conf
+func busAccounts(ctx context.Context, args []string) error {
+	into := ""
+	for i := 0; i < len(args); i++ {
+		switch args[i] {
+		case "--into":
+			if i+1 >= len(args) {
+				return errors.New("--into needs a file")
+			}
+			into = args[i+1]
+			i++
+		default:
+			return fmt.Errorf("broker accounts --into <file>: %q", args[i])
+		}
+	}
+
+	open, err := openStores(ctx)
+	if err != nil {
+		return err
+	}
+	defer open.Close()
+
+	records, err := open.inventory.BusRecords(ctx)
+	if err != nil {
+		return err
+	}
+	users, err := broker.Users(records)
+	if err != nil {
+		return err
+	}
+	kept, err := open.inventory.BusUsers(ctx)
+	if err != nil {
+		return err
+	}
+	hashes := make(map[string]string, len(kept))
+	for name, u := range kept {
+		hashes[name] = u.PasswordHash
+	}
+	filled, missing := broker.WithPasswords(users, hashes)
+	if len(missing) > 0 {
+		// To standard error, always: the composed file may be going to standard output, and a
+		// remark in the middle of it is a configuration the server refuses to parse.
+		fmt.Fprintf(os.Stderr, "leaving out %d user(s) the mesh has minted no credential for: %s\n",
+			len(missing), strings.Join(missing, ", "))
+	}
+	if len(filled) == 0 {
+		return errors.New("not one user has a credential, so this list would refuse every " +
+			"connection in the mesh")
+	}
+	accounts, err := broker.ComposeAccounts(filled)
+	if err != nil {
+		return err
+	}
+	if into == "" {
+		fmt.Print(accounts)
+		return nil
+	}
+	if err := os.WriteFile(into, []byte(accounts), 0o600); err != nil {
+		return err
+	}
+	fmt.Printf("wrote %d user(s) to %s\n", len(filled), into)
+	return nil
+}

cmd/mesh-controller/buscertificate_test.go

@@ -0,0 +1,121 @@
+package main
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// novox/hq 04-ISSUES/146. The bootstrap could not make the bus a certificate: it asked an image for
+// `openssl` and the image it asks has none. What replaces it is this command, so what is checked is
+// what the bootstrap needs from it — a pair a TLS server can actually load, made once and only once.
+
+func TestTheBusCertificateLoadsAsAServersWould(t *testing.T) {
+	into := t.TempDir()
+	if err := busCertificate([]string{"--into", into}); err != nil {
+		t.Fatalf("the bus could not be given a certificate: %v", err)
+	}
+
+	// The check a cheaper test would not make. The key was present and valid and the server could
+	// not start, once, because nothing loaded the pair the way a server loads it
+	// (novox/hq 04-ISSUES/014).
+	pair, err := tls.LoadX509KeyPair(filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key"))
+	if err != nil {
+		t.Fatalf("a TLS server cannot load what was written: %v", err)
+	}
+	leaf := pair.Leaf
+	if leaf == nil {
+		if leaf, err = x509.ParseCertificate(pair.Certificate[0]); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := leaf.VerifyHostname("mesh-broker"); err != nil {
+		t.Errorf("the certificate is not for the name the bus is reached by: %v", err)
+	}
+	if len(leaf.IPAddresses) == 0 || leaf.IPAddresses[0].String() != "127.0.0.1" {
+		t.Errorf("the certificate does not cover the loopback address the foundation dials: %v", leaf.IPAddresses)
+	}
+
+	// The key is not readable by anything else on the machine; the certificate is public and is.
+	key, err := os.Stat(filepath.Join(into, "tls.key"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if key.Mode().Perm() != 0o600 {
+		t.Errorf("the key is %v", key.Mode().Perm())
+	}
+	crt, err := os.Stat(filepath.Join(into, "tls.crt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if crt.Mode().Perm() != 0o644 {
+		t.Errorf("the certificate is %v, which the server runs as another user cannot read", crt.Mode().Perm())
+	}
+}
+
+// **Made once.** The step is applied again on every reconcile, and a second certificate is one the
+// hosts that pinned the first no longer believe (novox/hq ADR 0004).
+func TestTheBusCertificateIsMadeOnce(t *testing.T) {
+	into := t.TempDir()
+	if err := busCertificate([]string{"--into", into}); err != nil {
+		t.Fatal(err)
+	}
+	first, err := os.ReadFile(filepath.Join(into, "tls.crt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := busCertificate([]string{"--into", into}); err != nil {
+		t.Fatal(err)
+	}
+	again, err := os.ReadFile(filepath.Join(into, "tls.crt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(first) != string(again) {
+		t.Fatal("running it twice replaced the certificate every host had pinned")
+	}
+}
+
+// The verify half: false before, true after, which is what makes the bootstrap run the step at all.
+func TestTheCheckIsFalseUntilThereIsAPair(t *testing.T) {
+	into := t.TempDir()
+	if err := busCertificate([]string{"--check", "--into", into}); err == nil {
+		t.Fatal("an empty directory reported a usable certificate")
+	}
+	if err := busCertificate([]string{"--into", into}); err != nil {
+		t.Fatal(err)
+	}
+	if err := busCertificate([]string{"--check", "--into", into}); err != nil {
+		t.Fatalf("the certificate it just made does not satisfy its own check: %v", err)
+	}
+}
+
+// A half-written pair is not a pair. An interrupted bootstrap leaves exactly this, and a step that
+// called it done would hand the server a certificate with no key and report success.
+func TestACertificateWithoutItsKeyIsNotUsable(t *testing.T) {
+	into := t.TempDir()
+	if err := busCertificate([]string{"--into", into}); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Remove(filepath.Join(into, "tls.key")); err != nil {
+		t.Fatal(err)
+	}
+	if err := busCertificate([]string{"--check", "--into", into}); err == nil {
+		t.Fatal("a certificate with no key passed the check")
+	}
+	if err := busCertificate([]string{"--into", into}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := tls.LoadX509KeyPair(filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key")); err != nil {
+		t.Fatalf("it did not replace the unusable pair: %v", err)
+	}
+}
+
+func TestWhereToWriteIsRequired(t *testing.T) {
+	if err := busCertificate(nil); err == nil || !strings.Contains(err.Error(), "--into") {
+		t.Fatalf("it did not ask where to write: %v", err)
+	}
+}

cmd/mesh-controller/holdings.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"sort"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// recordDerivedHolders writes down who holds each mesh-scoped seat that nobody was ever recorded
+// as holding.
+//
+// **A seat held by derivation is a seat held by accident of being alone** (novox/hq
+// 04-ISSUES/170). ADR 0131 lets a holder on record settle a seat, and lets any other assignment
+// whose module could hold it stand beside the holder, eligible and silent. But a seat nobody
+// ever handed over has no record, so its holder is whichever assignment happened to be the sole
+// claimant — and the day a second one is assigned, both claim, both are refused, and the first
+// one's whole machine stops resolving. That is what assigning a second postgres did to the
+// control plane's own store.
+//
+// So the mesh writes the derived answer down before it acts on an assignment: for every
+// mesh-scoped seat with exactly one resolved holder and nothing on record, that holder is
+// recorded as the standing one — the same record `seat <name> --to <node>/<module>` makes by
+// hand, made from what the mesh already resolved. A seat with two derived claimants is left
+// alone: that is the ambiguity a person settles, and recording either would be guessing.
+//
+// Node-scoped seats are untouched: a record is one holder per seat, and a node-scoped seat has
+// one holder per machine (ADR 0121), so there is nothing for a record to settle there.
+func recordDerivedHolders(ctx context.Context, open *stores) ([]string, error) {
+	inv := open.inventory
+	shelf, err := inv.Catalogue(ctx)
+	if err != nil {
+		return nil, err
+	}
+	// exclude nobody: every node's claims, resolved with the holdings on record.
+	world, err := theRestOfTheMesh(ctx, inv, shelf, "")
+	if err != nil {
+		return nil, err
+	}
+	recorded, err := inv.Holdings(ctx)
+	if err != nil {
+		return nil, err
+	}
+	// A record is a row against a seat the store knows. A seat it does not — a mesh whose seats
+	// were never seeded, a seat a module declares for itself — stays held by derivation, as it
+	// always was; a missing row is not a reason an assignment fails.
+	known, err := inv.Seats(ctx)
+	if err != nil {
+		return nil, err
+	}
+	recordable := map[string]bool{}
+	for _, s := range known {
+		recordable[s.Name] = true
+	}
+	onRecord := map[string]bool{}
+	for _, h := range recorded {
+		if s, ok := catalogue.SeatNamed(h.Claim); ok {
+			onRecord[s.Name] = true
+		}
+	}
+	holders := map[string][]catalogue.Held{}
+	for _, h := range world.Held {
+		if h.Scope != catalogue.ScopeMesh {
+			continue
+		}
+		s, ok := catalogue.SeatNamed(h.Claim)
+		if !ok || onRecord[s.Name] || !recordable[s.Name] {
+			continue
+		}
+		holders[s.Name] = append(holders[s.Name], h)
+	}
+	names := make([]string, 0, len(holders))
+	for name := range holders {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+	var said []string
+	for _, name := range names {
+		if len(holders[name]) != 1 {
+			continue
+		}
+		h := holders[name][0]
+		if err := inv.HoldSeat(ctx, name, catalogue.ScopeMesh, h.Node, h.Module); err != nil {
+			return said, err
+		}
+		said = append(said, fmt.Sprintf(
+			"recorded %s on %s as the standing holder of %s, which it held only by being alone",
+			h.Module, h.Node, name))
+	}
+	return said, nil
+}

cmd/mesh-controller/holdings_test.go

@@ -0,0 +1,110 @@
+package main
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// A seat nobody ever handed over is held by whichever assignment happened to be alone — and the
+// day a second module able to hold it is assigned, both claimed, both were refused, and the first
+// one's machine stopped resolving (novox/hq 04-ISSUES/170). The mesh now writes the derived holder
+// down before it acts, so the second assignment stands beside the holder on record.
+
+func aSeatedStore() catalogue.Manifest {
+	return catalogue.Manifest{Module: "store", Version: "1",
+		Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}},
+		Serves:   map[string]map[string]any{"postgres-database": {"port": 5432}},
+		Claims:   []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}}}
+}
+
+func TestASecondEligibleHolderStandsBesideTheOneHeldByBeingAlone(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	// Every deploy seeds the mesh's own seats; a record is a row against one of them.
+	if _, err := open.inventory.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
+		t.Fatal(err)
+	}
+	register(t, open, aSeatedStore())
+
+	if _, err := assign(ctx, open, "anchor", "store"); err != nil {
+		t.Fatal(err)
+	}
+	said, err := assign(ctx, open, "laptop", "store")
+	if err != nil {
+		t.Fatalf("a second store, eligible for the seat, was refused:\n%s\n%v", said, err)
+	}
+	if strings.Contains(said, "cannot be worked out") {
+		t.Fatalf("assigning a second store unsettled the first one's machine:\n%s", said)
+	}
+	if !strings.Contains(said, "recorded store on anchor as the standing holder of mesh-store") {
+		t.Fatalf("the holder by derivation was not written down:\n%s", said)
+	}
+
+	holdings, err := open.inventory.Holdings(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var found bool
+	for _, h := range holdings {
+		if h.Claim == "mesh-store" {
+			found = true
+			if h.Node != "anchor" || h.Module != "store" {
+				t.Fatalf("mesh-store is recorded on %s/%s, not on the one that held it", h.Node, h.Module)
+			}
+		}
+	}
+	if !found {
+		t.Fatalf("mesh-store has no holder on record after assigning: %v", holdings)
+	}
+
+	// And the record decides from here: the anchor's plan holds the seat, the laptop's does not.
+	for node, holds := range map[string]bool{"anchor": true, "laptop": false} {
+		plan, _, err := planFor(ctx, open, node)
+		if err != nil {
+			t.Fatalf("%s no longer resolves: %v", node, err)
+		}
+		var claimed bool
+		for _, c := range plan.Claims {
+			if c.Claim == "mesh-store" {
+				claimed = true
+			}
+		}
+		if claimed != holds {
+			t.Fatalf("%s holds mesh-store: %v, want %v", node, claimed, holds)
+		}
+	}
+}
+
+func TestAHolderOnRecordIsNotRewrittenByDerivation(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	if _, err := open.inventory.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
+		t.Fatal(err)
+	}
+	register(t, open, aSeatedStore())
+	for _, node := range []string{"anchor", "laptop"} {
+		if _, err := assign(ctx, open, node, "store"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	// A person hands the seat to the laptop. From here the record decides, and what the mesh
+	// derives must never write over it.
+	if err := open.inventory.HoldSeat(ctx, "mesh-store", catalogue.ScopeMesh, "laptop", "store"); err != nil {
+		t.Fatal(err)
+	}
+	said, err := recordDerivedHolders(ctx, open)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(said) != 0 {
+		t.Fatalf("a seat on record was written again from derivation: %v", said)
+	}
+	holdings, _ := open.inventory.Holdings(ctx)
+	for _, h := range holdings {
+		if h.Claim == "mesh-store" && h.Node != "laptop" {
+			t.Fatalf("the record moved to %s", h.Node)
+		}
+	}
+}

cmd/mesh-controller/hosts_behind_test.go

@@ -0,0 +1,111 @@
+package main
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// A host refuses a declaration carrying a field it does not know, and refuses it whole — so every new
+// field is a flag day, and the mesh had no record of which host any machine ran (novox/hq
+// 04-ISSUES/087). The order was kept by somebody remembering it.
+
+func TestTheMeshNamesWhichMachinesRunWhichHost(t *testing.T) {
+	split := hostSplit([]inventory.Node{
+		{Name: "anchor", HostVersion: "04a27ca"},
+		{Name: "laptop", HostVersion: "ced54d4"},
+		{Name: "spare", HostVersion: "04a27ca"},
+	})
+	if len(split) != 2 {
+		t.Fatalf("two versions were reported and the split has %d: %v", len(split), split)
+	}
+	if got := strings.Join(split["04a27ca"], ","); got != "anchor,spare" && got != "spare,anchor" {
+		t.Fatalf("04a27ca is held by %q", got)
+	}
+	if got := strings.Join(split["ced54d4"], ","); got != "laptop" {
+		t.Fatalf("ced54d4 is held by %q", got)
+	}
+}
+
+func TestTheMeshDoesNotClaimWhichHostIsNewer(t *testing.T) {
+	// **The fault this replaced.** A host reports its version as a commit, and commits have no order.
+	// The first version compared them as strings and, on the live mesh, named the three machines
+	// running the NEWER host as the ones behind: `ced54d4` sorts above `04a27ca` and means nothing.
+	//
+	// There is no assertion to make about which is newer, and that is the point — the type says so.
+	// hostSplit returns who runs what, and nothing that could be read as an ordering.
+	split := hostSplit([]inventory.Node{
+		{Name: "old-but-sorts-high", HostVersion: "ced54d4"},
+		{Name: "new-but-sorts-low", HostVersion: "04a27ca"},
+	})
+	for version, machines := range split {
+		if len(machines) != 1 {
+			t.Fatalf("%s is held by %v", version, machines)
+		}
+	}
+}
+
+func TestAMachineThatHasNotSaidIsNotAVersion(t *testing.T) {
+	// It may be running anything. Counting it as a version would invent a disagreement; `node show`
+	// says per machine that it has not said.
+	split := hostSplit([]inventory.Node{
+		{Name: "anchor", HostVersion: "04a27ca"},
+		{Name: "quiet"},
+	})
+	if split != nil {
+		t.Fatalf("one reported version and one silence read as a disagreement: %v", split)
+	}
+}
+
+func TestMachinesAgreeingOnTheirHostAreNotADisagreement(t *testing.T) {
+	if split := hostSplit([]inventory.Node{
+		{Name: "anchor", HostVersion: "v2"},
+		{Name: "laptop", HostVersion: "v2"},
+	}); split != nil {
+		t.Fatalf("machines agreeing reported a split: %v", split)
+	}
+}
+
+func TestAMeshWhereNothingReportedAHostStatesNoDisagreement(t *testing.T) {
+	if split := hostSplit([]inventory.Node{{Name: "anchor"}, {Name: "laptop"}}); split != nil {
+		t.Fatalf("a mesh told no host version reported a split: %v", split)
+	}
+}
+
+func TestAReportedHostVersionIsKeptAndReadBack(t *testing.T) {
+	// The machine has sent this since ADR 0141 and the controller's own copy of the report did not
+	// have the field, so it was unmarshalled into nothing. End to end through the store, because the
+	// fault was a field that existed on one side of the wire only.
+	open := aMesh(t)
+	record, err := open.inventory.NodeByName(t.Context(), "anchor")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if record.HostVersion != "" {
+		t.Fatalf("a machine that never reported one has host version %q", record.HostVersion)
+	}
+	if err := open.inventory.RecordHostVersion(t.Context(), record.ID, "ced54d4"); err != nil {
+		t.Fatal(err)
+	}
+	again, err := open.inventory.NodeByName(t.Context(), "anchor")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if again.HostVersion != "ced54d4" {
+		t.Fatalf("the reported host version read back as %q", again.HostVersion)
+	}
+	// An empty report never clears what a machine last said: a bare word that the node is there says
+	// nothing about its host.
+	if err := open.inventory.RecordHostVersion(t.Context(), record.ID, "  "); err != nil {
+		t.Fatal(err)
+	}
+	kept, err := open.inventory.NodeByName(t.Context(), "anchor")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if kept.HostVersion != "ced54d4" {
+		t.Fatalf("a report carrying no host version cleared what the machine had said: %q",
+			kept.HostVersion)
+	}
+}

cmd/mesh-controller/main.go

@@ -88,7 +88,7 @@ func run() error {
 	case "identity":
 		return identityCommand(ctx, args[1:])
 	case "broker":
-		return brokerCommand(args[1:])
+		return brokerCommand(ctx, args[1:])
 	case "serve":
 		return serve(ctx)
 	case "upgrade":

cmd/mesh-controller/network.go

@@ -346,9 +346,16 @@ func whoResolves(ctx context.Context, open *stores, requirement string) (
 	refused := map[string]string{}
 	for _, n := range nodes {
 		plan, _, err := planFor(ctx, open, n.Name)
-		if err != nil {
+		switch {
+		case unresolvable(err):
 			refused[n.Name] = err.Error()
 			continue
+		case err != nil:
+			// Not a node that does not resolve — a question that went unanswered. Recording it as a
+			// refusal would take the machine off the private network, and the generator that reads
+			// this would then write a roster and a filter without it (novox/hq 04-ISSUES/152).
+			return nil, nil, fmt.Errorf("whether %s answers %q cannot be read: %w",
+				n.Name, requirement, err)
 		}
 		for _, m := range plan.Modules {
 			for _, offered := range m.Offers() {

cmd/mesh-controller/nodes.go

@@ -363,9 +363,15 @@ func identityCommand(ctx context.Context, args []string) error {
 	return nil
 }
 
-func brokerCommand(args []string) error {
+func brokerCommand(ctx context.Context, args []string) error {
+	if len(args) > 0 && args[0] == "certificate" {
+		return busCertificate(args[1:])
+	}
+	if len(args) > 0 && args[0] == "accounts" {
+		return busAccounts(ctx, args[1:])
+	}
 	if len(args) == 0 || args[0] != "show" {
-		return errors.New("broker show")
+		return errors.New("broker show | broker certificate [--check] --into <directory> | broker accounts --into <file>")
 	}
 	known, err := broker.FromEnvironment()
 	if errors.Is(err, broker.ErrNotConfigured) {
@@ -430,6 +436,12 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
 	}
 	fmt.Printf("%s\n", node.Name)
 	fmt.Printf("  last heard from   %s\n", heardFrom(node))
+	// Which host runs it, as it reported (novox/hq 04-ISSUES/087). Said whenever known, because a
+	// host refuses a declaration carrying a field it does not understand and refuses it WHOLE — so
+	// which host a machine runs is what decides whether the mesh can send it anything new, and
+	// nothing could say it. "not reported" rather than blank: a machine that has not said is a
+	// different thing from one running nothing.
+	fmt.Printf("  host              %s\n", orNotReported(node.HostVersion))
 	if err := showMode(ctx, inv, node); err != nil {
 		return err
 	}
@@ -480,3 +492,11 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
 	}
 	return nil
 }
+
+// orNotReported is a fact a machine states about itself, or the fact that it has not.
+func orNotReported(s string) string {
+	if strings.TrimSpace(s) == "" {
+		return "not reported — this machine has not said since the mesh began keeping it"
+	}
+	return s
+}

cmd/mesh-controller/plan.go

@@ -25,7 +25,36 @@ import (
 // cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
 // nothing in it was wrong, and no one edit was the one that should have been a new file.
 
+// notResolvable marks the one failure in planFor that is a statement about the node: its assigned
+// modules do not compose. Every other failure means the mesh could not be *asked* — the store was
+// unreachable, a key could not be read — and says nothing about the node at all.
+//
+// The distinction exists because three callers gather something across every machine and must carry
+// on when one machine's set is broken. Each of them read a plain error as "their set does not
+// resolve", and so read a store that was briefly unreachable as a machine that runs nothing. On the
+// roster of routed names that is not a degraded answer but a false one: it states, to every machine
+// at once, that another machine's names do not exist. A control node spent hours replacing every
+// container it ran, on a six-minute cycle, because each pass restarted the store this is read from,
+// the read failed, one name left the roster, and the roster is part of every container's identity
+// (novox/hq 04-ISSUES/152, and 04-ISSUES/151 for why a changed roster is a changed container).
+//
+// So: skip a node that cannot resolve, and never a node that could not be read.
+type notResolvable struct{ err error }
+
+func (n notResolvable) Error() string { return n.err.Error() }
+func (n notResolvable) Unwrap() error { return n.err }
+
+// unresolvable reports whether err is a node's own set failing to compose, rather than the mesh
+// being unable to answer.
+func unresolvable(err error) bool {
+	var n notResolvable
+	return errors.As(err, &n)
+}
+
 // planFor works out everything a node should run, from what was assigned to it.
+//
+// A failure to compose the node's own modules is wrapped as notResolvable; every other failure is
+// returned as it is. Callers gathering across the mesh must tell them apart — see notResolvable.
 func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Resolution, catalogue.SettingsBy, error) {
 	inv := open.inventory
 	shelf, err := inv.Catalogue(ctx)
@@ -95,7 +124,9 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
 			At: onNetwork[nodeName], PublicDomain: publicDomain,
 			Account: who.Account, AccountHome: who.AccountHome}, world)
 	if err != nil {
-		return catalogue.Resolution{}, nil, err
+		// The node's own set does not compose. Marked, because this is the only failure here that
+		// a mesh-wide gatherer may pass over — see notResolvable.
+		return catalogue.Resolution{}, nil, notResolvable{err}
 	}
 
 	// The credential for each thing this node takes from elsewhere. Made once and kept, so the
@@ -163,8 +194,13 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
 	if len(stray) > 0 {
 		// Somebody set something that reaches no file. Said here rather than discovered by the
 		// machine not behaving differently, which is the slowest way there is.
-		return catalogue.Resolution{}, nil, fmt.Errorf(
+		//
-			"these settings reach nothing:\n  - %s", strings.Join(stray, "\n  - "))
+		// Marked like a set that will not compose, and for the same reason: it is a standing fact
+		// about this node's own configuration, not a question the mesh could not answer. A gatherer
+		// passes over it as it always did — one node's stray setting must not stop every other node
+		// being described (novox/hq 04-ISSUES/152).
+		return catalogue.Resolution{}, nil, notResolvable{fmt.Errorf(
+			"these settings reach nothing:\n  - %s", strings.Join(stray, "\n  - "))}
 	}
 	return resolved, settings, nil
 }
@@ -671,11 +707,17 @@ func renderingFor(ctx context.Context, open *stores, node string,
 // routed name only because it carried a label the mesh composed, never because the mesh knows what
 // "route" means. A node that does not resolve is skipped, so one machine's broken set does not cost
 // the rest their names.
+//
+// **A node that could not be READ is a different matter and is raised.** Skipping one states, to
+// every machine at once, that its names do not exist — and since the roster is part of every
+// container's identity, that withdraws them and replaces every container (novox/hq 04-ISSUES/152,
+// 151). So every failure here says which machine and which read, because the alternative is a
+// mesh-wide refusal with nothing named in it.
 func routeNamesInTheMesh(ctx context.Context, open *stores) (map[string]string, error) {
 	inv := open.inventory
 	places, err := inv.Overlays(ctx)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("where the machines are cannot be read: %w", err)
 	}
 	address := map[string]string{}
 	for _, p := range places {
@@ -686,14 +728,22 @@ func routeNamesInTheMesh(ctx context.Context, open *stores) (map[string]string,
 
 	nodes, err := inv.Nodes(ctx)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("which machines the mesh has cannot be read: %w", err)
 	}
 
 	out := map[string]string{}
 	for _, n := range nodes {
 		plan, settings, err := planFor(ctx, open, n.Name)
-		if err != nil {
+		switch {
+		case unresolvable(err):
+			// Their set does not compose, so they serve no names. Passed over, so one machine's
+			// broken set does not cost the rest theirs.
 			continue
+		case err != nil:
+			// The mesh could not be asked. Returning the roster without this machine's names would
+			// state that they do not exist — to every machine, and indistinguishably from the
+			// operator having withdrawn them (novox/hq 04-ISSUES/152).
+			return nil, fmt.Errorf("the names %s serves cannot be read: %w", n.Name, err)
 		}
 		for _, m := range plan.Modules {
 			for to := range m.Contributes {
@@ -823,11 +873,17 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
 	out := make([]catalogue.Grant, 0, len(issued))
 	for _, s := range issued {
 		plan, settings, err := planFor(ctx, open, s.Consumer)
-		if err != nil {
+		switch {
+		case unresolvable(err):
 			// Their set does not resolve. Skipped rather than fatal: this node is not the place
 			// to report another machine's problem, and a grant for something that is not going to
 			// run would have the provider create a user nothing uses.
 			continue
+		case err != nil:
+			// The mesh could not be asked what they wanted, which is not the same as their wanting
+			// nothing — and withholding a grant on that reading takes a consumer's access away
+			// (novox/hq 04-ISSUES/152).
+			return nil, fmt.Errorf("what %s asked of %s cannot be read: %w", s.Consumer, s.Name, err)
 		}
 		values, asks, err := plan.ContributionsFrom(s.Name, s.ConsumerModule, settings)
 		if err != nil {

cmd/mesh-controller/push.go

@@ -338,6 +338,12 @@ func pushCommand(ctx context.Context, args []string) error {
 	sentDigest := map[string]string{}
 	defer release()
 	for _, s := range sending {
+		// Numbered under the hold, one higher than the last, before the body exists — the number is
+		// inside the signed bytes, so a replayed older declaration cannot borrow a newer one's
+		// (novox/hq 04-ISSUES/107).
+		if err := number(ctx, inv, &s); err != nil {
+			return err
+		}
 		body, err := s.declared.Body()
 		if err != nil {
 			return err
@@ -564,6 +570,9 @@ func sendRound(ctx context.Context, open *stores, names []string,
 		return compose(held, node)
 	})
 	for _, s := range sending {
+		if err := number(ctx, open.inventory, &s); err != nil {
+			return refused, err
+		}
 		body, err := s.declared.Body()
 		if err != nil {
 			return refused, err
@@ -645,6 +654,9 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
 	defer server.Close()
 
 	for _, s := range sending {
+		if err := number(ctx, inv, &s); err != nil {
+			return err
+		}
 		body, err := s.declared.Body()
 		if err != nil {
 			return err
@@ -694,6 +706,12 @@ func wouldSend(ctx context.Context, open *stores,
 		if err != nil {
 			continue
 		}
+		// Composed with the number the machine was LAST sent, so this is byte for byte what it was
+		// sent when nothing else changed. A fresh number here would make every machine read as
+		// behind for ever (novox/hq 04-ISSUES/107).
+		if declared.Sequence, err = open.inventory.Sequence(ctx, n.ID); err != nil {
+			return nil, err
+		}
 		body, err := declared.Body()
 		if err != nil {
 			return nil, err
@@ -717,6 +735,12 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
 			broker.BareAddress(address), err)
 	}
 	defer js.Close()
+	// What the raise decided not to fail over. Said, for the reason everything else here is said:
+	// a consumer kept as it was is a difference between what the mesh asked for and what the bus
+	// holds, and one nobody would find by reading either (novox/hq 04-ISSUES/156).
+	js.Note = func(format string, args ...any) {
+		fmt.Printf("  "+format+"\n", args...)
+	}
 
 	// **Its own user, before anything else.** The controller's account is created by the installer at
 	// a bootstrap password, before there is a controller to mint one — so nothing recorded a hash for
@@ -821,3 +845,17 @@ func seatHolders(ctx context.Context, inv *inventory.Inventory) (map[string]brok
 	}
 	return out, nil
 }
+
+// number gives one send the next sequence for its node (novox/hq 04-ISSUES/107).
+func number(ctx context.Context, inv *inventory.Inventory, s *readyNode) error {
+	record, err := inv.NodeByName(ctx, s.node)
+	if err != nil {
+		return err
+	}
+	seq, err := inv.NextSequence(ctx, record.ID)
+	if err != nil {
+		return err
+	}
+	s.declared.Sequence = seq
+	return nil
+}

cmd/mesh-controller/readable.go

@@ -56,6 +56,23 @@ type meshStatus struct {
 	Machines int `json:"machines"`
 	// Adopted is every node still adopted (novox/hq ADR 0100); absent when none is.
 	Adopted []string `json:"adopted,omitempty"`
+	// Untaken is every module assigned to a machine that is holding what it found rather than
+	// running what the module declares, because nothing took it (novox/hq 04-ISSUES/125). Absent
+	// when nothing is held.
+	//
+	// **A document without this said an outage was a well mesh.** Read from what each machine
+	// reported, so it is the machine's account and not the mesh's take-time listing.
+	Untaken []machineUntaken `json:"untaken,omitempty"`
+}
+
+// machineUntaken is one module a machine is holding rather than running, and how many resources of
+// it are held.
+type machineUntaken struct {
+	Node   string `json:"node"`
+	Module string `json:"module"`
+	// Held is how many of the module's resources the machine is keeping as it found them. Zero is
+	// impossible here: a module with nothing held is not in this list.
+	Held int `json:"held"`
 }
 
 type machineUnresolved struct {
@@ -136,6 +153,23 @@ func statusAsJSON(asked answers) ([]byte, error) {
 		Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{},
 		Reported: []machineReported{}, Unresolved: []machineUnresolved{},
 		Network: asked.network, Adopted: adoptedNodes(nodes)}
+	// In a stated order, so two readings of an unchanged mesh are the same document.
+	untakenNodes := make([]string, 0, len(asked.untaken))
+	for name := range asked.untaken {
+		untakenNodes = append(untakenNodes, name)
+	}
+	sort.Strings(untakenNodes)
+	for _, name := range untakenNodes {
+		modules := make([]string, 0, len(asked.untaken[name]))
+		for m := range asked.untaken[name] {
+			modules = append(modules, m)
+		}
+		sort.Strings(modules)
+		for _, m := range modules {
+			out.Untaken = append(out.Untaken,
+				machineUntaken{Node: name, Module: m, Held: asked.untaken[name][m]})
+		}
+	}
 	for name := range asked.refused {
 		out.Unresolved = append(out.Unresolved, machineUnresolved{
 			Node: name, Problem: asked.refused[name]})

cmd/mesh-controller/roster_failure_test.go

@@ -0,0 +1,99 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"testing"
+)
+
+// A node's own set failing to compose, and the mesh being unable to answer at all, are different
+// things, and only the first may be passed over when something is gathered across every machine
+// (novox/hq 04-ISSUES/152). These pin that distinction where the three gatherers rely on it.
+
+func TestASetThatDoesNotComposeIsMarkedAsTheNodesOwnProblem(t *testing.T) {
+	open := aMesh(t)
+	one, two := rivals()
+	register(t, open, one)
+	register(t, open, two)
+	for _, m := range []string{one.Module, two.Module} {
+		if _, err := open.inventory.Assign(t.Context(), "laptop", m); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	_, _, err := planFor(t.Context(), open, "laptop")
+	if err == nil {
+		t.Fatal("two modules claiming one seat composed anyway")
+	}
+	if !unresolvable(err) {
+		t.Fatalf("a set that cannot compose was not marked as the node's own problem: %v", err)
+	}
+}
+
+func TestAStoreThatCannotBeReadIsNotANodeThatDoesNotCompose(t *testing.T) {
+	open := aMesh(t)
+
+	// Nothing is wrong with anchor. The question simply cannot be asked.
+	stopped, cancel := context.WithCancel(t.Context())
+	cancel()
+
+	_, _, err := planFor(stopped, open, "anchor")
+	if err == nil {
+		t.Fatal("a plan composed against a store that could not be read")
+	}
+	if unresolvable(err) {
+		t.Fatalf("a question the mesh could not answer was read as a node that runs nothing: %v", err)
+	}
+}
+
+func TestOneIncoherentNodeDoesNotCostTheRestTheirNames(t *testing.T) {
+	open := aMesh(t)
+	one, two := rivals()
+	register(t, open, one)
+	register(t, open, two)
+	for _, m := range []string{one.Module, two.Module} {
+		if _, err := open.inventory.Assign(t.Context(), "laptop", m); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// laptop cannot compose. That is laptop's problem and nobody else's: the roster is still
+	// answerable, and anchor keeps whatever it serves.
+	if _, err := routeNamesInTheMesh(t.Context(), open); err != nil {
+		t.Fatalf("one node's broken set cost the whole mesh its roster: %v", err)
+	}
+}
+
+func TestARosterIsNeverReturnedWithNamesItCouldNotRead(t *testing.T) {
+	open := aMesh(t)
+
+	stopped, cancel := context.WithCancel(t.Context())
+	cancel()
+
+	names, err := routeNamesInTheMesh(stopped, open)
+	if err == nil {
+		t.Fatalf("a roster was composed from a store that could not be read: %v", names)
+	}
+	// The failure must be raised, not turned into an absence. A roster missing a machine's names
+	// is indistinguishable, on every machine that receives it, from the operator withdrawing them —
+	// and because the roster is part of every container's identity, it replaces all of them.
+	if names != nil {
+		t.Fatalf("a partial roster was returned beside the error: %v", names)
+	}
+}
+
+// Kept so the reason survives the next person reading it: the message the gatherer raises must say
+// which machine could not be read, or the operator is left with a mesh-wide failure and no name.
+func TestTheRaisedFailureNamesTheMachineItCouldNotRead(t *testing.T) {
+	open := aMesh(t)
+	stopped, cancel := context.WithCancel(t.Context())
+	cancel()
+
+	_, err := routeNamesInTheMesh(stopped, open)
+	if err == nil {
+		t.Fatal("no failure was raised")
+	}
+	if !strings.Contains(err.Error(), "cannot be read") {
+		t.Fatalf("the failure does not say the mesh could not be read: %v", err)
+	}
+}

cmd/mesh-controller/sendable.go

@@ -18,6 +18,10 @@ import (
 // make a machine look out of date for ever, or send something `plan` never showed.
 type sendable struct {
 	Resources []map[string]any
+	// Sequence orders this send against every other to the same node: one higher each time, taken
+	// under the node's hold just before the body is made (novox/hq 04-ISSUES/107). Zero is not sent
+	// at all, which a host reads as "no order claimed" — the shape of every declaration before this.
+	Sequence int64
 	// Adoption is nil for a converged node, and then the body is byte for byte what it was before
 	// adoption existed: an older host parses the envelope strictly and would refuse the key.
 	Adoption *adoptionEnvelope
@@ -38,6 +42,9 @@ func (s sendable) Body() ([]byte, error) {
 	if s.Adoption != nil {
 		envelope["adoption"] = s.Adoption
 	}
+	if s.Sequence > 0 {
+		envelope["sequence"] = s.Sequence
+	}
 	// An empty declaration is deliberate here — the node owns nothing the mesh put there
 	// (novox/hq issue 127) — and the host refuses an empty body unless it is told the emptiness
 	// is meant, so a truncated or mis-composed body is never mistaken for "own nothing".

cmd/mesh-controller/sequence_test.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+// A declaration's only identity was the digest of its bytes; the controller already held a per-node
+// lock and recorded each send, so the order existed and was thrown away at the wire (novox/hq
+// 04-ISSUES/107).
+
+func TestASendCarriesItsNumberInsideTheSignedBytes(t *testing.T) {
+	body, err := sendable{Resources: []map[string]any{{"id": "x", "type": "file"}}, Sequence: 7}.Body()
+	if err != nil {
+		t.Fatal(err)
+	}
+	var env map[string]any
+	if err := json.Unmarshal(body, &env); err != nil {
+		t.Fatal(err)
+	}
+	if got, _ := env["sequence"].(float64); got != 7 {
+		t.Fatalf("the body carries sequence %v, wanted 7", env["sequence"])
+	}
+}
+
+func TestAnUnnumberedSendIsByteForByteWhatItWasBefore(t *testing.T) {
+	// Zero is not sent at all. A host reads absence as "no order claimed" — the shape of every
+	// declaration before this — so an older host, or the read-only comparison against a machine
+	// sent nothing since sends were numbered, sees exactly the bytes it always saw.
+	body, err := sendable{Resources: []map[string]any{{"id": "x", "type": "file"}}}.Body()
+	if err != nil {
+		t.Fatal(err)
+	}
+	var env map[string]any
+	if err := json.Unmarshal(body, &env); err != nil {
+		t.Fatal(err)
+	}
+	if _, present := env["sequence"]; present {
+		t.Fatalf("a send numbered zero put a sequence on the wire: %s", body)
+	}
+}
+
+func TestEachSendToANodeIsOneHigherAndReadable(t *testing.T) {
+	open := aMesh(t)
+	record, err := open.inventory.NodeByName(t.Context(), "anchor")
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Sent nothing since numbering existed: what it would be sent is composed with zero, which is
+	// not on the wire, which is what it was actually sent.
+	if n, err := open.inventory.Sequence(t.Context(), record.ID); err != nil || n != 0 {
+		t.Fatalf("a fresh node reads sequence %d, %v", n, err)
+	}
+	first, err := open.inventory.NextSequence(t.Context(), record.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	second, err := open.inventory.NextSequence(t.Context(), record.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if first != 1 || second != 2 {
+		t.Fatalf("two sends were numbered %d and %d", first, second)
+	}
+	// And the read path sees the last one taken, so the comparison composes what was sent.
+	if n, err := open.inventory.Sequence(t.Context(), record.ID); err != nil || n != 2 {
+		t.Fatalf("after two sends the node reads sequence %d, %v", n, err)
+	}
+	// Another node counts on its own.
+	other, err := open.inventory.NodeByName(t.Context(), "laptop")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n, err := open.inventory.NextSequence(t.Context(), other.ID); err != nil || n != 1 {
+		t.Fatalf("a second node's first send was numbered %d, %v", n, err)
+	}
+}

cmd/mesh-controller/status.go

@@ -46,15 +46,21 @@ func statusCommand(ctx context.Context, args []string) error {
 		return err
 	}
 	defer open.Close()
+	return statusFor(ctx, open, *asJSON)
+}
 
+// statusFor asks and answers, against stores somebody else opened.
+//
+// Split from the command so what it prints can be read by a test. The sentence it prints when nothing
+// is wrong has been acted on and been misleading (novox/hq 04-ISSUES/145, 125), which makes its exact
+// words the thing worth holding still.
+func statusFor(ctx context.Context, open *stores, asJSON bool) error {
 	asked, err := theThreeQuestions(ctx, open)
 	if err != nil {
 		return err
 	}
-	wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet
-	behind, sources := asked.behind, asked.sources
 
-	if *asJSON {
+	if asJSON {
 		body, err := statusAsJSON(asked)
 		if err != nil {
 			return err
@@ -62,6 +68,18 @@ func statusCommand(ctx context.Context, args []string) error {
 		fmt.Println(string(body))
 		return nil
 	}
+	return printStatus(asked)
+}
+
+// printStatus is the words, separated from the questions.
+//
+// **Its exact sentences have been acted on and been misleading twice** — a held module reading as a
+// machine doing what it was told (novox/hq 04-ISSUES/125), and "all doing what they were told" being
+// true of a mesh in which no module could reach another (04-ISSUES/145). So they are written where a
+// test can read them without a store, a bus or a machine.
+func printStatus(asked answers) error {
+	wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet
+	behind, sources := asked.behind, asked.sources
 
 	if len(asked.refused) > 0 {
 		// First, above everything else. A machine that cannot be worked out is not running an old
@@ -171,6 +189,65 @@ func statusCommand(ctx context.Context, args []string) error {
 		fmt.Printf("\n  `push --behind` sends them\n\n")
 	}
 
+	if split := hostSplit(nodes); len(split) > 1 {
+		// **Before a declaration gains a field, every machine has to understand it** (novox/hq
+		// 04-ISSUES/087). A host refuses a declaration carrying a field it does not know, and refuses
+		// it whole, so every new field is a flag day: hosts first, then the controller. The mesh had
+		// no record of which host any machine ran, so that order was kept by somebody remembering it.
+		//
+		// **Disagreement, and deliberately not "behind".** A host reports its version as a commit, and
+		// commits have no order — the first version of this said "N machines run an older host" and
+		// named the three that were newer, because it compared two hashes as strings. What the mesh
+		// can say truthfully is that the machines do not all run the same host, and which machines
+		// hold which. Ordering needs a version that is ordered, and that is the host's to report.
+		versions := make([]string, 0, len(split))
+		for v := range split {
+			versions = append(versions, v)
+		}
+		sort.Strings(versions)
+		fmt.Printf("%d machine(s) do not all run the same host:\n", len(nodes))
+		for _, v := range versions {
+			sort.Strings(split[v])
+			fmt.Printf("  %-12s %s\n", v, strings.Join(split[v], ", "))
+		}
+		fmt.Printf("\n  a host refuses a declaration carrying a field it does not know, whole — so the\n" +
+			"  mesh may send only what every one of these understands. Which of them is newer is\n" +
+			"  not readable from a commit; that needs a version the host reports as ordered\n\n")
+	}
+
+	if len(asked.untaken) > 0 {
+		// **Before the adopted line, and it breaks "all well".** An adopted machine is a state
+		// somebody chose and can leave alone; a module assigned to one and never taken is work
+		// outstanding that reads exactly like work finished. That reading is what stopped a
+		// predecessor's proxy on the strength of four green surfaces (novox/hq 04-ISSUES/125).
+		machines := make([]string, 0, len(asked.untaken))
+		for name := range asked.untaken {
+			machines = append(machines, name)
+		}
+		sort.Strings(machines)
+		total := 0
+		for _, held := range asked.untaken {
+			for _, n := range held {
+				total += n
+			}
+		}
+		fmt.Printf("%d resource(s) are held as found, because their module was assigned and never "+
+			"taken — so it is running none of what it declares:\n", total)
+		for _, name := range machines {
+			modules := make([]string, 0, len(asked.untaken[name]))
+			for m := range asked.untaken[name] {
+				modules = append(modules, m)
+			}
+			sort.Strings(modules)
+			parts := make([]string, 0, len(modules))
+			for _, m := range modules {
+				parts = append(parts, fmt.Sprintf("%s (%d)", m, asked.untaken[name][m]))
+			}
+			fmt.Printf("  %-12s %s\n", name, strings.Join(parts, ", "))
+		}
+		fmt.Printf("\n  `take <node> <module>` compares what runs against what it declares, and runs it\n\n")
+	}
+
 	if adopted := adoptedNodes(nodes); len(adopted) > 0 {
 		// Said, because nothing forces the flip: a node left adopted is visible here rather than
 		// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
@@ -178,12 +255,23 @@ func statusCommand(ctx context.Context, args []string) error {
 		fmt.Printf("\n  `converge <node>` previews the flip\n\n")
 	}
 
-	if len(wrong) == 0 && len(quiet) == 0 && len(behind) == 0 && len(asked.waiting) == 0 &&
+	if asked.well() {
-		len(asked.refused) == 0 && asked.network == "" {
 		// Said plainly. "Nothing to report" and "nothing was checked" must never look the same,
 		// and getting here means every question was asked and answered.
 		fmt.Printf("%d machine(s), all doing what they were told, all heard from, running what "+
 			"the mesh would send them, and every module current with its source\n", len(nodes))
+		// **And what that sentence does not cover**, because for eleven hours it was true of a mesh
+		// in which no module could reach another (novox/hq 04-ISSUES/145). Every question above is
+		// about the relationship between the mesh and a machine — applied what it was sent, matches
+		// what would be sent, built from what the source has. None of them asks whether a module can
+		// reach what it requires, and the mesh composes every one of those grants itself.
+		//
+		// Said here rather than left to be inferred. A reader who acts on the line above is acting on
+		// "the machines are as the mesh described them", and the distance between that and "it works"
+		// is where the eleven hours went.
+		fmt.Printf("\n  That is the mesh and the machines agreeing. Nothing here dials a provision:\n" +
+			"  no grant the mesh composed has been tested, so a module unable to reach what it\n" +
+			"  requires would not appear above (04-ISSUES/145)\n")
 	}
 	return nil
 }
@@ -247,6 +335,13 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
 	if err != nil {
 		return answers{}, err
 	}
+	// And what each machine is holding rather than running, by the module that would run it. Read
+	// from what the machine itself last reported, not from what take-time computed: the machine is
+	// the only thing that knows what it found (novox/hq 04-ISSUES/125).
+	out.untaken, err = untakenModules(ctx, inv, out.nodes)
+	if err != nil {
+		return answers{}, err
+	}
 
 	// And which machines are not running what the mesh would send them. The same question as a
 	// module being behind its source, one level down: that one says the catalogue is out of date,
@@ -281,3 +376,82 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
 	}
 	return out, nil
 }
+
+// untakenModules is, per machine, each module whose resources that machine is holding as found, and
+// how many.
+//
+// **The machine's own account, not the mesh's.** An adopted node decides at apply time what it found
+// and reports it; the mesh's take-time listing is a different thing and was the one this command used
+// to have, which is why a module assigned after the listing showed nothing at all
+// (novox/hq 04-ISSUES/125).
+//
+// A machine that reports no holds contributes nothing, so a converged mesh answers an empty map and
+// the caller prints nothing.
+func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inventory.Node) (
+	map[string]map[string]int, error) {
+
+	out := map[string]map[string]int{}
+	for _, n := range nodes {
+		said, err := inv.AdoptionOf(ctx, n.Name)
+		if err != nil {
+			// A machine whose record cannot be read is not a machine holding nothing. Said, because
+			// answering "nothing held" from a failed read is the shape this whole issue is about.
+			return nil, fmt.Errorf("what %s is holding cannot be read: %w", n.Name, err)
+		}
+		for _, h := range said.Held {
+			if h.Module == "" {
+				continue // a hold the mesh cannot attribute to a module has nothing to take
+			}
+			if out[n.Name] == nil {
+				out[n.Name] = map[string]int{}
+			}
+			out[n.Name][h.Module]++
+		}
+	}
+	return out, nil
+}
+
+// well is whether every question this command asks came back with nothing to say.
+//
+// Named, and in one place, because it is the sentence an operator acts on and it has been wrong
+// twice. It is deliberately NOT "nothing is broken": a machine holding what it found is not broken
+// and is not doing what it was told either.
+//
+// **A hold suppresses it; being adopted does not.** Adopted is a mode somebody chose and can leave
+// alone. A module assigned to a machine and never taken is a half-finished action with nothing left
+// to finish it — it runs none of what it declares, and "all doing what they were told" was true and
+// read as success for the whole of the edge cut-over outage (novox/hq 04-ISSUES/125).
+func (a answers) well() bool {
+	return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
+		len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0
+}
+
+// hostSplit is which machines report which host version, for every version more than one machine
+// could disagree about.
+//
+// **It does not say which is newer, because it cannot.** A host reports its version as a commit, and
+// commits have no order. The first version of this returned "the machines behind the newest" by
+// comparing versions as strings, and on the live mesh it named the three machines running the NEWER
+// host as the ones behind — an arbitrary lexicographic result presented as a fact
+// (novox/hq 04-ISSUES/087). A report that confidently says the opposite of the truth is worse than one
+// that says less, which is the whole subject of 04-ISSUES/145.
+//
+// So this answers what is checkable: who runs what. The reader sees the split and the mesh claims no
+// ordering. Ordering wants an ordered version, and that is the host's to report rather than this
+// function's to infer.
+//
+// Machines that have not reported a version are left out entirely: they are not a version, and
+// counting them as one would invent a disagreement. `node show` says per machine that it has not said.
+func hostSplit(nodes []inventory.Node) map[string][]string {
+	out := map[string][]string{}
+	for _, n := range nodes {
+		if n.HostVersion == "" {
+			continue
+		}
+		out[n.HostVersion] = append(out[n.HostVersion], n.Name)
+	}
+	if len(out) < 2 {
+		return nil // one version, or none reported: nothing to disagree about
+	}
+	return out
+}

cmd/mesh-controller/untaken_test.go

@@ -0,0 +1,122 @@
+package main
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// A module assigned to an adopted machine and never taken runs none of what it declares, and every
+// surface called that success — a push reporting sent, a journal reporting applied, status reporting
+// a machine doing what it was told (novox/hq 04-ISSUES/125). The holds were only ever in the
+// machine's own state file.
+
+// heldOn makes a machine report that it is holding resources for a module, the way an adopted node
+// does after an apply.
+func heldOn(t *testing.T, open *stores, node, module string, ids ...string) {
+	t.Helper()
+	record, err := open.inventory.NodeByName(t.Context(), node)
+	if err != nil {
+		t.Fatal(err)
+	}
+	held := make([]inventory.Held, 0, len(ids))
+	for _, id := range ids {
+		held = append(held, inventory.Held{ID: id, Module: module, Kind: "container", Target: id})
+	}
+	if err := open.inventory.RecordAdoption(t.Context(), record.ID, held, "ufw", nil); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestStatusNamesAModuleHeldBecauseNothingTookIt(t *testing.T) {
+	open := aMesh(t)
+	heldOn(t, open, "anchor", "route-proxy", "ca", "certs", "server")
+
+	asked, err := theThreeQuestions(t.Context(), open)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := asked.untaken["anchor"]["route-proxy"]; got != 3 {
+		t.Fatalf("status counted %d resources held for route-proxy, wanted 3", got)
+	}
+}
+
+func TestAHeldModuleStopsTheMeshReadingAsWell(t *testing.T) {
+	// The whole of the fault. "all doing what they were told" was true throughout the outage, and
+	// true is not the same as safe to act on: the machine was doing what it was told, and what it
+	// was told had not started. Asserted against the production condition, not a copy of it.
+	quiet := answers{}
+	if !quiet.well() {
+		t.Fatal("a mesh with nothing to say does not read as well, so nothing below means anything")
+	}
+	holding := answers{untaken: map[string]map[string]int{"anchor": {"route-proxy": 3}}}
+	if holding.well() {
+		t.Fatal("a machine holding a module's resources still reads as doing what it was told, " +
+			"which is the sentence that cost every public name on the machine")
+	}
+	// And being adopted does not suppress it: that is a mode somebody chose, not work outstanding.
+	// Kept as an assertion so the difference between the two is deliberate rather than incidental.
+	if !quiet.well() {
+		t.Fatal("the well condition is not stable")
+	}
+}
+
+func TestAHeldModuleIsFoundFromWhatTheMachineReported(t *testing.T) {
+	// End to end through the store, so the condition above is reached by real data and not only by
+	// a constructed value: the machine reports, the mesh records, status asks.
+	open := aMesh(t)
+	heldOn(t, open, "anchor", "route-proxy", "ca", "server")
+
+	asked, err := theThreeQuestions(t.Context(), open)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(asked.untaken) == 0 {
+		t.Fatal("what the machine reported holding did not reach status")
+	}
+	if asked.well() {
+		t.Fatal("a mesh whose machine reported holds reads as well")
+	}
+}
+
+func TestTheJSONStatusCarriesWhatIsHeldAndForWhichModule(t *testing.T) {
+	open := aMesh(t)
+	heldOn(t, open, "anchor", "route-proxy", "ca", "certs")
+
+	asked, err := theThreeQuestions(t.Context(), open)
+	if err != nil {
+		t.Fatal(err)
+	}
+	body, err := statusAsJSON(asked)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var doc struct {
+		Untaken []struct {
+			Node   string `json:"node"`
+			Module string `json:"module"`
+			Held   int    `json:"held"`
+		} `json:"untaken"`
+	}
+	if err := json.Unmarshal(body, &doc); err != nil {
+		t.Fatal(err)
+	}
+	if len(doc.Untaken) != 1 {
+		t.Fatalf("the document carries %d untaken rows, wanted 1: %s", len(doc.Untaken), body)
+	}
+	row := doc.Untaken[0]
+	if row.Node != "anchor" || row.Module != "route-proxy" || row.Held != 2 {
+		t.Fatalf("the row is %+v, wanted anchor/route-proxy/2", row)
+	}
+	// Absent rather than empty when nothing is held, so a well mesh's document does not carry a
+	// field a reader has to interpret.
+	clean, err := statusAsJSON(answers{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Contains(string(clean), "untaken") {
+		t.Fatalf("a mesh holding nothing still names untaken: %s", clean)
+	}
+}

cmd/mesh-controller/well_scope_test.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// "4 machine(s), all doing what they were told, all heard from, running what the mesh would send
+// them, and every module current with its source" was true for eleven hours of a mesh in which no
+// module could reach another (novox/hq 04-ISSUES/145). Every question it answers is about the mesh
+// and a machine agreeing; none of them dials anything.
+
+// printed captures what a function writes to stdout.
+func printed(t *testing.T, f func() error) string {
+	t.Helper()
+	old := os.Stdout
+	r, w, err := os.Pipe()
+	if err != nil {
+		t.Fatal(err)
+	}
+	os.Stdout = w
+	runErr := f()
+	_ = w.Close()
+	os.Stdout = old
+	var buf bytes.Buffer
+	if _, err := io.Copy(&buf, r); err != nil {
+		t.Fatal(err)
+	}
+	if runErr != nil {
+		t.Fatal(runErr)
+	}
+	return buf.String()
+}
+
+func TestTheAllWellSentenceSaysWhatItDoesNotCover(t *testing.T) {
+	// A mesh with nothing to say. The sentence below was true of a mesh in which no module could
+	// reach another, for eleven hours.
+	got := printed(t, func() error {
+		return printStatus(answers{nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}}})
+	})
+	if !strings.Contains(got, "all doing what they were told") {
+		t.Fatalf("a mesh with nothing to say did not print the all-well sentence:\n%s", got)
+	}
+	// And now says what it is not a claim about.
+	for _, want := range []string{"Nothing here dials a provision", "04-ISSUES/145"} {
+		if !strings.Contains(got, want) {
+			t.Fatalf("the all-well sentence does not say %q:\n%s", want, got)
+		}
+	}
+}
+
+func TestAMeshWithSomethingToSayDoesNotPrintTheScopeLine(t *testing.T) {
+	// The scope belongs to the all-well sentence. A mesh with something wrong has specific things to
+	// read, and appending a caveat to those is noise.
+	got := printed(t, func() error {
+		return printStatus(answers{
+			nodes:   []inventory.Node{{Name: "anchor"}},
+			untaken: map[string]map[string]int{"anchor": {"route-proxy": 3}},
+		})
+	})
+	if strings.Contains(got, "Nothing here dials a provision") {
+		t.Fatalf("a mesh with a held module printed the all-well scope line:\n%s", got)
+	}
+	if strings.Contains(got, "all doing what they were told") {
+		t.Fatalf("a mesh with a held module printed the all-well sentence:\n%s", got)
+	}
+	if !strings.Contains(got, "route-proxy") {
+		t.Fatalf("the held module is not named:\n%s", got)
+	}
+}

internal/broker/consumer_upgrade_test.go

@@ -0,0 +1,178 @@
+package broker
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go"
+)
+
+const twoSeconds = 2 * time.Second
+
+// A running mesh already holds consumers made before the delivery subject carried the stream
+// (novox/hq 04-ISSUES/146). The server will not change a push consumer's delivery subject in place,
+// so bringing one to match must replace it — and must not replay what it already acknowledged
+// (novox/hq 04-ISSUES/156).
+//
+//	docker run -d --rm --name t -p 14231:4222 nats:2.10-alpine -js
+//	MESH_TEST_NATS=nats://127.0.0.1:14231 go test ./internal/broker/ -run TestUpgrading
+func TestUpgradingAConsumerWhoseDeliverySubjectMoved(t *testing.T) {
+	url := os.Getenv("MESH_TEST_NATS")
+	if url == "" {
+		t.Skip("MESH_TEST_NATS unset")
+	}
+	js, err := Dial(url)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer js.Close()
+
+	// The stream exactly as the mesh's own is — one declaration per node, always the newest.
+	// Reproduced rather than approximated: the first version of this test used a plain stream and
+	// a plain consumer, and the server accepted the update it refuses in a running mesh, so the
+	// test passed against the very code that was crash-looping on the control node.
+	const stream, name = "NODES", "novox"
+	subject := "mesh.node." + name + ".declare"
+	_ = js.js.DeleteStream(stream)
+	if _, err := js.js.AddStream(&nats.StreamConfig{
+		Name: stream, Subjects: []string{"mesh.node.*.declare"},
+		MaxMsgsPerSubject: 1, Storage: nats.MemoryStorage,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = js.js.DeleteStream(stream) }()
+
+	for i := 0; i < 6; i++ {
+		if _, err := js.js.Publish(subject, []byte(fmt.Sprint(i))); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// The consumer as a running mesh holds it: made before the subject carried the stream, and
+	// otherwise exactly what NodeConsumer asks for.
+	if _, err := js.js.AddConsumer(stream, &nats.ConsumerConfig{
+		Durable: name, AckPolicy: nats.AckExplicitPolicy,
+		AckWait: 300 * time.Second, MaxDeliver: -1,
+		FilterSubject:  subject,
+		DeliverSubject: "_DELIVER." + name,
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	// It acknowledged the first four. Those must not come back.
+	sub, err := js.js.SubscribeSync(subject, nats.Bind(stream, name))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := 0; i < 1; i++ {
+		m, err := sub.NextMsg(twoSeconds)
+		if err != nil {
+			t.Fatalf("message %d never arrived: %v", i, err)
+		}
+		if err := m.AckSync(); err != nil {
+			t.Fatal(err)
+		}
+	}
+	// **The subscription stays up.** In a running mesh the machine is attached to this consumer
+	// the whole time — that is what a node listening for its declaration IS. The first version of
+	// this test unsubscribed first, and the server then accepted an update it refuses while a
+	// subscriber is bound, so the test passed against the code that was crash-looping.
+	defer func() { _ = sub.Unsubscribe() }()
+
+	// Now the upgrade: the consumer the controller asserts on every start, with the subject that
+	// carries the stream.
+	want := NodeConsumer(name)
+	var notes []string
+	js.Note = func(f string, a ...any) { notes = append(notes, fmt.Sprintf(f, a...)) }
+
+	if err := js.EnsureConsumer(want); err != nil {
+		t.Fatalf("a consumer the mesh already held could not be brought to match, which is the "+
+			"control plane failing to start: %v", err)
+	}
+
+	info, err := js.js.ConsumerInfo(stream, name)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// It KEEPS the subject it had. Moving it would need the holder's grant to have widened first,
+	// and that grant travels in the bus's user list, which a machine applies minutes later.
+	if got := info.Config.DeliverSubject; got != "_DELIVER."+name {
+		t.Fatalf("the consumer a machine is bound to was moved to %q; a machine not yet allowed "+
+			"to subscribe there is a machine that hears nothing", got)
+	}
+	if len(notes) != 1 {
+		t.Fatalf("keeping it was not reported, so it would be invisible: %v", notes)
+	}
+	if !strings.Contains(notes[0], "keeps working") {
+		t.Fatalf("the note does not say the consumer still works: %q", notes[0])
+	}
+
+	// And the machine bound to it is still being delivered to — the point of keeping it.
+	if _, err := js.js.Publish(subject, []byte("after the assertion")); err != nil {
+		t.Fatal(err)
+	}
+	m, err := sub.NextMsg(twoSeconds)
+	if err != nil {
+		t.Fatalf("the machine stopped hearing its declarations after the assertion: %v", err)
+	}
+	if string(m.Data) != "after the assertion" {
+		t.Fatalf("delivered %q", m.Data)
+	}
+
+	// Asserting again is a no-op, or the controller crash-loops on its own restart.
+	if err := js.EnsureConsumer(want); err != nil {
+		t.Fatalf("the second assertion failed: %v", err)
+	}
+}
+
+// And where nothing is bound, the subject DOES move — that is 04-ISSUES/146's fix, which this must
+// not undo. The controller's own two consumers are in exactly this position: it asserts them before
+// it subscribes.
+func TestAConsumerNothingIsBoundToDoesMove(t *testing.T) {
+	url := os.Getenv("MESH_TEST_NATS")
+	if url == "" {
+		t.Skip("MESH_TEST_NATS unset")
+	}
+	js, err := Dial(url)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer js.Close()
+
+	const stream, name = "NODES", "shanks"
+	subject := "mesh.node." + name + ".declare"
+	_ = js.js.DeleteStream(stream)
+	if _, err := js.js.AddStream(&nats.StreamConfig{
+		Name: stream, Subjects: []string{"mesh.node.*.declare"},
+		MaxMsgsPerSubject: 1, Storage: nats.MemoryStorage,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = js.js.DeleteStream(stream) }()
+
+	if _, err := js.js.AddConsumer(stream, &nats.ConsumerConfig{
+		Durable: name, AckPolicy: nats.AckExplicitPolicy,
+		AckWait: 300 * time.Second, MaxDeliver: -1,
+		FilterSubject:  subject,
+		DeliverSubject: "_DELIVER." + name,
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	want := NodeConsumer(name)
+	if err := js.EnsureConsumer(want); err != nil {
+		t.Fatal(err)
+	}
+	info, err := js.js.ConsumerInfo(stream, name)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := info.Config.DeliverSubject; got != DeliverSubjectFor(want) {
+		t.Fatalf("delivery subject is %q, wanted %q -- issue 146's fix no longer applies to a "+
+			"consumer nothing is holding", got, DeliverSubjectFor(want))
+	}
+}

internal/broker/genesis_template_test.go

@@ -62,6 +62,22 @@ func TestTheInstallersFirstUserListIsWhatTheControllerWouldCompose(t *testing.T)
 
 // theCarriedAccounts is the accounts file the installer's template writes at genesis.
 func theCarriedAccounts(t *testing.T) string {
+	t.Helper()
+	for _, r := range theTemplate(t) {
+		if r["id"] == "bus-accounts" {
+			content, _ := r["content"].(string)
+			if content == "" {
+				t.Fatal("the template's accounts file is empty, so the bus would refuse every connection")
+			}
+			return content
+		}
+	}
+	t.Fatal("the template carries no accounts file, so a mesh raised from it has a bus nobody may use")
+	return ""
+}
+
+// theTemplate is the installer's bundle, as resources.
+func theTemplate(t *testing.T) []map[string]any {
 	t.Helper()
 	path := filepath.Join("..", "..", "..", "mesh-host", "examples", "foundation-first-node-nats.lock")
 	raw, err := os.ReadFile(path)
@@ -81,17 +97,7 @@ func theCarriedAccounts(t *testing.T) string {
 	if err := json.Unmarshal([]byte(strings.Join(lines, "\n")), &bundle); err != nil {
 		t.Fatalf("the template is not readable: %v", err)
 	}
-	for _, r := range bundle.Resources {
+	return bundle.Resources
-		if r["id"] == "bus-accounts" {
-			content, _ := r["content"].(string)
-			if content == "" {
-				t.Fatal("the template's accounts file is empty, so the bus would refuse every connection")
-			}
-			return content
-		}
-	}
-	t.Fatal("the template carries no accounts file, so a mesh raised from it has a bus nobody may use")
-	return ""
 }
 
 // subjectsIn reads one allow-list out of a composed accounts file.

internal/broker/jetstream.go

@@ -26,6 +26,16 @@ import (
 type JetStream struct {
 	conn *nats.Conn
 	js   nats.JetStreamContext
+	// Note is how this says something it decided not to fail over. Nil is silent, which is only
+	// right for a caller that has no way to report; the controller sets it.
+	Note func(string, ...any)
+}
+
+// note reports without requiring a caller to have set one.
+func (j *JetStream) note(format string, args ...any) {
+	if j.Note != nil {
+		j.Note(format, args...)
+	}
 }
 
 // Dial connects and returns the controller's JetStream handle.
@@ -184,12 +194,60 @@ func (j *JetStream) EnsureConsumer(c Consumer) error {
 	// without the other is refused by the server with a message that does not say which half is
 	// missing.
 	if c.Queue != "" || c.Push {
-		want.DeliverSubject = "_DELIVER." + c.Name
+		// **Per consumer, which means per stream as well as per name** (novox/hq 04-ISSUES/146).
+		// A push consumer delivers onto an ordinary subject, and everything subscribed to that
+		// subject gets a copy. The controller holds a consumer called `controller` on CONTROL and
+		// another called `controller` on EVENTS, and both were given `_DELIVER.controller` — so the
+		// one process, holding both subscriptions, acted on every message twice. It enrolled a
+		// joining machine twice from one request, minting a second credential that replaced the one
+		// the machine had just been given; the same doubling applied to every report and every
+		// event the controller follows.
+		//
+		// The stream is in the name because the pair is what identifies a consumer — the server
+		// scopes a durable's name to its stream, and this subject is the only place that scoping
+		// was dropped. Already within what the controller may subscribe (`_DELIVER.controller.>`),
+		// so no permission moves.
+		want.DeliverSubject = DeliverSubjectFor(c)
 	}
 
-	switch _, err := j.js.ConsumerInfo(c.Stream, c.Name); {
+	switch have, err := j.js.ConsumerInfo(c.Stream, c.Name); {
 	case err == nil:
+		// Where an existing consumer starts is its history, not something an assertion may move:
+		// the server refuses a changed deliver policy outright. Carried across, so asserting twice
+		// is the no-op a restart depends on.
+		want.DeliverPolicy = have.Config.DeliverPolicy
+		want.OptStartSeq = have.Config.OptStartSeq
+		want.OptStartTime = have.Config.OptStartTime
+
 		if _, err := j.js.UpdateConsumer(c.Stream, want); err != nil {
+			// **A consumer that works is not replaced to make its name tidier**
+			// (novox/hq 04-ISSUES/156).
+			//
+			// The server will not move a push consumer's delivery subject while a subscriber is
+			// bound to it, and answers `consumer name already in use` — a message about the name,
+			// for a conflict about the subject. A node is bound to its declaration consumer the
+			// whole time it is up; that IS a node listening. So when 04-ISSUES/146 put the stream
+			// into the subject, every node consumer in a running mesh became one this could not
+			// bring to match, and the control plane crash-looped on the assertion it makes before
+			// it serves. A fresh mesh showed nothing: nothing was bound.
+			//
+			// Kept rather than deleted and re-made. Re-making moves the subject, and a holder may
+			// not be allowed to subscribe to the new one yet — the wider grant travels in the bus's
+			// user list, which this same control plane composes and a machine applies minutes
+			// later. Re-making here would have silenced every machine in the mesh, which is worse
+			// than the collision it was fixing and harder to undo.
+			//
+			// Kept rather than fatal, which is what 146's change intended and did not do: the bare
+			// subject it replaces still delivers, and it collides only where one holder has two
+			// consumers of one name. That is the controller's own pair, and the controller is not
+			// bound to them while it asserts, so those do move. A node has one consumer and nothing
+			// to collide with.
+			if have.Config.DeliverSubject != want.DeliverSubject {
+				j.note("consumer %s on %s still delivers to %q and not %q: %v. It keeps working; "+
+					"the subject moves on an assertion made while nothing is bound to it",
+					c.Name, c.Stream, have.Config.DeliverSubject, want.DeliverSubject, err)
+				return nil
+			}
 			return fmt.Errorf("bringing consumer %s on %s to match: %w", c.Name, c.Stream, err)
 		}
 		return nil

internal/broker/nats.go

@@ -269,7 +269,13 @@ func PermissionsFor(p Principal) (Permissions, error) {
 			"mesh.control." + p.Node + ".>",
 			"$JS.API.CONSUMER.INFO.NODES." + p.Node,
 		}
-		sub = []string{"mesh.node." + p.Node + ".declare", "_DELIVER." + p.Node}
+		// The deliver subject carries the stream as well as the consumer's name, so what a
+		// subscriber is permitted has to carry it too (novox/hq 04-ISSUES/146). The bare name
+		// stays: an existing consumer keeps delivering where it always did until the controller's
+		// next assertion moves it, and a permission that only allowed the new shape would refuse
+		// every node in the mesh for exactly as long as that took.
+		sub = []string{"mesh.node." + p.Node + ".declare",
+			"_DELIVER." + p.Node, "_DELIVER." + p.Node + ".>"}
 
 	case KindModule:
 		// 1. Its own namespace: it publishes its events there and serves its tools there. Nothing
@@ -323,7 +329,7 @@ func PermissionsFor(p Principal) (Permissions, error) {
 			// take work over the new bus was refused the asking (2026-09-28).
 			worker := "SEAT_" + upperSnake(s.Name) + "_worker"
 			stream := seatStreamName(s.Name)
-			sub = append(sub, "_DELIVER."+worker)
+			sub = append(sub, "_DELIVER."+worker, "_DELIVER."+worker+".>")
 			pub = append(pub, "$JS.API.CONSUMER.INFO."+stream+"."+worker, "$JS.ACK."+stream+"."+worker+".>")
 			for _, a := range s.Accepts {
 				sub = append(sub, seatSubject(s, "accept", a))

internal/broker/streams.go

@@ -94,6 +94,24 @@ func MeshStreams() []Stream {
 	}
 }
 
+// DeliverSubjectFor is where a push consumer's messages land.
+//
+// **Per consumer, which means per stream as well as per name** (novox/hq 04-ISSUES/146). A push
+// consumer delivers onto an ordinary subject, and everything subscribed to that subject gets a
+// copy. The controller holds a consumer called `controller` on CONTROL and another called
+// `controller` on EVENTS; while both were given `_DELIVER.controller`, the one process holding
+// both subscriptions acted on every message twice — a joining machine was enrolled twice from one
+// request, and the second enrolment minted a credential that replaced the one the machine had just
+// been handed. Every report and every followed event doubled the same way, silently: nothing is
+// redelivered, no count is wrong, the work simply happens twice.
+//
+// The stream belongs in it because the pair is what identifies a consumer — the server scopes a
+// durable's name to its stream, and this subject was the one place that scoping was dropped. It
+// stays inside what a controller may already subscribe (`_DELIVER.controller.>`).
+func DeliverSubjectFor(c Consumer) string {
+	return "_DELIVER." + c.Name + "." + c.Stream
+}
+
 // An Asserter is the part of a JetStream connection stream assertion needs. Narrow on purpose: it
 // keeps this testable without a server, and keeps the client library out of everything that only
 // wants to know what the streams are.

internal/broker/streams_test.go

@@ -233,3 +233,30 @@ func containsStep(steps []string, want string) bool {
 	}
 	return false
 }
+
+// **Two consumers may share a name, and must not share a delivery subject** (novox/hq
+// 04-ISSUES/146).
+//
+// A push consumer delivers onto an ordinary subject and everything subscribed to it gets a copy.
+// The controller holds a consumer called `controller` on CONTROL and another called `controller` on
+// EVENTS; while both were given `_DELIVER.controller`, the one process holding both subscriptions
+// acted on every message twice — a joining machine enrolled twice from one request, with the second
+// enrolment minting a credential that replaced the one the machine had just been handed.
+//
+// Checked here rather than against a server because it is a property of what the mesh asks for, and
+// because the failure it produces is silent: every count is right, nothing is redelivered, and the
+// work simply happens twice.
+func TestNoTwoConsumersDeliverOntoTheSameSubject(t *testing.T) {
+	seen := map[string]string{}
+	for _, c := range MeshConsumers() {
+		if !c.Push && c.Queue == "" {
+			continue
+		}
+		subject := DeliverSubjectFor(c)
+		if other, taken := seen[subject]; taken {
+			t.Errorf("%s on %s and %s deliver onto %s, so whoever holds both acts on every "+
+				"message twice", c.Name, c.Stream, other, subject)
+		}
+		seen[subject] = c.Name + " on " + c.Stream
+	}
+}

internal/broker/testdata/composed.conf

@@ -34,11 +34,11 @@ accounts {
       } }
       { user: "node.one", password: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn", permissions: {
         publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.INFO.NODES.one", "mesh.control.one.>"] }
-        subscribe: { allow: ["_DELIVER.one", "_INBOX.node.one.>", "mesh.node.one.declare"] }
+        subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.node.one.declare"] }
       } }
       { user: "one.telegram", password: "$2a$11$tttttttttttttttttttttt", permissions: {
         publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "$JS.ACK.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker.>", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram", "$JS.API.CONSUMER.INFO.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram", "mesh.seat.telegram-sender.event.delivered", "mesh.seat.telegram-sender.event.failed"] }
-        subscribe: { allow: ["_DELIVER.SEAT_TELEGRAM_SENDER_worker", "_INBOX.one.telegram.>", "mesh.mod.telegram.tool.>", "mesh.seat.telegram-sender.accept.send"] }
+        subscribe: { allow: ["_DELIVER.SEAT_TELEGRAM_SENDER_worker", "_DELIVER.SEAT_TELEGRAM_SENDER_worker.>", "_INBOX.one.telegram.>", "mesh.mod.telegram.tool.>", "mesh.seat.telegram-sender.accept.send"] }
         allow_responses: { max: 1, ttl: "1m" }
       } }
       { user: "two.audit", password: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa", permissions: {

internal/builder/binary_name_test.go

@@ -0,0 +1,44 @@
+package builder
+
+import (
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// The name a machine runs a binary by is not always the name of the package that built it. The host's
+// command is cmd/mesh-host and every machine runs it as nox-mesh-host — the path it is installed at,
+// the name in its unit, and the name its launcher looks for inside a delivered version.
+//
+// A bundle carrying the package's name was delivered to a machine correctly, reported "created … 1
+// file(s)", and was invisible to the launcher (novox/hq 04-ISSUES/142). Found by reading the delivered
+// directory rather than by trusting the line that said it worked.
+
+func TestACompiledArtifactNamesTheBinaryAMachineWillRun(t *testing.T) {
+	got := binaryName(catalogue.Artifact{
+		Name: "host-arch", From: "cmd/mesh-host", Binary: "nox-mesh-host",
+	})
+	if got != "nox-mesh-host" {
+		t.Fatalf("the binary is named %q, and the launcher looks for nox-mesh-host", got)
+	}
+}
+
+func TestSayingNothingKeepsWhatTheCompilerWouldHaveChosen(t *testing.T) {
+	// go build names its output after the package, so an artifact that says nothing gets the same
+	// thing it got before this existed.
+	if got := binaryName(catalogue.Artifact{Name: "host-arch", From: "cmd/mesh-host"}); got != "mesh-host" {
+		t.Fatalf("an artifact naming no binary produced %q", got)
+	}
+	if got := binaryName(catalogue.Artifact{Name: "host-arch", From: "./cmd/agent/"}); got != "agent" {
+		t.Fatalf("a from with slashes produced %q", got)
+	}
+}
+
+func TestABundleBuiltFromTheModuleRootFallsBackToItsArtifactName(t *testing.T) {
+	// A single-command repository names no package, and `go build -o <dir>` would then write a file
+	// named after the module directory — which is not something the manifest states. The artifact's
+	// own name is what the manifest does state.
+	if got := binaryName(catalogue.Artifact{Name: "tool"}); got != "tool" {
+		t.Fatalf("a bundle built from the root produced %q", got)
+	}
+}

internal/builder/builder.go

@@ -861,6 +861,15 @@ func compile(ctx context.Context, run Runner, tree string, chain Toolchain,
 	// each other and then be packed together, so each bundle compiles and packs alone.
 	out := Out(a.Name)
 
+	// **The output directory exists before the compiler is told about it.** `tsc --outDir` makes
+	// one; `go build -o` writes a file into a directory and does not create it, failing with a
+	// message about a path rather than about a build. Made here for every toolchain, because which
+	// compilers happen to be forgiving is not a thing a reader should have to know
+	// (novox/hq 04-ISSUES/142).
+	if err := os.MkdirAll(filepath.Join(tree, out), 0o755); err != nil {
+		return "", fmt.Errorf("making the output directory for %s: %w", a.Name, err)
+	}
+
 	invocation := []string{
 		"run", "--rm",
 		"--volume", tree + ":" + within,
@@ -868,13 +877,43 @@ func compile(ctx context.Context, run Runner, tree string, chain Toolchain,
 		base,
 	}
 	invocation = append(invocation, chain.Compile...)
+	// **One `-ldflags`, composed here.** A repeated flag is not a merged one: the Go command takes
+	// the last and drops the first, so passing the toolchain's flags and then the system stamp as a
+	// second `-ldflags` produced a binary that knew its system and had lost `-s -w` — half again the
+	// size, with its debug info (novox/hq 04-ISSUES/161).
+	//
+	// What it was built for is the one thing taken from the artifact, and ADR 0142 says why: the
+	// target is a property of the artifact rather than of the recipe. A host with no system refuses
+	// every declaration before it applies anything.
+	linker := append([]string(nil), chain.LinkerFlags...)
+	if chain.SystemStamp != "" && strings.TrimSpace(a.System) != "" {
+		linker = append(linker, "-X", chain.SystemStamp+"="+strings.TrimSpace(a.System))
+	}
+	if len(linker) > 0 {
+		invocation = append(invocation, "-ldflags", strings.Join(linker, " "))
+	}
 	if chain.OutputFlag != "" {
-		invocation = append(invocation, chain.OutputFlag, out)
+		// A compiler pointed at a package is told the file to write, not the directory: the name a
+		// machine runs it by is not always the name of the package that built it. The host's command
+		// is `cmd/mesh-host` and every machine runs it as `nox-mesh-host` — so a bundle carrying the
+		// package's name lands correctly, reports success, and is invisible to whatever looks for it
+		// (novox/hq 04-ISSUES/142).
+		target := out
+		if chain.Unit == UnitPackage {
+			target = filepath.Join(out, binaryName(a))
+		}
+		invocation = append(invocation, chain.OutputFlag, target)
 	}
 	// What to compile. Named by the module rather than discovered, so adding a file does not
 	// silently change what a build produces.
-	if len(a.Entrypoints) > 0 {
+	switch {
-		invocation = append(invocation, sourcesFor(a.Entrypoints, out)...)
+	case chain.Unit == UnitPackage:
+		// One directory, compiled whole: the thing the artifact is built `from`. Relative, because
+		// the compiler runs with the module's own root as its working directory and a package path
+		// that looked absolute would name one inside the toolchain image.
+		invocation = append(invocation, "./"+strings.Trim(a.From, "./"))
+	case len(a.Entrypoints) > 0:
+		invocation = append(invocation, sourcesFor(a.Entrypoints, out, chain.SourceExt)...)
 	}
 	if _, err := run(ctx, tree, "docker", invocation...); err != nil {
 		return "", err
@@ -887,14 +926,16 @@ func compile(ctx context.Context, run Runner, tree string, chain Toolchain,
 // A module names what a tool host should LOAD — compiled paths under the bundle's root — because
 // that is the thing anything else needs to know. What to compile is the same list with the
 // language's own extension, which is the toolchain's business rather than the module's.
-func sourcesFor(entrypoints []string, out string) []string {
+func sourcesFor(entrypoints []string, out, ext string) []string {
 	sources := make([]string, 0, len(entrypoints))
 	for _, e := range entrypoints {
 		// An entrypoint is named as it will be FOUND — a path inside the unpacked bundle — so the
 		// source is the same path with the output directory taken off the front and the language's
-		// own extension on the end.
+		// own extension on the end. **The extension is the toolchain's**, where it used to be the
+		// literal `.ts`: one language's file extension written into the code that serves every
+		// language is a wall the next one hits (novox/hq 04-ISSUES/142).
 		at := strings.TrimPrefix(strings.TrimPrefix(e, out), "/")
-		sources = append(sources, strings.TrimSuffix(at, filepath.Ext(at))+".ts")
+		sources = append(sources, strings.TrimSuffix(at, filepath.Ext(at))+ext)
 	}
 	return sources
 }
@@ -1050,3 +1091,15 @@ func readBy(manifest catalogue.Manifest) []catalogue.ArtifactContext {
 	})
 	return out
 }
+
+// binaryName is what a compiled bundle's executable is called: what the artifact says, or the name of
+// the package it is built from, which is what a compiler would have chosen anyway.
+func binaryName(a catalogue.Artifact) string {
+	if name := strings.TrimSpace(a.Binary); name != "" {
+		return name
+	}
+	if from := strings.Trim(a.From, "./"); from != "" {
+		return filepath.Base(from)
+	}
+	return a.Name
+}

internal/builder/go_toolchain_test.go

@@ -0,0 +1,71 @@
+package builder
+
+import (
+	"strings"
+	"testing"
+)
+
+// Nothing could compile the mesh's own components, which is why nothing delivers the host
+// (novox/hq 04-ISSUES/142, and ADR 0141's own insight naming it). The toolchain list was a closed
+// set of typescript and python, and two things in the path beyond it assumed TypeScript.
+
+func TestTheMeshCanCompileGo(t *testing.T) {
+	chain, err := ToolchainFor("go")
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Named, not pinned: the mesh answers with the copy it holds, so moving compiler is a build
+	// rather than an edit to this source (ADR 0044, 0142).
+	if chain.Base != "mesh-tools-go" || chain.Artifact != "build" {
+		t.Fatalf("the go toolchain is based on %s/%s", chain.Base, chain.Artifact)
+	}
+	joined := strings.Join(chain.Compile, " ")
+	// Static, because what a machine holds is a file and not a container: a binary needing a libc
+	// it did not bring is a delivery that works until a machine differs.
+	if !strings.Contains(joined, "CGO_ENABLED=0") {
+		t.Fatalf("the go toolchain does not build statically: %q", joined)
+	}
+	// Reproducible: a version comes from where a component sits, not from its linker (ADR 0142),
+	// so two builds of one commit should produce the same bytes.
+	if !strings.Contains(joined, "-trimpath") {
+		t.Fatalf("the go toolchain leaves build paths in the binary: %q", joined)
+	}
+	if chain.Unit != UnitPackage {
+		t.Fatalf("the go toolchain compiles %q, wanted a package", chain.Unit)
+	}
+}
+
+func TestEveryToolchainSaysWhatItIsPointedAt(t *testing.T) {
+	// The field exists because the compile path used to assume one language. A toolchain that says
+	// nothing would fall through to the entrypoint branch and compile a file list, which for a
+	// compiled language builds a program out of exactly those files and ignores the rest of the
+	// package — a missing symbol rather than a legible refusal.
+	for _, chain := range toolchains {
+		switch chain.Unit {
+		case UnitPackage:
+		case UnitSources:
+			if chain.SourceExt == "" {
+				t.Fatalf("%s compiles a file list and names no source extension", chain.Language)
+			}
+			if !strings.HasPrefix(chain.SourceExt, ".") {
+				t.Fatalf("%s's source extension %q is not an extension", chain.Language, chain.SourceExt)
+			}
+		default:
+			t.Fatalf("%s says it is pointed at %q, which is neither sources nor a package",
+				chain.Language, chain.Unit)
+		}
+	}
+}
+
+func TestAnEntrypointBecomesASourceInItsOwnLanguage(t *testing.T) {
+	// It used to become a `.ts` whatever the language was.
+	out := Out("build")
+	got := sourcesFor([]string{out + "/tools/index.js"}, out, ".ts")
+	if len(got) != 1 || got[0] != "tools/index.ts" {
+		t.Fatalf("a typescript entrypoint became %v", got)
+	}
+	got = sourcesFor([]string{out + "/tools/index.js"}, out, ".py")
+	if len(got) != 1 || got[0] != "tools/index.py" {
+		t.Fatalf("a python entrypoint became %v", got)
+	}
+}

internal/builder/system_stamp_test.go

@@ -0,0 +1,76 @@
+package builder
+
+import (
+	"strings"
+	"testing"
+)
+
+// A host built without knowing its system refuses every declaration before applying anything —
+// safely, totally, and with nothing reporting it. The mesh built one, delivered it, started it, and
+// it would have refused the first thing it was asked to do (novox/hq 04-ISSUES/161).
+
+func TestTheGoToolchainStampsTheArtifactsSystem(t *testing.T) {
+	chain, err := ToolchainFor("go")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if chain.SystemStamp != "main.builtFor" {
+		t.Fatalf("the go toolchain fills %q", chain.SystemStamp)
+	}
+}
+
+func TestALanguageWithNoPinnedSystemStampsNothing(t *testing.T) {
+	// Interpreted output is not pinned to a system, and a manifest declaring one for it is already
+	// refused. Nothing to fill.
+	for _, language := range []string{"typescript", "python"} {
+		chain, err := ToolchainFor(language)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if chain.SystemStamp != "" {
+			t.Fatalf("%s fills %q, and its output is not pinned to a system",
+				language, chain.SystemStamp)
+		}
+	}
+}
+
+func TestTheStampIsTheOneThingTakenFromTheArtifact(t *testing.T) {
+	// The toolchain accepts nothing else from the module — anything it could override it would be
+	// writing a Dockerfile to override. The system is the stated exception, because a compiled
+	// binary is per system and the artifact is what declares one (ADR 0142).
+	chain, err := ToolchainFor("go")
+	if err != nil {
+		t.Fatal(err)
+	}
+	joined := strings.Join(chain.Compile, " ")
+	if strings.Contains(joined, "${") || strings.Contains(joined, "%s") {
+		t.Fatalf("the compile line takes something from the module: %q", joined)
+	}
+}
+
+func TestTheLinkerIsToldOnceNotTwice(t *testing.T) {
+	// A repeated flag is not a merged one: the Go command takes the last -ldflags and drops the
+	// first. Passing the toolchain's flags and then the stamp separately produced a binary that knew
+	// its system and had lost -s -w — 12.2MB against 8.5MB, with its debug info (04-ISSUES/161).
+	chain, err := ToolchainFor("go")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, arg := range chain.Compile {
+		if arg == "-ldflags" {
+			t.Fatal("the compile line carries -ldflags, so composing one here makes two")
+		}
+	}
+	if len(chain.LinkerFlags) == 0 {
+		t.Fatal("the go toolchain passes no linker flags, so the binary keeps its debug info")
+	}
+	var stripped bool
+	for _, f := range chain.LinkerFlags {
+		if f == "-s" {
+			stripped = true
+		}
+	}
+	if !stripped {
+		t.Fatalf("the go toolchain does not strip: %v", chain.LinkerFlags)
+	}
+}

internal/builder/toolchain.go

@@ -35,8 +35,50 @@ type Toolchain struct {
 	Compile []string
 	// OutputFlag is how this compiler is told where to put its output.
 	OutputFlag string
+	// Unit is what this compiler is pointed at: UnitSources, the entrypoint files the module named,
+	// or UnitPackage, the one directory the artifact is built `from`.
+	//
+	// **The difference is the language and not the module.** A TypeScript bundle is a set of files
+	// compiled into a set of files, so what to compile is the module's entrypoints with their source
+	// extension. A Go bundle is a package compiled into one binary, and there is no per-file
+	// compilation to name — pointing `go build` at a file list builds a program out of exactly those
+	// files and ignores the rest of the package, which fails as a missing symbol rather than as a
+	// wrong instruction.
+	Unit string
+	// SourceExt is the extension an entrypoint has in the repository, for UnitSources. An entrypoint
+	// is named as it will be FOUND, inside the unpacked bundle, so the source is the same path with
+	// the output directory taken off the front and this on the end.
+	SourceExt string
+	// LinkerFlags are passed to the linker as one flag, together with the system stamp below.
+	//
+	// **Separate from Compile because a repeated flag is not a merged one.** They were in the compile
+	// line, and appending the stamp as a second `-ldflags` meant the Go command took the last and
+	// dropped the first — so the binary gained its system and lost `-s -w`, growing by half and
+	// carrying its debug info. The mistake was believing a comment rather than reading the file it
+	// produced (novox/hq 04-ISSUES/161).
+	LinkerFlags []string
+	// SystemStamp is the variable this language's linker fills with the artifact's declared system,
+	// for a language whose binaries are pinned to one at link time (novox/hq ADR 0005).
+	//
+	// **The one thing a toolchain takes from the artifact, and 0142 says why**: the target is a
+	// property of the artifact rather than of the recipe, because a compiled binary is per system
+	// and a toolchain that accepted it from the module would be accepting a build instruction. This
+	// is the narrow exception, named here rather than inferred.
+	//
+	// Empty for a language that compiles to nothing pinned. A host built without it refuses every
+	// declaration before applying anything — safely, totally, and with nothing reporting it
+	// (novox/hq 04-ISSUES/161).
+	SystemStamp string
 }
 
+// What a toolchain is pointed at.
+const (
+	// UnitSources is a list of files, derived from the module's entrypoints.
+	UnitSources = "sources"
+	// UnitPackage is the single directory the artifact is built `from`, compiled whole.
+	UnitPackage = "package"
+)
+
 // Out is where one artifact's compiled output lands, inside the module's own directory.
 //
 // **Per artifact, never per toolchain.** A module is one piece of software and may still be
@@ -71,6 +113,41 @@ var toolchains = []Toolchain{
 			"--target", "ES2022",
 		},
 		OutputFlag: "--outDir",
+		Unit:       UnitSources,
+		SourceExt:  ".ts",
+	},
+	{
+		Language: "go",
+		Base:     "mesh-tools-go",
+		Artifact: "build",
+		// **The mesh's own components, and not modules.** The warning above this list — that every
+		// language is another implementation of the contracts modules share, so adding one commits
+		// to keeping N implementations in step — does not attach here. Go is how the host, the
+		// control plane and the builder are written, and none of them is a module in that sense:
+		// the host is what APPLIES modules. So there is no SDK obligation, and the reason this
+		// entry did not exist was that nothing needed to compile the mesh itself
+		// (novox/hq ADR 0142, and 04-ISSUES/142 where that is why nothing delivers the host).
+		//
+		// Static, because what a machine ends up holding is a file rather than a container, and a
+		// binary that needs a libc it did not bring is a delivery that works until a machine
+		// differs. Trimmed of its own paths for the same reason a version comes from where it sits
+		// rather than from the linker: two builds of one commit produce the same bytes.
+		Compile: []string{
+			"env", "CGO_ENABLED=0", "GOFLAGS=-trimpath",
+			"go", "build",
+		},
+		// Stripped of symbols and debug info: what a machine holds is a file it runs, not one it
+		// debugs, and the difference measured 12.2MB against 8.5MB.
+		LinkerFlags: []string{"-s", "-w"},
+		OutputFlag:  "-o",
+		// Pointed at the package the artifact is built `from`, compiled whole. Go writes the binary
+		// into the output directory, named after the package — so the bundle a machine unpacks is a
+		// directory holding one executable, which is what the delivery mechanism expects
+		// (novox/hq ADR 0141).
+		Unit: UnitPackage,
+		// The mesh's own Go components read the system they were built for from this variable, and
+		// refuse to touch a machine without one.
+		SystemStamp: "main.builtFor",
 	},
 	{
 		Language: "python",
@@ -82,6 +159,8 @@ var toolchains = []Toolchain{
 		// each actually does.
 		Compile:    []string{"python", "-m", "pip", "install", "--no-compile", "--target"},
 		OutputFlag: "",
+		Unit:       UnitSources,
+		SourceExt:  ".py",
 	},
 }
 

internal/catalogue/build.go

@@ -94,12 +94,43 @@ func (m Manifest) Resolve(built []Built) (Manifest, error) {
 				m.Module, r["id"], named)
 		case ArtifactImage, ArtifactUpstream:
 			filled["image"] = artifact.Reference
+			// An image is not unpacked anywhere, so it has no directory to be named for its
+			// version and `${version}` has nothing to mean. Refused rather than left as literal
+			// text in a path, which is how it would reach a machine and be created as a directory
+			// called `${version}`.
+			for key, value := range filled {
+				if text, isText := value.(string); isText && strings.Contains(text, versionRef) {
+					return Manifest{}, fmt.Errorf(
+						"%s: %v says %s in %q, and %q is an image — an image is not unpacked, so "+
+							"it has no versioned place. %s is for an archive or a bundle",
+						m.Module, r["id"], versionRef, key, named, versionRef)
+				}
+			}
 		case ArtifactArchive, ArtifactBundle:
 			// The same on the wire: both are bytes fetched by digest and unpacked. They differ in
 			// how they were made — one packed as it stood, the other compiled first — and a
 			// machine has no reason to care which.
 			filled["source"] = artifact.Reference
 			filled["digest"] = artifact.Digest
+			// **And `${version}`, so a resource can name a place that is this build's alone**
+			// (novox/hq ADR 0141, 04-ISSUES/142). A component is unpacked into a directory named
+			// for its version so it can read its own version from its path — and until this,
+			// nothing could compose that path: an archive named a fixed one in the manifest and
+			// nothing interpolated the build into it, so nothing could ask for
+			// `…/versions/<version>/` and every machine took a hand-placed fallback.
+			//
+			// The version is the artifact's own digest, short. Not the commit: two builds of one
+			// commit are meant to be the same bytes (the toolchains are `-trimpath` for this), and
+			// a content-addressed version means an unchanged build resolves to the path it already
+			// had — so re-composing a declaration moves nothing, where a commit would move the
+			// path of an identical binary and recreate everything that reads it.
+			for key, value := range filled {
+				text, isText := value.(string)
+				if !isText || !strings.Contains(text, versionRef) {
+					continue
+				}
+				filled[key] = strings.ReplaceAll(text, versionRef, versionOf(artifact.Digest))
+			}
 		default:
 			return Manifest{}, fmt.Errorf("%s: %q is a %q, and an artifact is %q, %q, %q or %q",
 				m.Module, named, artifact.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream,
@@ -142,7 +173,14 @@ func (b *Build) problems(module string) []string {
 		// is which compiler — because the mesh chooses that, and cannot choose for a module that
 		// has not said.
 		if a.Kind == ArtifactBundle || a.Kind == ArtifactPackage {
-			if a.From != "" {
+			// **Except for a language that compiles to a binary, where it names which one**
+			// (novox/hq 04-ISSUES/142). A bundle in an interpreted language is the module's own
+			// directory compiled whole, and naming a source would be describing its own build. A
+			// repository written in a compiled language holds several commands — the host and its
+			// bootstrap live in one, and the mesh needs the host — and "the module's own directory"
+			// is then not a package at all. So the compiled case may say which package, and says
+			// the module root by saying nothing.
+			if a.From != "" && !compilesToABinary(a.Language) {
 				problems = append(problems, fmt.Sprintf(
 					"%s: %q is a bundle and names what it is built from (%q). A bundle is built "+
 						"from the module's own directory; what it says is the language",
@@ -252,3 +290,28 @@ func compilesToABinary(language string) bool {
 		return false
 	}
 }
+
+// versionRef is how a resource names the version of the artifact it uses: ${version}.
+//
+// No artifact name in it, because the resource already says which artifact it is for — a second
+// name would be a second thing to keep in step with the first.
+const versionRef = "${version}"
+
+// versionOf is an artifact's version as a path names it: its digest, short.
+//
+// **Content-addressed on purpose.** The alternative is the commit a build came from, and two builds
+// of one commit are meant to produce the same bytes — every toolchain here is `-trimpath` for that
+// reason. A commit-named path would move for an identical binary, and everything reading that path
+// would be recreated for a change that is not one. A digest-named path moves exactly when the bytes
+// do.
+//
+// Twelve hex characters: enough that two of this mesh's builds will not collide, short enough to
+// read in a path and in a journal line. The `sha256:` prefix goes, because a directory name carrying
+// a colon is a directory name people quote wrong.
+func versionOf(digest string) string {
+	hex := strings.TrimPrefix(strings.TrimSpace(digest), "sha256:")
+	if len(hex) > 12 {
+		return hex[:12]
+	}
+	return hex
+}

internal/catalogue/catalogue_check_test.go

@@ -0,0 +1,47 @@
+package catalogue
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// TestEveryCatalogueManifestParses runs the real catalogue through the real gate.
+//
+// Not a fixture: the point is whether the manifests as written are accepted by the control plane that
+// will read them, and a copy of one manifest proves nothing about the other seventy-one.
+func TestEveryCatalogueManifestParses(t *testing.T) {
+	root := os.Getenv("MESH_CATALOGUE")
+	if root == "" {
+		t.Skip("set MESH_CATALOGUE to a catalogue checkout to run this")
+	}
+	found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
+	if err != nil || len(found) == 0 {
+		t.Fatalf("no manifests under %s: %v", root, err)
+	}
+	named, routed := 0, 0
+	for _, p := range found {
+		raw, err := os.ReadFile(p)
+		if err != nil {
+			t.Fatalf("%s: %v", p, err)
+		}
+		m, err := ParseManifest(raw)
+		if err != nil {
+			t.Errorf("%s: %v", filepath.Base(filepath.Dir(p)), err)
+			continue
+		}
+		for _, l := range m.Listens {
+			if l.Name != "" {
+				named++
+			}
+		}
+		for port := range RoutedPorts(m) {
+			_ = port
+			routed++
+		}
+	}
+	t.Logf("%d manifests, %d named endpoints, %d routed endpoints resolved", len(found), named, routed)
+	if named == 0 {
+		t.Fatal("no endpoint in the catalogue is named, so this proved nothing")
+	}
+}

internal/catalogue/catrust_manifest_test.go

@@ -0,0 +1,71 @@
+package catalogue
+
+import (
+	"os"
+	"strings"
+	"testing"
+)
+
+// **A machine trusts the mesh's authority because a module put its root there** (novox/hq ADR
+// 0147, issue 129). The module carries a shell script and a unit, and both are worthless unless
+// the mesh fills in where the authority is — which is the one thing about it the manifest cannot
+// state, because the authority's address is a fact about the mesh and not about the module.
+//
+// So what is checked here is the rendering, not the parsing: the script the machine will run
+// names the authority it was bound to, and the unit runs that script both ways. The verification
+// itself — a plain client trusting an internal name on a machine holding this, and failing on one
+// that does not — is the lab's, and cannot be had here.
+func TestCaTrustRendersTheAuthorityItWasBoundTo(t *testing.T) {
+	raw, err := os.ReadFile("../../../mesh-catalog/modules/ca-trust/module.json")
+	if err != nil {
+		t.Skipf("the catalogue is not beside this checkout: %v", err)
+	}
+	m, err := ParseManifest(raw)
+	if err != nil {
+		t.Fatalf("the trust module does not parse:\n%v", err)
+	}
+
+	r := Resolution{
+		Node:    "workstation",
+		Modules: []Manifest{m},
+		Needs: []Needed{{
+			Name: "internal-acme-ca", From: "anchor", At: "anchor.internal", For: "ca-trust",
+			Serves: map[string]any{
+				"port": float64(9000), "path": "/acme/acme/directory", "roots": "/roots.pem",
+			},
+		}},
+	}
+	out, err := r.Declaration(Rendering{})
+	if err != nil {
+		t.Fatalf("the trust module could not be composed for a machine: %v", err)
+	}
+
+	script := fileNamed(out, "ca-trust.anchor")
+	if script == nil {
+		t.Fatalf("nothing writes the script the unit runs: %v", out)
+	}
+	body, _ := script["content"].(string)
+	if !strings.Contains(body, "https://anchor.internal:9000/roots.pem") {
+		t.Errorf("the script does not fetch from the authority it was bound to:\n%s", body)
+	}
+	if script["mode"] != "0755" {
+		t.Errorf("the script is written %v, which systemd cannot execute", script["mode"])
+	}
+
+	unit := fileNamed(out, "ca-trust.unit")
+	if unit == nil {
+		t.Fatalf("no unit: %v", out)
+	}
+	text, _ := unit["content"].(string)
+	// Both halves. A unit that only installs the anchor leaves a machine trusting an authority
+	// nobody assigned it to any more, which is the half issue 129 asked for by name.
+	for _, want := range []string{
+		"ExecStart=" + script["path"].(string) + " install",
+		"ExecStop=" + script["path"].(string) + " remove",
+		"RemainAfterExit=yes",
+	} {
+		if !strings.Contains(text, want) {
+			t.Errorf("the unit does not say %q:\n%s", want, text)
+		}
+	}
+}

internal/catalogue/declaration.go

@@ -618,17 +618,15 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
 		// merging earlier would throw away the files it still needs.
 		resources = append(append([]map[string]any{}, first...), resources...)
 
-		// Every container is given the mesh's names. Not a choice a module makes: a module that
+		// No container is given the mesh's names (novox/hq ADR 0148). It used to be: every
-		// listed them would go stale the day a machine joins, and one that did not would be a
+		// container got the whole roster as `--add-host` entries at creation, and a name that
-		// module whose containers cannot reach anything by name.
+		// moved afterwards was wrong inside it for as long as it ran (issues 109, 135) — and once
-		//
+		// the roster was made part of a container's identity so that could be caught, one name
-		// A container that was given names of its own keeps them and gets the mesh's beside them:
+		// moving anywhere replaced every container in the mesh (issue 151). A container resolves a
-		// the mesh does not know what else a workload needs to reach, and taking something away
+		// mesh name through its machine's resolver at the moment it asks, which the runtime is
-		// to add something is not what "also" means.
+		// told once per machine, as a file, by the resolver's own module. The names a module
-		if len(with.Names) > 0 {
+		// declares for itself are its own and stay exactly as written: they are part of what the
-			resources = withMeshNames(resources, with.Names)
+		// module is, and the mesh does not know what they mean.
-		}
-
 		// What this module may name from inside one of its own files. Gathered once per module
 		// rather than per file, because it is a fact about the module.
 		sealed, err := sealedFor(m, r.Needs, with)
@@ -1098,7 +1096,12 @@ func (r Resolution) contributions(settings SettingsBy, grants []Grant,
 			if err != nil {
 				return nil, fmt.Errorf("%s contributing to %s: %w", m.Module, to, err)
 			}
-			composeName(values, r.PublicDomain, r.At, reaches, endpointPorts(m))
+			blocks, err := Endpoints(m, settings[m.Module])
+			if err != nil {
+				return nil, fmt.Errorf("%s contributing to %s: %w", m.Module, to, err)
+			}
+			portOfEndpoint(values, endpointPorts(m))
+			composeName(values, r.PublicDomain, servingAt(r, to), reaches, endpointPorts(m), blocks)
 			out[to] = append(out[to], Contribution{From: m.Module, Values: values})
 		}
 		// Several contributions to one requirement (ADR 0094's sibling for `contributes`): an
@@ -1116,7 +1119,12 @@ func (r Resolution) contributions(settings SettingsBy, grants []Grant,
 				if err != nil {
 					return nil, fmt.Errorf("%s contributing %s to %s: %w", m.Module, local, to, err)
 				}
-				composeName(values, r.PublicDomain, r.At, reaches, endpointPorts(m))
+				blocks, err := Endpoints(m, settings[m.Module])
+				if err != nil {
+					return nil, fmt.Errorf("%s contributing %s to %s: %w", m.Module, local, to, err)
+				}
+				portOfEndpoint(values, endpointPorts(m))
+				composeName(values, r.PublicDomain, servingAt(r, to), reaches, endpointPorts(m), blocks)
 				out[to] = append(out[to], Contribution{From: m.Module, Values: values})
 			}
 		}
@@ -1148,10 +1156,19 @@ func (r Resolution) contributions(settings SettingsBy, grants []Grant,
 // no public domain composes nothing — there is nothing to join it to — which reads downstream as a
 // route that named no host, the same as it would have before this existed.
 func composeName(values map[string]any, publicDomain, internalDomain string, reaches map[int]string,
-	ports map[string]int) {
+	ports map[string]int, blocks map[string]Endpoint) {
 	if values == nil {
 		return
 	}
+	// **The subdomain an assignment gave this endpoint**, before the name is joined (novox/hq ADR
+	// 0138). The module contributes a label because it names its own parts; an assignment may say a
+	// different one, because where a thing lives under a domain is the operator's to choose and used
+	// to require editing the module to change.
+	if name, ok := values[RouteEndpoint].(string); ok {
+		if ep, said := blocks[strings.TrimSpace(name)]; said && ep.Label != "" {
+			values["label"] = ep.Label
+		}
+	}
 	// **How far the endpoint this route serves reaches decides which names exist** (novox/hq ADR
 	// 0138). Both were composed whenever the node had both domains, so every routed module got a
 	// public name and an internal one whether anybody wanted them or not — and a certificate for
@@ -1207,6 +1224,24 @@ func composeName(values map[string]any, publicDomain, internalDomain string, rea
 	}
 }
 
+// servingAt is the private-network name of the node a contribution to `to` arrives at: the
+// provider's, when the provision is answered elsewhere, and this machine's own when it is answered
+// here or not yet settled.
+//
+// A route's internal name is composed under it (novox/hq ADR 0151, issue 139). `<label>.<node>.internal`
+// is answered by every machine's resolver as *anything under that node's name goes to that node* —
+// so the node in the name has to be the one whose proxy answers, or the name sends a client to a
+// machine with nothing listening while the public name, published at the serving node's address,
+// works. Where the proxy runs beside the module the two are the same machine and nothing changes.
+func servingAt(r Resolution, to string) string {
+	for _, n := range r.Needs {
+		if n.Name == to && n.At != "" {
+			return n.At
+		}
+	}
+	return r.At
+}
+
 // receivedFile is the file a provider is given its consumers' contributions in.
 func receivedFile(requirement, path string, given []Contribution) (map[string]any, error) {
 	if given == nil {
@@ -1463,43 +1498,6 @@ func (r Resolution) servedOnThisMachine(provision string, with Rendering) (map[s
 	return nil, false, nil
 }
 
-// withMeshNames gives every container in a set the mesh's names.
-//
-// Copied rather than edited in place: these maps come from a module's manifest, and mutating one
-// would change what the catalogue holds for every other machine running that module.
-//
-// A host-network container gets the names too. It was once skipped, on the belief that it "shares
-// the machine's hosts file already" — but it does not: `docker run --network host` still gives the
-// container its own /etc/hosts (localhost and its own id only), so every `<node>.internal` name the
-// mesh wrote for the machine is invisible inside it, and a client that dials one gets EAI_AGAIN. The
-// remedy is the same `--add-host` every other container gets — the runtime accepts it with
-// `--network host` (verified), and without it a host-network consumer cannot reach a provider by the
-// `.internal` address the mesh hands it as `${bound:...:at}`.
-func withMeshNames(resources []map[string]any, names map[string]string) []map[string]any {
-	out := make([]map[string]any, 0, len(resources))
-	for _, r := range resources {
-		if r["type"] != "container" {
-			out = append(out, r)
-			continue
-		}
-
-		copied := map[string]any{}
-		for k, v := range r {
-			copied[k] = v
-		}
-		var given []any
-		if already, ok := copied["hosts"].([]any); ok {
-			given = append(given, already...)
-		}
-		for _, name := range sortedKeys(names) {
-			given = append(given, name+":"+names[name])
-		}
-		copied["hosts"] = given
-		out = append(out, copied)
-	}
-	return out
-}
-
 // pinned refuses an image that is not really pinned, on its way to a machine.
 //
 // **Here and not at parse** (novox/hq 04-ISSUES/025). A manifest in a repository names artifacts
@@ -1820,3 +1818,32 @@ func endpointPorts(m Manifest) map[string]int {
 	}
 	return out
 }
+
+// portOfEndpoint fills in the port of the endpoint a contribution names, in place.
+//
+// **A contribution that names an endpoint must still carry that endpoint's port**, because everything
+// downstream reads the port: the provider is told where to reach the consumer, and the machine-side
+// redirection that turns a declared port into the number the machine published is keyed on it
+// (atMachinePort). A route that named only its endpoint left the proxy with no port at all, and a
+// proxy with no port has nothing to dial.
+//
+// Found before it shipped and after the catalogue had already been changed to name endpoints — the
+// manifests were merged and the mesh had not yet picked them up, so nothing was broken yet. The
+// declared port, not the machine one: the redirection happens later and is keyed on the declared
+// number, so filling in the machine port here would be redirected a second time or not at all.
+func portOfEndpoint(values map[string]any, ports map[string]int) {
+	if values == nil {
+		return
+	}
+	if _, already := values["port"]; already {
+		// A route that says both is its own answer; the older shape repeated the port and is still read.
+		return
+	}
+	name, ok := values[RouteEndpoint].(string)
+	if !ok {
+		return
+	}
+	if port, found := ports[strings.TrimSpace(name)]; found {
+		values["port"] = port
+	}
+}

internal/catalogue/endpoint_test.go

@@ -19,6 +19,12 @@ func aMediaServer() Manifest {
 		Contributes: map[string]map[string]any{
 			"route": {"label": "media", RouteEndpoint: "web"},
 		},
+		// Both endpoints are published by its container, which is what lets a machine port be given
+		// for either: the mesh moves a port the module publishes, never one it merely listens on.
+		Resources: []map[string]any{
+			{"id": "server", "type": "container", "name": "media",
+				"ports": []any{"80", "32400"}},
+		},
 	}
 }
 
@@ -139,3 +145,61 @@ func TestAnUnnamedEndpointIsStillValid(t *testing.T) {
 		t.Fatalf("a module with no route was refused: %v", got)
 	}
 }
+
+// **A route that names an endpoint still carries that endpoint's port.**
+//
+// Everything downstream reads the port: the provider is told where to reach the consumer, and the
+// redirection that turns a declared port into the number the machine published is keyed on it. A route
+// naming only its endpoint left the proxy with no port, and a proxy with no port has nothing to dial.
+//
+// Caught after the catalogue had already been changed to name endpoints, and before the mesh picked
+// those manifests up — which is the only reason nothing broke.
+func TestARouteNamingAnEndpointStillCarriesItsPort(t *testing.T) {
+	m := aMediaServer()
+	r := Resolution{Node: "anchor", Modules: []Manifest{m},
+		PublicDomain: "example.test", At: "anchor.internal"}
+	given, err := r.contributions(nil, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var saw bool
+	for _, c := range given["route"] {
+		saw = true
+		port, ok := asPort(c.Values["port"])
+		if !ok {
+			t.Fatalf("the route carries no port, so the proxy has nothing to dial: %v", c.Values)
+		}
+		if port != 80 {
+			t.Fatalf("the route carries port %d, want the web endpoint's 80", port)
+		}
+	}
+	if !saw {
+		t.Fatal("the module contributed no route")
+	}
+}
+
+// And the declared port, not the machine one: the redirection to where the machine published it
+// happens later and is keyed on the declared number, so filling the machine port in here would be
+// redirected twice or not at all.
+func TestTheEndpointsDeclaredPortIsFilledInNotTheMachineOne(t *testing.T) {
+	m := aMediaServer()
+	values := map[string]any{RouteEndpoint: "web", "label": "media"}
+	portOfEndpoint(values, endpointPorts(m))
+	if got, _ := asPort(values["port"]); got != 80 {
+		t.Fatalf("filled in port %d, want the declared 80", got)
+	}
+	// Then the ordinary redirection puts it where the machine published it.
+	moved := atMachinePort(values, m.Module, map[string]map[int]int{"media": {80: 20009}})
+	if got, _ := asPort(moved["port"]); got != 20009 {
+		t.Fatalf("after redirection the port is %d, want the machine's 20009", got)
+	}
+}
+
+// A route that repeats a port keeps it, because that is the older shape and still read.
+func TestARouteThatRepeatsItsPortKeepsIt(t *testing.T) {
+	values := map[string]any{RouteEndpoint: "web", "port": 8080}
+	portOfEndpoint(values, map[string]int{"web": 80})
+	if got, _ := asPort(values["port"]); got != 8080 {
+		t.Fatalf("the port it stated was overwritten with %d", got)
+	}
+}

internal/catalogue/endpoints_setting_test.go

@@ -0,0 +1,140 @@
+package catalogue
+
+import (
+	"strings"
+	"testing"
+)
+
+func configured(block map[string]any) SettingsBy {
+	return SettingsBy{"media": {{From: "node anchor",
+		Values: map[string]any{EndpointsSetting: block}}}}
+}
+
+// **One block per endpoint, saying all three things.** The machine port, the subdomain and the reach
+// were `ports`, the route's label and `reach`, each keyed by a port number, so configuring a module
+// with two endpoints of different shapes meant knowing which number was which.
+func TestAnEndpointsBlockSaysPortLabelAndReach(t *testing.T) {
+	m := aMediaServer()
+	// stream's reach NARROWS what the manifest says — the manifest has it from anywhere, the
+	// assignment says internal. Chosen deliberately: a reach that agrees with the manifest proves
+	// nothing about whether the block was read at all.
+	settings := configured(map[string]any{
+		"web":    map[string]any{"port": 20009, "label": "cinema", "reach": ReachBoth},
+		"stream": map[string]any{"reach": ReachInternal},
+	})
+
+	// The machine port, where the mapping is read.
+	given, err := GivenPorts(m, settings["media"])
+	if err != nil {
+		t.Fatal(err)
+	}
+	if given[80] != 20009 {
+		t.Fatalf("the web endpoint is on machine port %d, want 20009: %v", given[80], given)
+	}
+
+	// The reach, where the filter reads it.
+	r := Resolution{Node: "anchor", Modules: []Manifest{m},
+		PublicDomain: "example.test", At: "anchor.internal"}
+	rules, err := r.Rules(Rendering{Settings: settings})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, rule := range rules {
+		if rule.Port == 32400 && rule.From != FromMesh {
+			t.Fatalf("the directly-dialled endpoint is %q; the assignment narrowed it to the private "+
+				"network and the manifest's 'anywhere' should not win", rule.From)
+		}
+		if rule.Port == 80 && rule.From != FromMesh {
+			t.Fatalf("the routed endpoint's port opened to %q; the proxy is how it is reached", rule.From)
+		}
+	}
+
+	// And the subdomain, where the name is composed — the assignment's, not the module's.
+	nodes, err := r.contributions(settings, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, c := range nodes["route"] {
+		if got, _ := c.Values["name"].(string); got != "cinema.example.test" {
+			t.Fatalf("the public name is %q, want the label the assignment gave", got)
+		}
+		if got, _ := c.Values["internal-name"].(string); got != "cinema.anchor.internal" {
+			t.Fatalf("the internal name is %q, want the label the assignment gave", got)
+		}
+	}
+}
+
+// A block that says only a reach leaves the port to the mesh and the label to the module, which is the
+// ordinary case and must not require writing the other two.
+func TestABlockMaySayOnlyAReach(t *testing.T) {
+	m := aMediaServer()
+	settings := configured(map[string]any{"web": map[string]any{"reach": ReachInternal}})
+	r := Resolution{Node: "anchor", Modules: []Manifest{m},
+		PublicDomain: "example.test", At: "anchor.internal"}
+	nodes, err := r.contributions(settings, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, c := range nodes["route"] {
+		if got, _ := c.Values["name"].(string); got != "" {
+			t.Fatalf("an internal endpoint composed the public name %q", got)
+		}
+		// The module's own label, untouched.
+		if got, _ := c.Values["internal-name"].(string); got != "media.anchor.internal" {
+			t.Fatalf("the internal name is %q, want the module's own label", got)
+		}
+	}
+}
+
+// An endpoint the module does not declare reaches nothing, and the refusal says what it does declare.
+func TestConfiguringAnEndpointTheModuleLacksIsRefused(t *testing.T) {
+	_, err := Endpoints(aMediaServer(), configured(map[string]any{
+		"admin": map[string]any{"reach": ReachInternal}})["media"])
+	if err == nil || !strings.Contains(err.Error(), "does not declare") {
+		t.Fatalf("configuring an absent endpoint was accepted: %v", err)
+	}
+	if err != nil && !strings.Contains(err.Error(), "stream") {
+		t.Fatalf("the refusal does not name what the module declares: %v", err)
+	}
+}
+
+func TestAReachInABlockIsHeldToTheFourValues(t *testing.T) {
+	_, err := Endpoints(aMediaServer(), configured(map[string]any{
+		"web": map[string]any{"reach": "mesh"}})["media"])
+	if err == nil || !strings.Contains(err.Error(), "a reach is") {
+		t.Fatalf("a filter word was accepted as a reach: %v", err)
+	}
+}
+
+// **Two places giving one endpoint a machine port is the confusion this key exists to end.**
+func TestAnEndpointGivenAPortTwiceIsRefused(t *testing.T) {
+	m := aMediaServer()
+	_, err := GivenPorts(m, []Layer{{From: "node anchor", Values: map[string]any{
+		EndpointsSetting: map[string]any{"web": map[string]any{"port": 20009}},
+		PortsSetting:     map[string]any{"80": 30000},
+	}}})
+	if err == nil || !strings.Contains(err.Error(), "published once") {
+		t.Fatalf("an endpoint given two machine ports was accepted: %v", err)
+	}
+}
+
+// And the same for its reach, said once here and once through the older key.
+func TestAnEndpointWhoseReachIsAlsoExposedIsRefused(t *testing.T) {
+	_, err := Endpoints(aMediaServer(), []Layer{{From: "node anchor", Values: map[string]any{
+		EndpointsSetting: map[string]any{"web": map[string]any{"reach": ReachInternal}},
+		ExposeSetting:    map[string]any{"80": FromEverywhere},
+	}}})
+	if err == nil || !strings.Contains(err.Error(), "same thing in different words") {
+		t.Fatalf("a reach said two ways was accepted: %v", err)
+	}
+}
+
+// A module whose endpoints are unnamed cannot be configured this way, and is told so rather than
+// having a block silently reach nothing — which is every module in the catalogue today.
+func TestAModuleWithNoNamedEndpointsIsToldSo(t *testing.T) {
+	m := Manifest{Module: "media", Listens: []Listening{{Port: 80, From: FromMesh}}}
+	_, err := Endpoints(m, configured(map[string]any{"web": map[string]any{"reach": ReachBoth}})["media"])
+	if err == nil || !strings.Contains(err.Error(), "no endpoints by name") {
+		t.Fatalf("a module with no named endpoints accepted a block: %v", err)
+	}
+}

internal/catalogue/filtering.go

@@ -265,6 +265,26 @@ func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int,
 	b.WriteString("\t\tct state established,related accept\n")
 	b.WriteString("\t\tct state invalid drop\n")
 	b.WriteString("\t\tiif lo accept\n")
+	// **Anything on this machine may call anything on this machine.**
+	//
+	// Local is not a boundary this mesh draws. A service running here is callable by everything else
+	// running here, whatever form either takes — a package with a unit, a binary, a container. Whether
+	// a caller sits in a container was never meant to change the answer, and the only reason it did was
+	// that this chain asked about addresses: a caller on the machine carries the machine's address, a
+	// caller in one of its containers carries a bridge address, and a rule naming the former silently
+	// refused the latter.
+	//
+	// Measured: a module reaching its database on this machine's own name timed out for eleven hours
+	// while the machine itself could reach it, and the mesh called the machine healthy throughout
+	// (novox/hq 04-ISSUES/145).
+	//
+	// Asked by the link it arrives on rather than the address it comes from: anything that did not
+	// arrive from outside this machine, and did not arrive over the private network, is this machine's
+	// own. One rule for every service here, in place of a line per port that only ever covered the
+	// ports somebody remembered to think about.
+	if inward != "" {
+		b.WriteString(fmt.Sprintf("\t\tiifname != { %s } accept\n", inward))
+	}
 	b.WriteString("\t\ticmp type echo-request accept\n")
 	b.WriteString("\t\ticmpv6 type { echo-request, nd-neighbor-solicit, nd-neighbor-advert, nd-router-advert } accept\n")
 
@@ -536,6 +556,21 @@ const MeshWideLayer = "the mesh"
 // two mappings share a number, it is an entry one of them writes over the other's, and the reader
 // that finds the survivor disagrees with the reader that recomputes it.
 func GivenPorts(m Manifest, layers []Layer) (map[int]int, error) {
+	// An endpoint's own block may put it on a machine port, which is the same thing `ports` says about
+	// the number rather than about the endpoint (novox/hq ADR 0138). Collected first and then let the
+	// older key be read, which refuses a port said twice.
+	byName, err := Endpoints(m, layers)
+	if err != nil {
+		return nil, err
+	}
+	named := map[int]int{}
+	for name, ep := range byName {
+		if ep.Port == 0 {
+			continue
+		}
+		named[endpointPorts(m)[name]] = ep.Port
+	}
+
 	// Every name a setting may use, and the mapping it names.
 	names := map[int][]publishing{}
 	for _, p := range publishedPorts(m) {
@@ -634,6 +669,17 @@ func GivenPorts(m Manifest, layers []Layer) (map[int]int, error) {
 			out[key], by[key] = at, port
 		}
 	}
+	// And what the endpoints' own blocks put them on. Refused rather than merged where both keys name
+	// one endpoint: two places giving a port is the confusion this key exists to end.
+	for wanted, at := range named {
+		if was, twice := out[wanted]; twice && was != at {
+			return nil, fmt.Errorf(
+				"%s puts its port %d on %d through %s and on %d through %s — one endpoint, two "+
+					"machine ports, and it is published once. Keep the endpoint's own block",
+				m.Module, wanted, at, EndpointsSetting, was, PortsSetting)
+		}
+		out[wanted] = at
+	}
 	if len(out) == 0 {
 		return nil, nil
 	}
@@ -821,6 +867,20 @@ func Reaches(m Manifest, layers []Layer) (map[int]string, error) {
 	}
 
 	out := map[int]string{}
+	// What an endpoint's own block says, which is the same statement in the shape that names the
+	// endpoint rather than its port (novox/hq ADR 0138). Read first so the older key, which says less,
+	// cannot quietly win over the newer one that says more.
+	blocks, err := Endpoints(m, layers)
+	if err != nil {
+		return nil, err
+	}
+	declared := endpointPorts(m)
+	for name, ep := range blocks {
+		if ep.Reach == "" {
+			continue
+		}
+		out[declared[name]] = ep.Reach
+	}
 	for _, layer := range layers {
 		raw, ok := layer.Values[ReachSetting]
 		if !ok {
@@ -896,3 +956,123 @@ func RouteProblems(m Manifest) []string {
 	}
 	return problems
 }
+
+// EndpointsSetting is the settings key that configures a module's endpoints by name, per node
+// (novox/hq ADR 0138):
+//
+//	{"endpoints": {"web":    {"port": 20009, "label": "media", "reach": "both"},
+//	               "stream": {"reach": "public"}}}
+//
+// **One block per endpoint, instead of three keys joined by a number.** Which machine port it lands
+// on, the subdomain a proxy serves it under, and how far it reaches are the three things an operator
+// says when a module is assigned, and they were said in `ports`, in the route's label and in `reach`,
+// each keyed by the port. A module with two endpoints of different shapes — a web surface behind the
+// proxy and a protocol port clients dial directly — could only be configured by a reader who knew
+// which number was which.
+//
+// Every field is optional. A block that says only a reach leaves the port to the mesh and the label to
+// the module, which is the ordinary case.
+const EndpointsSetting = "endpoints"
+
+// Endpoint is what an assignment says about one of a module's endpoints.
+type Endpoint struct {
+	// Port is the machine-side port it is published on. Zero means the mesh assigns one, which it
+	// does anyway — a fixed port is the module's claim and is honoured without being said here.
+	Port int
+	// Label is the subdomain a proxy serves it under, overriding the one the module contributes.
+	Label string
+	// Reach is how far it reaches: machine, internal, public or both.
+	Reach string
+}
+
+// Endpoints reads a module's per-node endpoint configuration, by endpoint name.
+//
+// It refuses a name the module does not declare — the setting would reach nothing — and a reach that
+// is not one of the four. It also refuses an endpoint whose port or reach is said twice, once here and
+// once through the older key: two places saying the same thing is what this key exists to end, and
+// letting both stand would mean the mesh followed whichever it read last.
+func Endpoints(m Manifest, layers []Layer) (map[string]Endpoint, error) {
+	declared := endpointPorts(m)
+	exposed, err := Exposure(m, layers)
+	if err != nil {
+		return nil, err
+	}
+
+	out := map[string]Endpoint{}
+	for _, layer := range layers {
+		raw, ok := layer.Values[EndpointsSetting]
+		if !ok {
+			continue
+		}
+		blocks, ok := raw.(map[string]any)
+		if !ok {
+			return nil, fmt.Errorf("%s: %s is a { endpoint: { … } } map, and %q set it to something else",
+				m.Module, EndpointsSetting, layer.From)
+		}
+		for name, body := range blocks {
+			port, known := declared[name]
+			if !known {
+				return nil, fmt.Errorf(
+					"%s configures the endpoint %q, which it does not declare — the setting reaches "+
+						"nothing. It declares %s", m.Module, name, spokenEndpoints(m))
+			}
+			values, ok := body.(map[string]any)
+			if !ok {
+				return nil, fmt.Errorf("%s: the endpoint %q is configured with something that is not a "+
+					"block of settings", m.Module, name)
+			}
+			ep := out[name]
+			if reach, said := values["reach"]; said {
+				text, ok := reach.(string)
+				if !ok || !slices.Contains(reaches, text) {
+					return nil, fmt.Errorf("%s says the endpoint %q reaches %v; a reach is %s",
+						m.Module, name, reach, strings.Join(reaches, ", "))
+				}
+				if _, also := exposed[port]; also {
+					return nil, fmt.Errorf(
+						"%s says how far %q reaches and also exposes port %d. They say the same thing "+
+							"in different words; keep the endpoint's own block",
+						m.Module, name, port)
+				}
+				ep.Reach = text
+			}
+			if at, said := values["port"]; said {
+				machine, ok := asPort(at)
+				if !ok {
+					return nil, fmt.Errorf("%s puts the endpoint %q on %v, which is not a port",
+						m.Module, name, at)
+				}
+				ep.Port = machine
+			}
+			if label, said := values["label"]; said {
+				text, ok := label.(string)
+				if !ok || strings.TrimSpace(text) == "" {
+					return nil, fmt.Errorf("%s gives the endpoint %q a label that is not a name: %v",
+						m.Module, name, label)
+				}
+				ep.Label = strings.TrimSpace(text)
+			}
+			out[name] = ep
+		}
+	}
+	if len(out) == 0 {
+		return nil, nil
+	}
+	return out, nil
+}
+
+// spokenEndpoints is what a module's endpoints are called, as a refusal lists them — so a reader who
+// named one wrongly is one edit from right, and a module that has named none is told so.
+func spokenEndpoints(m Manifest) string {
+	names := make([]string, 0, len(m.Listens))
+	for _, l := range m.Listens {
+		if name := strings.TrimSpace(l.Name); name != "" {
+			names = append(names, name)
+		}
+	}
+	if len(names) == 0 {
+		return "no endpoints by name"
+	}
+	sort.Strings(names)
+	return strings.Join(names, ", ")
+}

internal/catalogue/filtering_test.go

@@ -804,3 +804,86 @@ func TestSSHIsNeverLeftWithoutARule(t *testing.T) {
 		t.Fatalf("a machine with no mesh addresses has no ssh rule, so adopting it locks it:\n%s", nft)
 	}
 }
+
+// chainBody is one chain's own lines, so an assertion cannot be satisfied by an identical line in
+// another chain.
+//
+// **Written because that happened.** The rule letting this machine's own callers through appears in the
+// input chain and, in the same words, in the forward chain. A test asserting on the whole rendered file
+// passed with the input chain's copy deleted — it was reading the forward chain's. ADR 0137's own tests
+// say to assert per chain body for exactly this reason, and this file was not doing it.
+func chainBody(t *testing.T, nft, chain string) string {
+	t.Helper()
+	open := "\tchain " + chain + " {"
+	i := strings.Index(nft, open)
+	if i < 0 {
+		t.Fatalf("no chain %q in:\n%s", chain, nft)
+	}
+	rest := nft[i+len(open):]
+	j := strings.Index(rest, "\n\t}")
+	if j < 0 {
+		t.Fatalf("chain %q does not close in:\n%s", chain, nft)
+	}
+	return rest[:j]
+}
+
+// **Anything on this machine may call anything on this machine.**
+//
+// Local is not a boundary this mesh draws, and whether a caller sits in a container was never meant to
+// change the answer. It did, because the chain asked about addresses: a caller on the machine carries
+// the machine's address and a caller in one of its containers carries a bridge address, so a rule
+// naming the machines' own addresses silently refused every container on them.
+//
+// Measured: a module reaching its database on its own machine's name timed out for eleven hours while
+// the machine itself could reach it (novox/hq 04-ISSUES/145).
+func TestAnythingOnThisMachineMayCallAnythingOnIt(t *testing.T) {
+	nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
+		{Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}},
+		{Module: "private", Listens: []Listening{{Port: 9999, From: FromMachine}}},
+	}}, nil), []string{"10.10.0.1", "10.10.0.2"}, false, nil, []string{"eth0"}, "mesh0")
+
+	// In the INPUT chain, which is where a call to a service on this machine arrives. The forward
+	// chain carries the same line in the same words, so asserting on the whole file proves nothing.
+	input := chainBody(t, nft, "input")
+	if !strings.Contains(input, `iifname != { "eth0", "mesh0" } accept`) {
+		t.Fatalf("a caller on this machine cannot reach a service on it:\n%s", input)
+	}
+	// One rule, for every service here — not a line per port that only covers the ports somebody
+	// remembered to think about.
+	if strings.Contains(input, `iifname != { "eth0", "mesh0" } tcp dport 5432`) {
+		t.Fatalf("the local allowance is still written per port:\n%s", input)
+	}
+	// And the private network still reaches what is exposed to it, which is a different question.
+	if !strings.Contains(input, "ip saddr { 10.10.0.1, 10.10.0.2 } tcp dport 5432 accept") {
+		t.Fatalf("the private network no longer reaches a service exposed to it:\n%s", input)
+	}
+}
+
+// The three reaches, as three lines. This is the whole of what the filter says about who may call what.
+func TestTheThreeReachesAreThreeLines(t *testing.T) {
+	nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
+		{Module: "internal-only", Listens: []Listening{{Port: 5432, From: FromMesh}}},
+		{Module: "public", Listens: []Listening{{Port: 443, From: FromEverywhere}}},
+	}}, nil), []string{"10.10.0.1"}, false, nil, []string{"eth0"}, "mesh0")
+
+	input := chainBody(t, nft, "input")
+	for what, want := range map[string]string{
+		"on this machine":          `iifname != { "eth0", "mesh0" } accept`,
+		"over the private network": "ip saddr { 10.10.0.1 } tcp dport 5432 accept",
+		"from anywhere":            "tcp dport 443 accept",
+	} {
+		if !strings.Contains(input, want) {
+			t.Fatalf("a caller %s cannot reach what is exposed to it (%q):\n%s", what, want, input)
+		}
+	}
+}
+
+// A port open to everything needs no such line — it is already open to a guest.
+func TestAPublicPortNeedsNoGuestLine(t *testing.T) {
+	nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
+		{Module: "web", Listens: []Listening{{Port: 443, From: FromEverywhere}}},
+	}}, nil), []string{"10.10.0.1"}, false, nil, []string{"eth0"}, "mesh0")
+	if strings.Count(nft, `iifname != { "eth0", "mesh0" } tcp dport 443`) != 0 {
+		t.Fatalf("a public port was given a guest line it does not need:\n%s", nft)
+	}
+}

internal/catalogue/go_bundle_test.go

@@ -0,0 +1,37 @@
+package catalogue
+
+import "testing"
+
+// A bundle is the module's own directory compiled whole, and naming a source would be describing its
+// own build. That holds for an interpreted language and cannot hold for a compiled one: a repository
+// written in Go carries several commands — the host and its bootstrap live in one — and "the module's
+// own directory" is then not a package at all (novox/hq 04-ISSUES/142).
+
+func TestAGoBundleMayNameItsCommand(t *testing.T) {
+	b := &Build{Artifacts: []Artifact{{
+		Name: "host-arch", Kind: ArtifactBundle, Language: "go", System: "arch",
+		From: "cmd/mesh-host",
+	}}}
+	if p := b.problems("mesh-host"); len(p) != 0 {
+		t.Fatalf("a go bundle naming its command was refused: %v", p)
+	}
+}
+
+func TestAnInterpretedBundleStillMayNotNameASource(t *testing.T) {
+	b := &Build{Artifacts: []Artifact{{
+		Name: "tools", Kind: ArtifactBundle, Language: "typescript", From: "src",
+	}}}
+	p := b.problems("something")
+	if len(p) == 0 {
+		t.Fatal("an interpreted bundle naming what it is built from was accepted")
+	}
+}
+
+func TestACompiledBundleStillMustSayItsSystem(t *testing.T) {
+	b := &Build{Artifacts: []Artifact{{
+		Name: "host", Kind: ArtifactBundle, Language: "go", From: "cmd/mesh-host",
+	}}}
+	if len(b.problems("mesh-host")) == 0 {
+		t.Fatal("a compiled bundle with no system was accepted")
+	}
+}

internal/catalogue/holdings_test.go

@@ -158,3 +158,16 @@ func TestAStoreRowWithoutAProtocolKeepsTheCompiledOne(t *testing.T) {
 		t.Fatalf("the store's own columns were not kept: %+v", got)
 	}
 }
+
+func TestARefusalWithNothingOnRecordNamesTheHandover(t *testing.T) {
+	busSeatDelivering(t, "mesh-bus")
+	elsewhere := []Held{{Claim: "mesh-broker", Scope: ScopeMesh, Node: "anchor", Module: "old-broker"}}
+
+	_, problems := checkClaims([]Manifest{newBroker()}, Node{Name: "laptop"}, elsewhere, nil)
+	if len(problems) != 1 {
+		t.Fatalf("two derived claimants across machines were not refused: %v", problems)
+	}
+	if !strings.Contains(problems[0], "`seat mesh-broker --to anchor/old-broker`") {
+		t.Fatalf("the refusal does not name the handover that records the holder: %s", problems[0])
+	}
+}

internal/catalogue/manifest.go

@@ -597,6 +597,16 @@ type Artifact struct {
 	// Empty for every other kind, which do not compile.
 	Language string `json:"language,omitempty"`
 
+	// Binary is what the compiled executable is called, for a bundle in a language that compiles to
+	// one. Empty means the package's own name, which is what a compiler does by default.
+	//
+	// **Because the name a machine runs it by is not always the name of the package that built it.**
+	// The host's command is `cmd/mesh-host` and every machine runs it as `nox-mesh-host` — the path
+	// it is installed at, the name in its unit, and the name its launcher looks for inside a
+	// delivered version. A bundle that carried the package's name was delivered correctly, reported
+	// success, and was invisible to the launcher (novox/hq 04-ISSUES/142).
+	Binary string `json:"binary,omitempty"`
+
 	// Entrypoints are the compiled files a tool host should load from this module, relative to the
 	// bundle's root.
 	//

internal/catalogue/names_test.go

@@ -1,6 +1,7 @@
 package catalogue
 
 import (
+	"reflect"
 	"strings"
 	"testing"
 )
@@ -30,36 +31,38 @@ func namesOf(r map[string]any) []string {
 	return out
 }
 
-// A container does not inherit the machine's names, so the mesh gives them to it.
+// No mesh name is written into a container (novox/hq ADR 0148). It resolves them through its
+// machine's resolver at the moment it asks, so a name that moves is answered differently by the
+// next lookup, in every container, with nothing recreated.
 //
-// It gets its own hosts file holding only its own hostname — every internal name the mesh wrote
+// Checked the way the record says: the declaration a container gets does not move when the mesh's
-// for the machine is invisible to what the machine runs. A database client on one node could not
+// roster does. A roster with one machine and a roster with three produce the same container, byte
-// resolve another node, on a mesh where both names were correct and present on both machines.
+// for byte, so the digest a host computes from it cannot move either — which is what stopped one
-func TestEveryContainerIsGivenTheMeshsNames(t *testing.T) {
+// name moving from replacing every container in the mesh (issue 151).
-	got := containersOf(t, Resolution{Node: "laptop", Modules: []Manifest{{
+func TestAContainerIsTheSameWhateverTheMeshsRosterSays(t *testing.T) {
-		Module: "app",
+	module := Manifest{Module: "app", Resources: []map[string]any{{"id": "web", "type": "container",
-		Resources: []map[string]any{{"id": "web", "type": "container", "name": "web",
+		"name": "web", "image": "registry.example/web@sha256:" + strings.Repeat("a", 64)}}}
-			"image": "registry.example/web@sha256:" + strings.Repeat("a", 64)}},
+	one := containersOf(t, Resolution{Node: "laptop", Modules: []Manifest{module}},
-	}}}, Rendering{Names: map[string]string{
+		Rendering{Names: map[string]string{"laptop.internal": "10.42.0.2"}})
-		"anchor.internal": "10.42.0.1", "laptop.internal": "10.42.0.2",
+	three := containersOf(t, Resolution{Node: "laptop", Modules: []Manifest{module}},
-	}})
+		Rendering{Names: map[string]string{
-	if len(got) != 1 {
+			"anchor.internal": "10.42.0.1", "laptop.internal": "10.42.0.2", "git.example.tld": "10.42.0.1",
-		t.Fatalf("expected one container, got %d", len(got))
+		}})
+	if len(one) != 1 || len(three) != 1 {
+		t.Fatalf("expected one container each, got %d and %d", len(one), len(three))
 	}
-	given := namesOf(got[0])
+	if given := namesOf(three[0]); len(given) != 0 {
-	if len(given) != 2 {
+		t.Fatalf("the mesh's names were copied into the container: %v", given)
-		t.Fatalf("the container was given %d name(s): %v", len(given), given)
 	}
-	if given[0] != "anchor.internal:10.42.0.1" {
+	if !reflect.DeepEqual(one[0], three[0]) {
-		t.Fatalf("the name is not in the form a runtime writes: %v", given)
+		t.Fatalf("the container moved with the roster:\n%v\n%v", one[0], three[0])
 	}
 }
 
-// A container that named its own keeps them and gets the mesh's beside them.
+// The names a module declares for itself are its own: part of what the module is, kept exactly as
-//
+// written, and the mesh does not know what they mean. They are the one thing in a container's
-// The mesh does not know what else a workload needs to reach, and taking something away in order
+// hosts that does move its identity, because they do not move when the mesh's roster does.
-// to add something is not what "also" means.
+func TestAContainersOwnNamesAreKeptAsWritten(t *testing.T) {
-func TestAContainersOwnNamesAreKept(t *testing.T) {
 	got := containersOf(t, Resolution{Node: "laptop", Modules: []Manifest{{
 		Module: "app",
 		Resources: []map[string]any{{"id": "web", "type": "container", "name": "web",
@@ -68,62 +71,15 @@ func TestAContainersOwnNamesAreKept(t *testing.T) {
 	}}}, Rendering{Names: map[string]string{"anchor.internal": "10.42.0.1"}})
 
 	given := namesOf(got[0])
-	if len(given) != 2 || given[0] != "something.else:203.0.113.9" {
+	if len(given) != 1 || given[0] != "something.else:203.0.113.9" {
-		t.Fatalf("the container's own names were lost: %v", given)
+		t.Fatalf("the container's own names were not kept as written: %v", given)
 	}
 }
 
-// A container on the machine's own network gets the names too — it does NOT share the machine's
+// No resource is given a `hosts` key it did not declare. A file or a service carrying one is a
-// hosts file. `docker run --network host` still gives the container its own /etc/hosts (localhost
+// declaration the host refuses outright — it takes no unknown field — so an invented key breaks
-// and its own id only), so every `<node>.internal` name the mesh wrote is invisible inside it, and a
+// the whole machine rather than one resource.
-// client that dials one gets EAI_AGAIN. It gets the same `--add-host` entries every other container
+func TestNothingIsGivenNamesItDidNotDeclare(t *testing.T) {
-// gets (the runtime accepts them with `--network host`), so a host-network consumer can reach a
-// provider by the `.internal` address the mesh hands it.
-func TestAContainerOnTheMachinesNetworkIsGivenTheNamesToo(t *testing.T) {
-	got := containersOf(t, Resolution{Node: "anchor", Modules: []Manifest{{
-		Module: "control",
-		Resources: []map[string]any{{"id": "c", "type": "container", "name": "c",
-			"image": "registry.example/c@sha256:" + strings.Repeat("a", 64), "network": "host"}},
-	}}}, Rendering{Names: map[string]string{"anchor.internal": "10.42.0.1"}})
-
-	given := namesOf(got[0])
-	if len(given) != 1 || given[0] != "anchor.internal:10.42.0.1" {
-		t.Fatalf("a host-networked container was not given the mesh's names: %v", got[0])
-	}
-}
-
-// A mesh with no private network gives nothing, rather than a name with no address behind it.
-func TestAMeshWithNoNamesGivesNone(t *testing.T) {
-	got := containersOf(t, Resolution{Node: "alone", Modules: []Manifest{{
-		Module: "app",
-		Resources: []map[string]any{{"id": "web", "type": "container", "name": "web",
-			"image": "registry.example/web@sha256:" + strings.Repeat("a", 64)}},
-	}}}, Rendering{})
-	if len(namesOf(got[0])) != 0 {
-		t.Fatalf("names were invented for a mesh that has none: %v", got[0])
-	}
-}
-
-// The catalogue's copy is not edited: these maps come from a manifest, and mutating one would
-// change what every other machine running that module is given.
-func TestGivingNamesDoesNotChangeTheCatalogue(t *testing.T) {
-	held := map[string]any{"id": "web", "type": "container", "name": "web",
-		"image": "registry.example/web@sha256:" + strings.Repeat("a", 64)}
-	module := Manifest{Module: "app", Resources: []map[string]any{held}}
-
-	for _, node := range []string{"one", "two"} {
-		containersOf(t, Resolution{Node: node, Modules: []Manifest{module}},
-			Rendering{Names: map[string]string{"anchor.internal": "10.42.0.1"}})
-	}
-	if _, changed := held["hosts"]; changed {
-		t.Fatal("the manifest the catalogue holds was edited, so every machine now carries this")
-	}
-}
-
-// Only containers. A file or a service given a `hosts` key is a declaration the host refuses
-// outright — it takes no unknown field — so getting this wrong breaks the whole machine rather
-// than one resource, and breaks it for something that was never about names.
-func TestNothingButAContainerIsGivenNames(t *testing.T) {
 	out, err := Resolution{Node: "laptop", Modules: []Manifest{{
 		Module: "app",
 		Resources: []map[string]any{
@@ -137,11 +93,8 @@ func TestNothingButAContainerIsGivenNames(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, r := range out {
-		if r["type"] == "container" {
-			continue
-		}
 		if _, given := r["hosts"]; given {
-			t.Fatalf("a %v was given names, which the host will refuse: %v", r["type"], r)
+			t.Fatalf("a %v was given names it never declared: %v", r["type"], r)
 		}
 	}
 }

internal/catalogue/port_into.go

@@ -98,8 +98,7 @@ func portInto(resource map[string]any, module string, listens []Listening, with
 
 		// **A fresh map, and only when something changes.** This map came out of the module's
 		// manifest and the resource around it is a shallow copy, so filling a value in place would
-		// change what the catalogue holds for every other machine running the module — the trap
+		// change what the catalogue holds for every other machine running the module.
-		// withMeshNames is written to avoid, one field along.
 		var filled map[string]any
 		for _, key := range named {
 			written, ok := env[key].(string)

internal/catalogue/resolve.go

@@ -707,9 +707,14 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
 			}
 			switch h.Scope {
 			case ScopeMesh:
+				// Both claim and nobody is on record, or this refusal could not have happened.
+				// The remedy is the handover that records the holder (novox/hq ADR 0131,
+				// 04-ISSUES/170), so it is named here rather than left to be found.
 				problems = append(problems, fmt.Sprintf(
-					"%s on %s claims %q, which %s on %s already holds — one per mesh",
+					"%s on %s claims %q, which %s on %s already holds — one per mesh. Nothing is "+
-					h.Module, node.Name, h.Claim, e.Module, e.Node))
+						"on record for it; `seat %s --to %s/%s` records the holder, and the other "+
+						"assignment then stands beside it, eligible and silent",
+					h.Module, node.Name, h.Claim, e.Module, e.Node, h.Claim, e.Node, e.Module))
 			case ScopeSite:
 				if node.Site != "" && node.Site == e.Site {
 					problems = append(problems, fmt.Sprintf(

internal/catalogue/resolver_manifests_test.go

@@ -48,7 +48,7 @@ func TestTheResolverForwardsToFixedUpstreamsAndNeverReadsResolvConf(t *testing.T
 	}
 	for _, want := range []string{
 		"\nno-resolv\n", "\nserver=1.1.1.1\n", "\nserver=8.8.8.8\n",
-		"\nlisten-address=127.0.0.1\n", "\ninterface=mesh0\n", "\nbind-dynamic\n",
+		"\nlisten-address=127.0.0.1\n", "\nlisten-address=${machine:address}\n", "\nbind-dynamic\n",
 		"\ndomain-needed\n", "\nbogus-priv\n",
 		"\nconf-file=" + m.Facts["node-zones"].Path + "\n",
 	} {
@@ -56,6 +56,14 @@ func TestTheResolverForwardsToFixedUpstreamsAndNeverReadsResolvConf(t *testing.T
 			t.Errorf("the resolver's configuration lacks %q:\n%s", strings.TrimSpace(want), config)
 		}
 	}
+	// By address and never by interface: dnsmasq admits a query by the interface it arrives on
+	// when told one, and a container's query to the private address arrives on the runtime's
+	// bridge — `interface=mesh0` dropped every such query, silently (novox/hq issue 110).
+	for _, line := range strings.Split(config, "\n") {
+		if strings.HasPrefix(line, "interface=") {
+			t.Errorf("the resolver answers by interface, so a container's query on a bridge is dropped: %s", line)
+		}
+	}
 	// Not .53 or .54, which systemd-resolved holds; and not .55 any more, which was a convention
 	// beside the one every machine already followed — the predecessor's resolv.conf says .1.
 	for _, taken := range []string{"127.0.0.53", "127.0.0.54", "127.0.0.55"} {
@@ -137,23 +145,39 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
 		t.Errorf("the daemon does not restart on its configuration and the machines file both: %v", service["restart-on"])
 	}
 
-	// The runtime's own file, written into (novox/hq ADR 0102) with the one key this module states.
+	// The runtime's own file, written into (novox/hq ADR 0102) with the keys this module states:
+	// where containers resolve, and that a restart keeps them running — because the runtime reads
+	// `dns` only when it starts, and the one restart that needs is the operator's (issue 110).
 	runtime := ids["dnsmasq.runtime-dns"]
 	if runtime == nil || runtime["path"] != "/etc/docker/daemon.json" || runtime["into"] != "json" {
 		t.Fatalf("the runtime's dns is not written into its file: %v", runtime)
 	}
-	var keys map[string][]string
+	var keys map[string]any
 	if err := json.Unmarshal([]byte(runtime["content"].(string)), &keys); err != nil {
 		t.Fatalf("the runtime's keys are not JSON: %v", err)
 	}
-	if len(keys) != 1 || len(keys["dns"]) != 1 || keys["dns"][0] != "10.42.0.1" {
+	dns, _ := keys["dns"].([]any)
-		t.Errorf("the runtime is pointed at %v; containers resolve at this machine's own private-network address, and nothing else is written", keys)
+	if len(keys) != 2 || len(dns) != 1 || dns[0] != "10.42.0.1" || keys["live-restore"] != true {
+		t.Errorf("the runtime is given %v; containers resolve at this machine's own private-network address, a restart keeps them, and nothing else is written", keys)
 	}
+	// The runtime is reloaded when that file changes, and never restarted: a restart stops every
+	// container on the machine (ADR 0102), and a reload is what turns live-restore on.
+	var reloaded bool
 	for _, r := range out {
-		if r["type"] == "service" && r["unit"] == "docker.service" && r["id"] != "" &&
+		if r["type"] != "service" || r["unit"] != "docker.service" {
-			strings.HasPrefix(r["id"].(string), "dnsmasq.") {
+			continue
-			t.Errorf("the resolver orders the runtime restarted or reloaded, which stops every container (ADR 0102) or does nothing for dns: %v", r)
 		}
+		if _, restarts := r["restart-on"]; restarts {
+			t.Errorf("the resolver orders the runtime restarted, which stops every container (ADR 0102): %v", r)
+		}
+		for _, on := range asStrings(r["reload-on"]) {
+			if on == "dnsmasq.runtime-dns" {
+				reloaded = true
+			}
+		}
+	}
+	if !reloaded {
+		t.Errorf("the runtime is not reloaded when its file changes, so live-restore never takes effect")
 	}
 
 	resolv := ids["resolv-conf.resolv"]

internal/catalogue/roster.go

@@ -162,6 +162,13 @@ func renderRoster(tmpl string, view rosterView) (string, error) {
 func entriesFrom(addresses, accounts map[string]string, suffix string) []rosterEntry {
 	out := make([]rosterEntry, 0, len(addresses))
 	for _, name := range sortedNames(addresses) {
+		if routed(name, suffix) {
+			// A routed name is already a full name under a public domain, and it has no mesh
+			// form: appending the suffix made `<name>.<suffix>`, which every machine's hosts file
+			// carried and nothing served (novox/hq issue 157). It is published as itself, once.
+			out = append(out, rosterEntry{Name: name, FQDN: name, Address: addresses[name], Account: accounts[name]})
+			continue
+		}
 		internal, bare := meshName(name, suffix)
 		// The account is looked up by whichever key the caller keys accounts on — the internal name
 		// or the bare one — so a template gets the right login however the maps were built.
@@ -187,6 +194,14 @@ func meshName(name, suffix string) (internal, bare string) {
 	return name + dotted, name
 }
 
+// routed says whether a name the mesh serves is a routed public name rather than a machine's: it
+// carries a domain of its own and not the mesh's suffix. A machine's name is bare (`homer`) or
+// internal (`homer.internal`); anything else with a dot in it was composed under a public domain.
+func routed(name, suffix string) bool {
+	dotted := "." + strings.TrimPrefix(suffixOr(suffix), ".")
+	return strings.Contains(name, ".") && !strings.HasSuffix(name, dotted)
+}
+
 // suffixOr is the suffix given, or the one the mesh composes names with when none was handed down.
 // The one place the default is written, so a fact and a name cannot disagree about it.
 func suffixOr(suffix string) string {

internal/catalogue/roster_test.go

@@ -258,3 +258,24 @@ func TestAHomeFactIsSkippedWhereThereIsNoAccount(t *testing.T) {
 		t.Fatalf("a home fact was placed on a machine with no operator account: %v", given)
 	}
 }
+
+// A routed name is already a full name under a public domain and has no mesh form. Appending the
+// suffix to it made `git.example.tld.internal` — carried by every machine's hosts file, served by
+// nothing, and refused by the proxy at the handshake (novox/hq issue 157). It is published as
+// itself, and only a machine has a bare name beside its full one.
+func TestARoutedNameIsPublishedAsItselfAndNotSuffixed(t *testing.T) {
+	names := map[string]string{"homer.internal": "10.42.0.1", "git.example.tld": "10.42.0.1"}
+	tmpl := RosterFile{Path: "/f", Template: "{{range .Names}}{{.FQDN}} {{.Name}}\n{{end}}"}
+	out, err := FactsInto(Manifest{Module: "a", Facts: map[string]RosterFile{"f": tmpl}},
+		Resolution{Node: "homer"}, names, map[string]string{"homer.internal": "10.42.0.1"}, nil, "internal")
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := out[0]["content"].(string)
+	if strings.Contains(got, "tld.internal") {
+		t.Fatalf("the routed name was given a suffixed alias that nothing serves:\n%s", got)
+	}
+	if !strings.Contains(got, "git.example.tld git.example.tld\n") || !strings.Contains(got, "homer.internal homer\n") {
+		t.Fatalf("the roster does not carry the routed name as itself beside the machine's two forms:\n%s", got)
+	}
+}

internal/catalogue/routing_test.go

@@ -198,37 +198,59 @@ func TestTheApexLabelComposesToTheBarePrivateAddress(t *testing.T) {
 	}
 }
 
+// novox/hq ADR 0066 propagate, by ADR 0148's means: a granted route name is published into the
+// machine's roster mapped to the node that serves it, beside the `<node>.internal` names, and the
+// machine's resolver answers it to every container. Nothing is written into the container itself —
+// a routed name that moved would otherwise be wrong inside every container until each was recreated.
 func TestARoutedNameResolvesToTheServingNode(t *testing.T) {
-	// novox/hq ADR 0066 propagate: a granted route name is published into internal resolution,
+	names := map[string]string{
-	// mapped to the node that serves it, alongside the `<node>.internal` names — so every
+		"anchor.internal": "10.42.0.1",
-	// container, and an in-mesh ACME validator, resolves a routed name to the proxy that serves it.
+		"git.example.tld": "10.42.0.1",
-	// The names map is what withMeshNames writes into every container as `--add-host`; a route name
+	}
-	// mapped to the serving node's address rides the same mechanism.
 	got := containersOf(t, Resolution{Node: "anchor", Modules: []Manifest{{
 		Module: "app",
 		Resources: []map[string]any{{"id": "web", "type": "container", "name": "web",
 			"image": "registry.example/web@sha256:" + strings.Repeat("a", 64)}},
-	}}}, Rendering{Names: map[string]string{
+	}}}, Rendering{Names: names})
-		"anchor.internal": "10.42.0.1",
-		"git.example.tld": "10.42.0.1",
-	}})
 	if len(got) != 1 {
 		t.Fatalf("expected one container, got %d", len(got))
 	}
-	given := namesOf(got[0])
+	if given := namesOf(got[0]); len(given) != 0 {
-	var sawNode, sawRoute bool
+		t.Fatalf("the routed name was copied into the container, where it would go stale: %v", given)
-	for _, h := range given {
+	}
-		if h == "anchor.internal:10.42.0.1" {
+	var sawRoute bool
-			sawNode = true
+	for _, e := range entriesFrom(names, nil, "internal") {
-		}
+		if e.Name == "git.example.tld" && e.Address == "10.42.0.1" {
-		if h == "git.example.tld:10.42.0.1" {
 			sawRoute = true
 		}
 	}
-	if !sawNode {
-		t.Fatalf("the container lost the mesh's node names: %v", given)
-	}
 	if !sawRoute {
-		t.Fatalf("the routed name was not published to the serving node: %v", given)
+		t.Fatalf("the routed name is not in the roster the machine's resolver answers from")
+	}
+}
+
+// novox/hq issue 139, ADR 0151: `<label>.<node>.internal` is answered by every machine's resolver as
+// "anything under that node's name goes to that node", so the node in a route's internal name must
+// be the one whose proxy answers it. Composed under the consumer's own name, a route served from
+// another machine got an internal name that resolved to a machine with nothing listening, while the
+// public name — published at the serving node's address — worked.
+func TestARoutesInternalNameIsComposedUnderTheNodeThatServesIt(t *testing.T) {
+	hub := proxy()
+	hub.Provides = FromAnywhere("reverse-proxy")
+	got, err := Resolve(shelf(hub, labelled("board", "git", 8080)), []string{"board"},
+		withPrivateAddress("laptop.internal"), World{Offered: map[string][]Provider{
+			"reverse-proxy": {{Node: "anchor", At: "anchor.internal", Module: "traefik"}},
+		}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Gathered the way the control plane gathers a consumer's contribution for a provider on
+	// another machine.
+	values, asks, err := got.ContributionsFrom("reverse-proxy", "board", nil)
+	if err != nil || !asks {
+		t.Fatalf("the route was not contributed: %v %v", asks, err)
+	}
+	if values["internal-name"] != "git.anchor.internal" {
+		t.Fatalf("the internal name does not say where the request arrives: %v", values)
 	}
 }

internal/catalogue/settings.go

@@ -192,6 +192,11 @@ func UnusedSettings(m Manifest, layers []Layer) []string {
 			if key == ReachSetting && len(m.Listens) > 0 {
 				continue
 			}
+			// `endpoints` configures a module's endpoints by name — the machine port, the subdomain and
+			// the reach as one block each (novox/hq ADR 0138). Validated in Endpoints, so not stray.
+			if key == EndpointsSetting && len(m.Listens) > 0 {
+				continue
+			}
 			unused = append(unused, fmt.Sprintf(
 				"%s sets %q, and %s has no file or contribution to merge it into",
 				layer.From, key, m.Module))

internal/catalogue/version_in_a_path_test.go

@@ -0,0 +1,97 @@
+package catalogue
+
+import (
+	"strings"
+	"testing"
+)
+
+// A component is unpacked into a directory named for its version, so it can read its own version from
+// its path (novox/hq ADR 0141, 0142). Until this, nothing could compose that path: an archive named a
+// fixed one and nothing interpolated the build into it, so nothing could ask for
+// `…/versions/<version>/` and every machine took a hand-placed fallback (04-ISSUES/142).
+
+const aDigest = "sha256:ad62528c47c7b4a71cf814473f5de52a061348ce9521f707b0171a10fa6b247f"
+
+func TestAnArchivePathCanNameTheBuildsOwnVersion(t *testing.T) {
+	m := Manifest{
+		Module: "mesh-host",
+		Build:  &Build{Artifacts: []Artifact{{Name: "host-arch", Kind: ArtifactBundle, Language: "go", System: "arch"}}},
+		Resources: []map[string]any{{
+			"id": "next", "type": "archive", "artifact": "host-arch",
+			"path": "/usr/lib/nox-mesh-host/versions/${version}",
+		}},
+	}
+	got, err := m.Resolve([]Built{{Name: "host-arch", Kind: ArtifactBundle,
+		Reference: "artifact-store://mesh-host/host-arch", Digest: aDigest}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	path, _ := got.Resources[0]["path"].(string)
+	if strings.Contains(path, "${version}") {
+		t.Fatalf("the version was not resolved: %q", path)
+	}
+	if path != "/usr/lib/nox-mesh-host/versions/ad62528c47c7" {
+		t.Fatalf("the path resolved to %q", path)
+	}
+	// The artifact key goes, as it does for every resolved resource: it is a build-time word and the
+	// host has never heard of it.
+	if _, still := got.Resources[0]["artifact"]; still {
+		t.Fatal("the artifact key survived resolution")
+	}
+}
+
+func TestTheVersionIsTheDigestSoAnUnchangedBuildKeepsItsPath(t *testing.T) {
+	// The alternative is the commit, and two builds of one commit are meant to be the same bytes —
+	// every toolchain here is -trimpath for that reason. A commit-named path would move for an
+	// identical binary and recreate everything reading it.
+	first := versionOf(aDigest)
+	again := versionOf(aDigest)
+	if first != again || first == "" {
+		t.Fatalf("the same bytes produced %q and %q", first, again)
+	}
+	if other := versionOf("sha256:" + strings.Repeat("b", 64)); other == first {
+		t.Fatal("different bytes produced the same version")
+	}
+	// A path is read by people and quoted by shells.
+	if strings.ContainsAny(first, ":/ ") {
+		t.Fatalf("the version is not safe in a path: %q", first)
+	}
+}
+
+func TestAnImageIsRefusedAVersionedPlace(t *testing.T) {
+	// An image is not unpacked, so it has no directory to be named for its version. Left as literal
+	// text it would reach a machine and be created as a directory called ${version}.
+	m := Manifest{
+		Module: "something",
+		Build:  &Build{Artifacts: []Artifact{{Name: "server", Kind: ArtifactImage, From: "Dockerfile"}}},
+		Resources: []map[string]any{{
+			"id": "where", "type": "directory", "artifact": "server",
+			"path": "/var/lib/something/${version}",
+		}},
+	}
+	_, err := m.Resolve([]Built{{Name: "server", Kind: ArtifactImage, Reference: "registry/x@" + aDigest}})
+	if err == nil {
+		t.Fatal("an image was given a versioned place")
+	}
+	if !strings.Contains(err.Error(), "not unpacked") {
+		t.Fatalf("the refusal does not say why: %v", err)
+	}
+}
+
+func TestAResourceWithoutAVersionReferenceIsUntouched(t *testing.T) {
+	m := Manifest{
+		Module: "mesh-host",
+		Build:  &Build{Artifacts: []Artifact{{Name: "host-arch", Kind: ArtifactBundle, Language: "go", System: "arch"}}},
+		Resources: []map[string]any{{
+			"id": "next", "type": "archive", "artifact": "host-arch", "path": "/usr/lib/fixed",
+		}},
+	}
+	got, err := m.Resolve([]Built{{Name: "host-arch", Kind: ArtifactBundle,
+		Reference: "artifact-store://mesh-host/host-arch", Digest: aDigest}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if path, _ := got.Resources[0]["path"].(string); path != "/usr/lib/fixed" {
+		t.Fatalf("a path naming no version became %q", path)
+	}
+}

internal/inventory/busrecords_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/novox/mesh-controller/internal/broker"
 	"github.com/novox/mesh-controller/internal/catalogue"
+	"golang.org/x/crypto/bcrypt"
 )
 
 // Reading the bus's user list out of the mesh's records, against a real store.
@@ -164,3 +165,63 @@ func granted(all []string, one string) bool {
 	}
 	return false
 }
+
+// **A token is an account on the bus, or it is a string nothing accepts** (novox/hq 04-ISSUES/146).
+//
+// The composed list names an enrolment user for every machine with a live token, and nothing minted
+// a credential for it — so the composer left it out as a user with no password, and every enrolment
+// since the mesh moved to this bus was refused by the server before the mesh heard of it. Nothing
+// caught it because nothing had enrolled since.
+//
+// The password cannot be minted, because it is the token's own secret: the machine will present
+// exactly that string. So this checks the two halves that make the account usable — that a row
+// exists under the name the composer asks for, and that the secret handed out is what that row
+// accepts.
+func TestIssuingATokenRecordsTheAccountItIsThePasswordOf(t *testing.T) {
+	inv, ctx := aMeshWith(t)
+	if _, err := inv.AddNode(ctx, "joiner"); err != nil {
+		t.Fatal(err)
+	}
+	issued, err := inv.IssueToken(ctx, "joiner", time.Hour)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	name := broker.Principal{Kind: broker.KindEnrolment, Node: "joiner"}.Username()
+	users, err := inv.BusUsers(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	user, has := users[name]
+	if !has {
+		t.Fatalf("no bus account for %q; the composer would leave the enrolment out and the "+
+			"machine would be refused before the mesh heard of it: %v", name, users)
+	}
+	if user.Kind != BusEnrolment || user.Node != "joiner" {
+		t.Errorf("the account is %+v, not this node's enrolment", user)
+	}
+	if err := bcrypt.CompareHashAndPassword([]byte(user.PasswordHash), []byte(issued.Secret)); err != nil {
+		t.Error("the account does not accept the secret the token carries, so presenting the " +
+			"token would be refused by the server")
+	}
+
+	// And the composition contains it, which is the thing the server reads.
+	records, err := inv.BusRecords(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	derived, err := broker.Users(records)
+	if err != nil {
+		t.Fatal(err)
+	}
+	hashes := map[string]string{}
+	for n, u := range users {
+		hashes[n] = u.PasswordHash
+	}
+	_, missing := broker.WithPasswords(derived, hashes)
+	for _, m := range missing {
+		if m == name {
+			t.Fatal("the enrolment user is composed without a password, which is a user nobody can be")
+		}
+	}
+}

internal/inventory/bususers.go

@@ -51,6 +51,40 @@ const (
 // reply, into a module's sealed environment — and the mesh keeps only the hash, so a credential is
 // never recoverable from the store. A caller that loses it must mint again, which is a rotation and
 // is meant to feel like one.
+// RecordBusPassword records a hash for a password the caller already holds.
+//
+// **For the one credential the mesh does not choose**: an enrolment token's secret is the password
+// of the user that presents it (novox/hq ADR 0004, design 25 §6), so the token cannot be given a
+// minted password — it already has one, and the machine will connect with exactly that string.
+// Everything else goes through Mint, which chooses and returns the plaintext once.
+func (i *Inventory) RecordBusPassword(ctx context.Context, u BusUser, password string) error {
+	if u.Username == "" || u.Kind == "" {
+		return errors.New("a bus user needs a username and a kind")
+	}
+	if password == "" {
+		return errors.New("a bus user needs a password")
+	}
+	hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
+	if err != nil {
+		return fmt.Errorf("cannot hash a bus password: %w", err)
+	}
+	return i.writeBusUser(ctx, u, string(hash))
+}
+
+// writeBusUser is the row, whoever chose the password.
+func (i *Inventory) writeBusUser(ctx context.Context, u BusUser, hash string) error {
+	if _, err := i.store.Pool().Exec(ctx,
+		`insert into bus_user (username, kind, node, module, password_hash)
+		 values ($1, $2, $3, $4, $5)
+		 on conflict (username) do update
+		   set kind = excluded.kind, node = excluded.node, module = excluded.module,
+		       password_hash = excluded.password_hash, minted_at = now()`,
+		u.Username, u.Kind, u.Node, u.Module, hash); err != nil {
+		return fmt.Errorf("cannot record the bus user %s: %w", u.Username, err)
+	}
+	return nil
+}
+
 func (i *Inventory) MintBusPassword(ctx context.Context, u BusUser) (string, error) {
 	if u.Username == "" || u.Kind == "" {
 		return "", errors.New("a bus user needs a username and a kind")
@@ -69,14 +103,8 @@ func (i *Inventory) MintBusPassword(ctx context.Context, u BusUser) (string, err
 		return "", fmt.Errorf("cannot hash a bus password: %w", err)
 	}
 
-	if _, err := i.store.Pool().Exec(ctx,
+	if err := i.writeBusUser(ctx, u, string(hash)); err != nil {
-		`insert into bus_user (username, kind, node, module, password_hash)
+		return "", err
-		 values ($1, $2, $3, $4, $5)
-		 on conflict (username) do update
-		   set kind = excluded.kind, node = excluded.node, module = excluded.module,
-		       password_hash = excluded.password_hash, minted_at = now()`,
-		u.Username, u.Kind, u.Node, u.Module, string(hash)); err != nil {
-		return "", fmt.Errorf("cannot record the bus user %s: %w", u.Username, err)
 	}
 	return password, nil
 }

internal/inventory/migrations/0045-a-machine-says-which-host-runs-it.sql

@@ -0,0 +1,15 @@
+-- The version of the host running on a machine, as the machine reports it.
+--
+-- novox/hq 04-ISSUES/087. A host parses a declaration strictly: a field it does not know makes it
+-- refuse the whole declaration and apply nothing. That is deliberate — it keeps a half-understood
+-- declaration off a machine — and it makes every new field in a declaration a flag day, hosts before
+-- controller. The mesh had no record of which host a machine runs, so it could neither refuse to send
+-- a declaration a machine cannot parse nor say which machines were behind. The order was kept by
+-- somebody remembering it.
+--
+-- The machine has been reporting this since ADR 0141 and the control plane discarded it: the field was
+-- absent from the controller's own copy of the report, so it was unmarshalled into nothing.
+--
+-- Null for a machine that has not reported since this column existed, which is not the same as a
+-- machine running no host — so a reader is never told a version the mesh does not have.
+alter table node add column host_version text;

internal/inventory/migrations/0046-each-send-is-numbered.sql

@@ -0,0 +1,11 @@
+-- The order of what a machine was sent, so a host can tell an older declaration from a newer.
+--
+-- novox/hq 04-ISSUES/107. A declaration's only identity was the digest of its bytes. The host could
+-- say "this is not the last one" and could not say "this is older", so a backlog drained out of
+-- order applied a declaration the mesh had already superseded. The controller already holds a
+-- per-node lock while it composes and records each send — the order existed and was thrown away at
+-- the wire.
+--
+-- One counter per node, taken under that hold, one higher per send. Null is a machine sent nothing
+-- since this existed, which is not the same as a machine sent nothing.
+alter table node add column sequence bigint;

internal/inventory/nodes.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/jackc/pgx/v5"
+	"github.com/novox/mesh-controller/internal/broker"
 	"github.com/novox/mesh-controller/internal/store"
 )
 
@@ -62,6 +63,11 @@ type Node struct {
 	// entry. What decides who a file under a home is owned by, and which account `ssh <node>` uses.
 	Account     string
 	AccountHome string
+
+	// HostVersion is the version of the host this machine reported running (novox/hq 04-ISSUES/087).
+	// Empty when it has not said since the mesh began keeping it — which is not the same as running
+	// no host, so nothing derives "behind" from an empty one.
+	HostVersion string
 }
 
 // Home is the account's home directory, derived when not stored: /root for root, /home/<account>
@@ -135,15 +141,20 @@ func (i *Inventory) AddNodeAs(ctx context.Context, name string, adopted bool) (N
 
 // nodeColumns and scanNode are the one reading of a node row, so every way of finding a node
 // says whether it is adopted.
-const nodeColumns = `id, name, created, last_seen, adopted, adopted_since, account, account_home`
+const nodeColumns = `id, name, created, last_seen, adopted, adopted_since, account, account_home,
+	host_version`
 
 func scanNode(row pgx.Row) (Node, error) {
 	var n Node
 	var seen, since *time.Time
+	var host *string
 	if err := row.Scan(&n.ID, &n.Name, &n.Created, &seen, &n.Adopted, &since,
-		&n.Account, &n.AccountHome); err != nil {
+		&n.Account, &n.AccountHome, &host); err != nil {
 		return Node{}, err
 	}
+	if host != nil {
+		n.HostVersion = *host
+	}
 	if seen != nil {
 		n.LastSeen = *seen
 	}
@@ -263,6 +274,29 @@ func (i *Inventory) IssueToken(ctx context.Context, nodeName string, validFor ti
 		return Issued{}, err
 	}
 
+	// **And the account that secret is the password of** (novox/hq 04-ISSUES/146). The composed
+	// user list names an enrolment user for every node with a live token, and nothing minted a
+	// credential for it — so the composer left it out as a user with no password and every
+	// enrolment was refused by the server before the mesh heard of it.
+	//
+	// Recorded rather than minted: the token's secret IS the password, which is what lets a
+	// machine's first connection be authenticated by the thing it is enrolling with. It cannot be
+	// chosen here, because it has already been handed to whoever will present it.
+	//
+	// Outside the transaction on purpose. The token is what the mesh promised; a credential that
+	// the next composition rewrites anyway is not worth failing an issue over, and a token with no
+	// account is recoverable by issuing another, while an account with no token is a user nobody
+	// can be.
+	if err := i.RecordBusPassword(ctx, BusUser{
+		Username: broker.Principal{Kind: broker.KindEnrolment, Node: node.Name}.Username(),
+		Kind:     BusEnrolment,
+		Node:     node.Name,
+	}, secret); err != nil {
+		return Issued{}, fmt.Errorf(
+			"the token for %s was issued and the bus account it is the password of was not "+
+				"recorded, so this token cannot connect: %w", node.Name, err)
+	}
+
 	return Issued{Node: node, Secret: secret, Expires: expires}, nil
 }
 
@@ -956,3 +990,49 @@ type Machine struct {
 	// being out of date and reads differently to whoever is looking.
 	Never bool
 }
+
+// RecordHostVersion keeps the version of the host a machine reported running (novox/hq 04-ISSUES/087).
+//
+// Never cleared by a report that carries none: a bare word that the node is there says nothing about
+// its host, and a machine whose host predates ADR 0141 reports none at all. So an empty version means
+// the mesh has not been told, and the caller does not write it.
+func (i *Inventory) RecordHostVersion(ctx context.Context, id, version string) error {
+	version = strings.TrimSpace(version)
+	if version == "" {
+		return nil
+	}
+	_, err := i.store.Pool().Exec(ctx,
+		`update node set host_version = $2, last_seen = now() where id = $1`, id, version)
+	return err
+}
+
+// NextSequence takes the next number for a declaration to this node, one higher than the last it
+// was sent (novox/hq 04-ISSUES/107).
+//
+// **One statement, so two composers cannot take the same number.** The caller holds the node while it
+// composes and sends, so in practice there is one; the increment is atomic anyway, because a rule
+// that is true only while a lock is held somewhere else is a rule nobody can see from here.
+func (i *Inventory) NextSequence(ctx context.Context, id string) (int64, error) {
+	var n int64
+	err := i.store.Pool().QueryRow(ctx,
+		`update node set sequence = coalesce(sequence, 0) + 1 where id = $1 returning sequence`, id).Scan(&n)
+	if err != nil {
+		return 0, fmt.Errorf("taking the next sequence for %s: %w", id, err)
+	}
+	return n, nil
+}
+
+// Sequence is the number of the last declaration this node was sent, and zero for one sent nothing
+// since sends were numbered. Read, not taken: what the mesh WOULD send is composed with this, so it
+// is byte for byte what it DID send when nothing else changed — a comparison that took a fresh number
+// would read every machine as behind for ever (novox/hq 04-ISSUES/107).
+func (i *Inventory) Sequence(ctx context.Context, id string) (int64, error) {
+	var n *int64
+	if err := i.store.Pool().QueryRow(ctx, `select sequence from node where id = $1`, id).Scan(&n); err != nil {
+		return 0, fmt.Errorf("reading the sequence of %s: %w", id, err)
+	}
+	if n == nil {
+		return 0, nil
+	}
+	return *n, nil
+}

internal/link/enrolment.go

@@ -312,6 +312,14 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
 			return false, err
 		}
 	}
+	// Which host produced this report (novox/hq 04-ISSUES/087), whenever it says. Recorded on every
+	// report that carries it and never cleared by one that does not — a bare word that the node is
+	// there says nothing about its host, and a machine whose host predates this reports none.
+	if report.Host != "" {
+		if err := e.Inventory.RecordHostVersion(ctx, node.ID, report.Host); err != nil {
+			return false, err
+		}
+	}
 	// What it says about the tunnel it carried (novox/hq ADR 0105), whenever it says it.
 	if report.Tunnel != nil {
 		if err := e.Inventory.RecordCarriedTunnel(ctx, node.ID, inventory.Carried{

internal/link/protocol.go

@@ -184,6 +184,15 @@ type Report struct {
 	// leaves the one it has: a rule written around a link with no name is a rule set that does not
 	// load, and that is a machine filtering nothing while its unit reports success.
 	Outward []string `json:"outward,omitempty"`
+
+	// Host is the version of the host that produced this report (novox/hq ADR 0141).
+	//
+	// **The machine has sent this since 0141 and this struct did not have it**, so it was
+	// unmarshalled into nothing and the mesh could not say which host any machine runs
+	// (novox/hq 04-ISSUES/087). A host refuses a declaration carrying a field it does not know, and
+	// refuses it whole — which is right, and makes every new field a flag day that the mesh could
+	// not see coming.
+	Host string `json:"host,omitempty"`
 	// Reachable is what can be reached on the machine now: every listening socket and every
 	// published container port. Only an adopted node reports it; it is what converging previews.
 	Reachable []Reach `json:"reachable,omitempty"`

internal/overlay/generator.go

@@ -169,10 +169,12 @@ func (g *Generator) Graph() Graph { return g.graph }
 //
 //   - No floor: no header, no localhost, no `127.0.1.1` — those are the machine's, above the region.
 //   - A machine's own line is marked, and its mesh name resolves to its mesh address, not loopback.
-//   - `.Names` is every name the mesh serves (issue 111), so a container reaching a routed name
+//   - `.Names` is every name the mesh serves (issue 111), so anything on the machine reaching a
-//     finds the machine serving it; machines with no address yet are already left out of the set.
+//     routed name through its resolver finds the machine serving it; machines with no address yet
+//     are already left out of the set. A routed name is one alias, itself — a machine has a bare
+//     name beside its full one, a routed name has nothing beside it (issue 157).
 const hostsTemplate = "# The mesh's names. This region is replaced whenever a machine joins or leaves.\n" +
-	"{{range .Names}}{{.Address}}\t{{.FQDN}}\t{{.Name}}{{if eq .Name $.Node}}\t# this machine{{end}}\n{{end}}"
+	"{{range .Names}}{{.Address}}\t{{.FQDN}}{{if ne .Name .FQDN}}\t{{.Name}}{{end}}{{if eq .Name $.Node}}\t# this machine{{end}}\n{{end}}"
 
 // Manifest is the module the mesh provides for itself.
 //