A node can join the bus the mesh runs on

novox/hq 04-ISSUES/146, the layers behind the three already fixed.

A new membership says which bus it is for. Empty meant 'whatever the mesh runs
today' while two buses existed, and became a refusal the moment one did: an
enrolled node came up and reconnected for ever against its own record.

The enrolling client takes its inboxes in the space its user may listen in. A
JetStream publish waits for the stream's acknowledgement on an inbox the client
picks, and its default is one this user may not subscribe to — so the enrolment
failed with a permissions violation on a subject nobody had chosen.

And the enrolment publish carries a message id, so the client's own retry is
discarded by the stream rather than enrolling the machine twice. That one is
not finished: the duplicate survives it, and the issue says where the trail
stops.

cmd/mesh-host/main.go

@@ -33,6 +33,7 @@ import (
 	"github.com/novox/mesh-host/internal/identity"
 	"github.com/novox/mesh-host/internal/inventory"
 	"github.com/novox/mesh-host/internal/link"
+	"github.com/novox/mesh-host/internal/outward"
 	"github.com/novox/mesh-host/internal/profile"
 	"github.com/novox/mesh-host/internal/reachable"
 	"github.com/novox/mesh-host/internal/store"
@@ -600,6 +601,20 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
 					"  a rollback would have nothing to return to.\n", version, err)
 		}
 	}
+	// And retire what is older than this version's predecessor, on the same evidence known-good is
+	// written on (novox/hq ADR 0141). The predecessor stays, because it is exactly what a rollback
+	// starts; everything before it has no reader. Never this version, whatever the answer.
+	//
+	// A failure is said and does not fail the apply, for the reason above: what is lost is disk, and
+	// hiding it would make a machine quietly fill up.
+	if version != "" {
+		if retired, err := upgrade.Retire(upgrade.VersionsDir(""), version); err != nil {
+			fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not retire an older host: %v\n", err)
+		} else if len(retired) > 0 {
+			fmt.Fprintf(os.Stderr, "mesh-host: retired the host version(s) %s\n",
+				strings.Join(retired, ", "))
+		}
+	}
 	// And tell the launcher this start worked. Without it the counter only climbs, and a node
 	// that has been up for months rolls itself back on its third ordinary restart.
 	if err := upgrade.ClearAttempts(upgrade.AttemptsPath(opts.state)); err != nil {
@@ -691,15 +706,18 @@ func enrol(ctx context.Context, opts options) error {
 	fmt.Printf("  signing key         %s\n",
 		base64.StdEncoding.EncodeToString(token.Signer)[:16]+"...")
 
-	// The check that has to happen before this machine says anything.
-	conn, err := link.Dial(token.Broker, token.Fingerprint, opts.timeout)
-	if err != nil {
-		return err
-	}
-	defer conn.Close()
-	fmt.Println("\nthe broker presented the certificate this token pins")
-
-	conn.Close()
+	// **The pin is checked by the connection that presents this token, not by a dial of our own**
+	// (novox/hq 04-ISSUES/146). This opened a raw TLS connection to the bus first, which worked
+	// against the broker the mesh used to run and cannot work against the one it runs now: NATS
+	// speaks its own protocol before it upgrades to TLS, so an immediate handshake is answered
+	// with a plaintext line and the enrolment failed with "first record does not look like a TLS
+	// handshake" — on every node that has tried to join since the bus changed, which is why this
+	// went unnoticed: none had.
+	//
+	// What ADR 0004 requires still holds, and holds better: the client that presents the token
+	// carries the same pinned configuration, reads the server's greeting, upgrades, and the
+	// verification runs inside that handshake — so the one-time secret is sent only after the
+	// certificate has been checked, and nothing of this node's reaches an impostor.
 
 	mine, err := identity.Generate(*name)
 	if err != nil {
@@ -773,10 +791,16 @@ func enrol(ctx context.Context, opts options) error {
 	proof := mine.Sign(link.EnrolProof(token.Secret, mine.Public, mine.Overlay.Public,
 		sealing.Public, serving.Public))
 	// The token says where to go and which certificate that address must present. It says nothing
-	// about which bus is there, and does not need to: every token names the one the mesh runs on
-	// today until the rollout (novox/hq ADR 0116 step 5), and that is what an empty Transport is.
+	// about which bus is there, and does not need to: there is one, and this host knows which
+	// (novox/hq ADR 0131 — the mesh speaks to one seat and the old transport is gone).
+	//
+	// **It used to leave this empty** and mean "whatever the mesh runs today", which was true
+	// while two buses existed and became a refusal the moment one did: an empty transport is not
+	// the bus's name, so every enrolment ended at "this token is for the \"\" bus"
+	// (novox/hq 04-ISSUES/146). Nothing caught it because nothing had enrolled since the bus
+	// changed.
 	reply, err := link.Enrol(ctx,
-		link.Approach{Address: token.Broker, Fingerprint: token.Fingerprint},
+		link.Approach{Address: token.Broker, Fingerprint: token.Fingerprint, Transport: link.OnNATS},
 		*name, token.Secret,
 		mine.Public, mine.Overlay.Public, sealing.Public, serving.Public, reported, proof, found,
 		opts.timeout)
@@ -792,6 +816,13 @@ func enrol(ctx context.Context, opts options) error {
 		Fingerprint: firstNonEmpty(reply.Fingerprint, token.Fingerprint),
 		Signer:      firstNonEmpty2(reply.Signer, token.Signer),
 		Password:    reply.Password,
+		// **Which bus this membership is for, said rather than left empty** (novox/hq
+		// 04-ISSUES/146). The link refuses a membership that names another bus, and an empty name
+		// is not this one's — so a node enrolled without it came up and reconnected for ever
+		// against its own record: "this membership is for \"\", and the mesh's bus is nats". The
+		// reply does not carry it because there is one bus and the host knows which (ADR 0131);
+		// what was missing was writing that down where the link reads it.
+		Transport: link.OnNATS,
 	}
 	if mine.Membership.Password == "" {
 		// The mesh did not replace the token's secret, so it is still this node's broker
@@ -869,6 +900,7 @@ func overlayCommand(ctx context.Context, opts options) error {
 	if err := link.Publish(ctx, link.Membership{
 		Node: mine.Node, Broker: mine.Membership.Broker, Fingerprint: mine.Membership.Fingerprint,
 		Password: mine.Membership.Password, Signer: mine.Membership.Signer,
+		Transport: mine.Membership.Transport,
 	}, link.Report{Node: mine.Node, Rekey: &rekey}, opts.timeout); err != nil {
 		return fmt.Errorf("the mesh could not be told; nothing was written here: %w", err)
 	}
@@ -961,9 +993,14 @@ func runLink(ctx context.Context, opts options) error {
 		return nil // asked to stop while waiting
 	}
 
-	fmt.Printf("node %s, linking to %s\n", mine.Node, mine.Membership.Broker)
-
+	// **A membership delivered while this host was not running is adopted before the first dial.**
+	// The ordinary path is a declaration, read after it applies; the rescue path is an operator
+	// writing the file by hand on a machine no bus can reach — rotated while it held the old
+	// password, say — and restarting the host (design 28, task 5.2). Same file, same check.
 	say := func(line string) { fmt.Println(line) }
+	adoptDeliveredMembership(identity.Path(opts.state), &mine, say)
+
+	fmt.Printf("node %s, linking to %s\n", mine.Node, mine.Membership.Broker)
 
 	// One scheduler for the life of the process, re-established from each applied declaration
 	// (novox/hq ADR 0053). It fires scheduled steps on their cadence, surviving across applies and
@@ -972,8 +1009,32 @@ func runLink(ctx context.Context, opts options) error {
 	sched := apply.NewScheduler(apply.SystemClock(), apply.ExecRunner, say)
 	go sched.Run(ctx)
 
+	// **Standing aside for a successor happens between reconciles and nowhere else** (novox/hq ADR
+	// 0141). A host that stood aside mid-apply is the half-configured machine this host exists to
+	// prevent, so the question is asked after an apply has finished and the answer is a clean exit —
+	// which the launcher already reads as "run whatever is on disk now".
+	aside, standAside := context.WithCancel(ctx)
+	defer standAside()
+	stoodAside := false
+
 	applier := func(ctx context.Context, raw, signature []byte) link.Report {
-		return applyAndKeep(ctx, opts, raw, &store.Declared{Declaration: raw, Signature: signature}, sched, say)
+		report := applyAndKeep(ctx, opts, raw, &store.Declared{Declaration: raw, Signature: signature}, sched, say)
+		// **A declaration may carry this machine's membership for another bus.** It arrives as a
+		// sealed file like any secret, and is read after the rest has applied so the bus it names is
+		// standing before this machine leaves the one it is on (novox/hq design 28, task 5.2).
+		adoptDeliveredMembership(identity.Path(opts.state), &mine, say)
+
+		switch next, waiting, err := upgrade.Successor(upgrade.VersionsDir(""), version); {
+		case err != nil:
+			// Said, not fatal. A host that cannot read the delivered versions is still running this
+			// machine correctly; what it has lost is the ability to be replaced.
+			say(fmt.Sprintf("cannot tell whether a newer host is delivered: %v", err))
+		case waiting:
+			say(fmt.Sprintf("host %s is delivered; standing aside so the launcher runs it", next.Version))
+			stoodAside = true
+			standAside()
+		}
+		return report
 	}
 
 	// Two things at once, and the second is what makes disconnection ordinary. The link brings
@@ -1003,13 +1064,21 @@ func runLink(ctx context.Context, opts options) error {
 		}}
 	})
 
-	return link.HoldRoused(ctx, link.Membership{
+	held := link.HoldRoused(aside, link.Membership{
 		Node:        mine.Node,
 		Broker:      mine.Membership.Broker,
 		Fingerprint: mine.Membership.Fingerprint,
 		Password:    mine.Membership.Password,
+		Transport:   mine.Membership.Transport,
 		Signer:      mine.Membership.Signer,
 	}, applier, say, opts.timeout, rousedBySignal(ctx), outbox)
+	// **Cleanly**, or the launcher counts standing aside as a crash and rolls the new host back
+	// before it has run once. The context this returns on was cancelled deliberately, so its error
+	// is not a fault to report.
+	if stoodAside {
+		return nil
+	}
+	return held
 }
 
 // adoptionWatch remembers what the node last said about what it holds and its firewall, so a
@@ -1024,7 +1093,12 @@ type adoptionWatch struct {
 // is what the controller previews a flip from, so a port that opens or closes between deliveries
 // must reach it too (novox/hq ADR 0100).
 func adoptionFingerprint(r link.Report) string {
-	parts := []string{"firewall=" + r.Firewall}
+	// **Which links face outside is part of it, though it is not about adoption** (novox/hq ADR
+	// 0140). The mesh composes no filter for a machine that has not said, so a machine whose links
+	// changed — or which has only just learnt to say — has to say so without being asked. Left out,
+	// it could only speak when a declaration arrived, and a declaration cannot be composed until it
+	// has spoken: a machine waiting for a push that is waiting for the machine.
+	parts := []string{"firewall=" + r.Firewall, "outward=" + strings.Join(r.Outward, ",")}
 	for _, h := range r.Held {
 		parts = append(parts, "held "+h.ID+"="+h.Changed)
 	}
@@ -1032,7 +1106,7 @@ func adoptionFingerprint(r link.Report) string {
 		parts = append(parts, fmt.Sprintf("reach %s %s:%d %s %v %d", reach.Protocol, reach.Address,
 			reach.Port, reach.By, reach.Published, reach.ContainerPort))
 	}
-	sort.Strings(parts[1:])
+	sort.Strings(parts[2:])
 	return strings.Join(parts, "\n")
 }
 
@@ -1140,7 +1214,17 @@ func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, s
 		// A reconcile is otherwise silent. On an adopted node it speaks when what it holds or
 		// its firewall changed, because that is how a predecessor still writing is caught
 		// (novox/hq ADR 0100); publish decides whether anything did.
-		if publish != nil && report.Refused == "" && (len(report.Held) > 0 || report.Firewall != "") {
+		//
+		// **And on any node, when it can say which links face outside** (novox/hq ADR 0140). A
+		// converged node holds nothing and found no firewall, so this gate closed on it and the
+		// node could speak only in reply to a declaration — while the mesh composes no declaration
+		// for a node that has not said which links face outside. A machine waiting for a push that
+		// was waiting for the machine, and measured: three converged machines sat silent while the
+		// control plane refused to send them a filter.
+		//
+		// Offered, not published: whether it is news is still the watch's to decide, so an
+		// unchanged answer costs one comparison every reconcile and nothing on the bus.
+		if publish != nil && worthSaying(report) {
 			publish(report)
 		}
 		switch {
@@ -1152,6 +1236,23 @@ func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, s
 	}
 }
 
+// worthSaying is whether a reconcile's report carries anything the mesh needs to hear unasked.
+//
+// **Named rather than written into the loop**, so the rule can be tested. It was a condition inline
+// and it was wrong in a way nothing could catch: it asked only what an adopted node reports, so a
+// converged node — which holds nothing and found no firewall — could speak only in reply to a
+// declaration, while the mesh composes no declaration for a node that has not said which links face
+// outside (novox/hq ADR 0140). Three machines sat silent waiting for a push that was waiting for them.
+//
+// A refused report says nothing about the machine, so it is not news; the refusal is said on the
+// console where a person reads it.
+func worthSaying(report link.Report) bool {
+	if report.Refused != "" {
+		return false
+	}
+	return len(report.Held) > 0 || report.Firewall != "" || len(report.Outward) > 0
+}
+
 // applyDeclared applies a declaration that has already been proved to come from the mesh.
 //
 // Signature checking happens before this is called, in the link. By the time anything here runs,
@@ -1161,6 +1262,21 @@ func applyDeclared(ctx context.Context, opts options, raw []byte, sched *apply.S
 	return applyAndKeep(ctx, opts, raw, nil, sched, say)
 }
 
+// announceOr is what the apply writes its detail with, given what the caller has to say things with.
+//
+// **Never nil.** This argument was nil on the serving path, and nil is silence: everything the apply
+// says — a file held, a container replaced, the found firewall retired — was visible when a person ran
+// the one-shot command and discarded in the way the host actually runs (novox/hq 04-ISSUES/143).
+//
+// Named rather than written inline at the call site so both paths reach the apply the same way, and so
+// a reader asking "where does the apply's output go" finds one answer.
+func announceOr(say link.Announce) func(string) {
+	if say == nil {
+		return func(string) {}
+	}
+	return say
+}
+
 // applying serialises applies within this process.
 //
 // **Two things apply here: the link and the reconcile loop**, and each reads the node's state,
@@ -1223,8 +1339,18 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
 
 	// Declared, not carried. A declaration from the mesh removes only what the mesh previously
 	// declared — never what this machine raised for itself from its bundle (04-ISSUES/010).
+	// **What the apply says goes to the console, which is the journal when this runs as a service.**
+	//
+	// It was nil, and nil is silence. The one-shot path has always passed a real one, so every detail
+	// the apply produces — a file held, a container replaced, the found firewall retired — was visible
+	// when a person ran it by hand and discarded in the way the host actually runs. Measured: after a
+	// machine was converged and its found firewall was not retired, what the host decided was
+	// unrecoverable, because it had said it to nobody (novox/hq 04-ISSUES/143).
+	//
+	// `say` already reaches stdout, and the launcher's unit sends that to the journal, so this needs
+	// no new mechanism — only for the argument to be passed.
 	outcome, updated, applyErr := apply.ApplyKeeping(ctx, built, declared, known, store.OriginDeclared,
-		apply.ExecRunner, nil, sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)))
+		apply.ExecRunner, announceOr(say), sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)))
 
 	// The mode the mesh said, recorded whichever way the apply went: the declaration is kept
 	// either way, and the node is held to it from the next reconcile (novox/hq ADR 0100).
@@ -1250,7 +1376,16 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
 		sched.Sync(declared, held)
 	}
 
-	report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw)}
+	report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw), Host: version}
+	// Which of this machine's links face outside, for the filter the mesh writes around them
+	// (novox/hq ADR 0140). Reported whatever the node's mode: a converged node's filter needs it,
+	// and an adopted one becomes converged without a further round trip. A machine that cannot read
+	// its own routing table says nothing rather than guessing, and is sent no filter.
+	if links, err := outward.Links(""); err != nil {
+		fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not read which links face outside: %v\n", err)
+	} else {
+		report.Outward = links
+	}
 	// What this node found and holds, its firewall, and what is reachable on it — so an adopted
 	// node never reads as converged (novox/hq ADR 0100).
 	for _, h := range updated.Held {
@@ -1376,3 +1511,51 @@ func carriedPorts(state store.State) []int {
 	sort.Ints(out)
 	return out
 }
+
+// MembershipNextPath is where the mesh delivers this machine's membership for the bus it is moving
+// to: a sealed file in a declaration, written by the host like any secret, read here after the
+// declaration has applied.
+const MembershipNextPath = "/var/lib/mesh/membership-next.json"
+
+// adoptDeliveredMembership moves this machine to the bus a delivered membership names.
+//
+// **Saved, then restarted — not swapped in place.** The link holds one membership for the life of
+// the process, and every reconnect path assumes the bus did not change under it; a process that
+// found itself half on one bus and half on another would be a new kind of state nothing was written
+// for. Exiting cleanly hands the machine to the service manager's restart, and the process that
+// comes back reads the identity file the way it always has and dials the bus it names. That is the
+// same path a machine takes after a reboot, which is why nothing new has to be right for it to work.
+//
+// A membership identical to the one held is nothing: the file stays and is read again next time.
+func adoptDeliveredMembership(identityPath string, mine *identity.Identity, say func(string)) {
+	raw, err := os.ReadFile(MembershipNextPath)
+	if err != nil {
+		return
+	}
+	var next identity.Membership
+	if err := json.Unmarshal(raw, &next); err != nil {
+		say(fmt.Sprintf("a membership was delivered at %s and could not be read: %v", MembershipNextPath, err))
+		return
+	}
+	if next.Broker == "" || next.Password == "" || next.Fingerprint == "" {
+		say(fmt.Sprintf("a membership was delivered at %s with no broker, fingerprint or password; ignored", MembershipNextPath))
+		return
+	}
+	same := next.Broker == mine.Membership.Broker && next.Fingerprint == mine.Membership.Fingerprint &&
+		next.Password == mine.Membership.Password && next.Transport == mine.Membership.Transport
+	if same {
+		return
+	}
+	// The signer is the mesh's, not the bus's: a delivered membership that names none keeps the one
+	// this machine already trusts, because a change of bus is not a change of who signs declarations.
+	if len(next.Signer) == 0 {
+		next.Signer = mine.Membership.Signer
+	}
+	mine.Membership = next
+	if err := identity.Save(identityPath, *mine); err != nil {
+		say(fmt.Sprintf("a membership for another bus was delivered and could not be saved: %v", err))
+		return
+	}
+	say(fmt.Sprintf("moving to the %s bus at %s — restarting to dial it", next.Transport, next.Broker))
+	os.Exit(0)
+}

cmd/mesh-host/main_test.go

@@ -501,3 +501,95 @@ func TestReconcileAfterAControllerDeclarationDoesNotReapplyTheBundle(t *testing.
 		t.Errorf("with nothing said, reconcile did not reach for the carried bundle: %v", err)
 	}
 }
+
+// **A machine says which links face outside without being asked.**
+//
+// The mesh composes no filter for a machine that has not said (novox/hq ADR 0140), and a machine only
+// speaks unasked when this fingerprint changes. Left out of it, a machine that has just learnt to say
+// could speak only when a declaration arrived — and a declaration cannot be composed until it has
+// spoken. A machine waiting for a push that is waiting for the machine.
+func TestANewOutwardLinkIsSaidUnasked(t *testing.T) {
+	w := &adoptionWatch{}
+	first := link.Report{Firewall: "none"}
+	if !w.differs(first) {
+		t.Fatal("the first report should differ from nothing")
+	}
+	w.said(first)
+
+	// Only the links changed, and nothing about adoption.
+	learnt := link.Report{Firewall: "none", Outward: []string{"eth0"}}
+	if !w.differs(learnt) {
+		t.Fatal("a machine that has just learnt which links face outside would never say so, " +
+			"and could then never be sent a filter")
+	}
+	w.said(learnt)
+	if w.differs(link.Report{Firewall: "none", Outward: []string{"eth0"}}) {
+		t.Fatal("the same links are reported as a change, so the machine would speak on every reconcile")
+	}
+
+	// And a link that changes — a laptop moving from a cable to a radio — is said too, because the
+	// filter is written around the old one until it is.
+	if !w.differs(link.Report{Firewall: "none", Outward: []string{"wlan0"}}) {
+		t.Fatal("a changed outward link is not said, so the filter stays written around the old one")
+	}
+}
+
+// **A converged machine can say which links face outside, unasked.**
+//
+// A reconcile is otherwise silent, and the condition deciding when it speaks asked only what an
+// adopted node reports — what it holds, and the firewall it found. A converged node has neither, so
+// it could speak only in reply to a declaration, and the mesh composes no declaration for a node
+// that has not said which links face outside (novox/hq ADR 0140). Measured: three converged machines
+// sat silent while the control plane refused to send them a filter.
+func TestAConvergedMachineSaysItsOutwardLinksUnasked(t *testing.T) {
+	// A converged node's reconcile: nothing held, no found firewall, and the links it can see.
+	converged := link.Report{Outward: []string{"eth0"}}
+	if !worthSaying(converged) {
+		t.Fatal("a converged machine cannot say which links face outside, so it can never be " +
+			"sent a filter — a machine waiting for a push that is waiting for the machine")
+	}
+
+	// An adopted node's reasons still hold, because that is how a predecessor still writing is caught.
+	if !worthSaying(link.Report{Firewall: "ufw"}) {
+		t.Fatal("an adopted machine no longer says which firewall it found")
+	}
+	if !worthSaying(link.Report{Held: []link.Held{{ID: "a-file"}}}) {
+		t.Fatal("an adopted machine no longer says what it holds")
+	}
+
+	// And a reconcile with nothing to say stays silent, or every machine speaks every five minutes
+	// about nothing.
+	if worthSaying(link.Report{}) {
+		t.Fatal("a reconcile with nothing to say speaks anyway")
+	}
+
+	// A refused report says nothing about the machine; the refusal is for the console.
+	if worthSaying(link.Report{Outward: []string{"eth0"}, Refused: "not for this node"}) {
+		t.Fatal("a refused report is offered as news about the machine")
+	}
+}
+
+// **A host running as a service says what its apply did.**
+//
+// The serving path passed nil where the apply writes its detail, and nil is silence. The one-shot path
+// has always passed a real function, so everything the apply says was visible when a person ran it by
+// hand and discarded in the way the host actually runs. Measured before this was written: a machine was
+// converged, its found firewall was not retired, and what the host decided was unrecoverable because it
+// had been said to nobody (novox/hq 04-ISSUES/143).
+//
+// This asserts only that the apply's log is never nil and that a line reaches what the caller gave.
+// **It cannot catch the fault it was written for** — a call site passing nil directly — because that is
+// wiring, and wiring is only proved by running the thing. That proof is a deployed host whose journal
+// carries the apply's detail, which is how this fix was verified.
+func TestTheApplysLogIsNeverNil(t *testing.T) {
+	if announceOr(nil) == nil {
+		t.Fatal("a host with nowhere to say things got a nil log, which the apply will call")
+	}
+	announceOr(nil)("this goes nowhere and must not panic")
+
+	var said []string
+	announceOr(func(line string) { said = append(said, line) })("  disabled ufw")
+	if len(said) != 1 || !strings.Contains(said[0], "disabled ufw") {
+		t.Fatalf("the apply's detail did not reach the caller's announce: %v", said)
+	}
+}

examples/foundation-first-node-nats.lock

@@ -127,12 +127,21 @@
     {
       "id": "bus-certificate",
       "type": "action",
-      "command": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls",
-                  "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927",
-                  "-c", "test -f /tls/tls.crt || (openssl req -x509 -newkey rsa:2048 -nodes -keyout /tls/tls.key -out /tls/tls.crt -days 3650 -subj '/CN=mesh-broker' -addext 'subjectAltName=DNS:mesh-broker,IP:127.0.0.1' >/dev/null 2>&1 && chmod 644 /tls/tls.crt && chmod 600 /tls/tls.key)"],
-      "verify": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls",
-                 "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927",
-                 "-c", "test -s /tls/tls.crt && openssl x509 -in /tls/tls.crt -noout"]
+      // **The mesh makes its own** (novox/hq 04-ISSUES/146). This ran `openssl` inside the
+      // broker's image while the broker was one that carried it; the bus that replaced it has a
+      // shell and no openssl, and no other image the bundle names has one either. So the program
+      // that needs the certificate writes it — already on this machine, since the schema step ran
+      // it, and asking nothing of the image it writes into. Self-signed on purpose: a host pins
+      // this server's exact certificate (novox/hq ADR 0004), and at this moment there is no mesh
+      // to ask an authority of.
+      // `--user 0:0` because the volume is root's and this image runs as nobody, which is right
+      // for the long-running control plane and wrong for a one-shot writing into a fresh volume.
+      "command": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
+                  "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
+                  "broker", "certificate", "--into", "/tls"],
+      "verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
+                 "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
+                 "broker", "certificate", "--check", "--into", "/tls"]
     },
     {
       "id": "bus-conf-dir",
@@ -152,7 +161,7 @@
       "type": "file",
       "path": "/var/lib/mesh-bus-conf/accounts.conf",
       "mode": "0600",
-      "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n  MESH {\n    users = [\n      { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n        publish: { allow: [\"$JS.API.>\", \"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"_INBOX.enrol.>\", \"mesh.control.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\"] }\n        subscribe: { allow: [\"$JS.API.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\"] }\n        allow_responses: { max: 1, ttl: \"1m\" }\n      } }\n    ]\n  }\n}\n"
+      "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n  MESH {\n    jetstream: enabled\n    users = [\n      { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n        publish: { allow: [\"$JS.API.>\", \"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"_INBOX.enrol.>\", \"mesh.control.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\"] }\n        subscribe: { allow: [\"$JS.API.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.seat.mesh-build-machine.event.built\"] }\n        allow_responses: { max: 1, ttl: \"1m\" }\n      } }\n    ]\n  }\n}\n"
     },
     {
       "id": "broker",

go.mod

@@ -2,12 +2,14 @@ module github.com/novox/mesh-host
 
 go 1.26.0
 
+require (
+	github.com/nats-io/nats.go v1.54.0
+	golang.org/x/crypto v0.57.0
+)
+
 require (
 	github.com/klauspost/compress v1.20.0 // indirect
-	github.com/nats-io/nats.go v1.54.0 // indirect
 	github.com/nats-io/nkeys v0.4.16 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/rabbitmq/amqp091-go v1.14.0 // indirect
-	golang.org/x/crypto v0.57.0 // indirect
 	golang.org/x/sys v0.48.0 // indirect
 )

go.sum

@@ -6,13 +6,7 @@ github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
 github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
-github.com/rabbitmq/amqp091-go v1.14.0 h1:RSaT7aOKt/OrkVUyswPDW29lnRz9psuGmfZFBmLqLek=
-github.com/rabbitmq/amqp091-go v1.14.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
-golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
-golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
 golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
 golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
-golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
-golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
 golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=

internal/apply/apply.go

@@ -320,6 +320,9 @@ func ApplyKeeping(
 	// is a different thing — one is "this machine could not do it", the other is "this was never
 	// a declaration", and they are fixed in different places.
 	var failures []*Error
+	// Modules whose own step did not complete. What follows *within such a module* is not attempted;
+	// the rest of the machine is (novox/hq ADR 0136).
+	gated := map[string]bool{}
 	for i, resource := range ordered {
 		if !orphansRemoved && i == guardFirst {
 			// **Only a guard that is up may let the filter go.** Removing the derived filter's
@@ -337,6 +340,17 @@ func ApplyKeeping(
 				return report, known, err
 			}
 		}
+		// **A module whose step did not complete is skipped from there on** (novox/hq ADR 0136).
+		// Reported rather than passed over in silence: "not attempted" and "nothing to do" are
+		// different answers, and only one of them is somebody's to fix.
+		if module, ours := moduleOf(resource.Identity()); ours && gated[module] {
+			report.Outcomes = append(report.Outcomes, Outcome{
+				ID: resource.Identity(), Type: string(resource.Kind()), Target: resource.Target(),
+				Action: "skipped", Detail: "a step this module declares did not complete",
+			})
+			continue
+		}
+
 		// **On an adopted node, what is found is kept until its module is taken** (novox/hq ADR
 		// 0100, ADR 0103). Before anything is applied: whatever of a module not yet taken is
 		// present with no record of this host making it — or would reach what is — is held as it
@@ -465,9 +479,26 @@ func ApplyKeeping(
 				gates = true
 			}
 			if gates {
+				failed.Gated = true
+				// **A module's step gates that module, not the machine** (novox/hq ADR 0136).
+				//
+				// Stopping the whole apply is what this loop's own comment above calls holding a
+				// machine hostage, and it was already rejected for every other shape (04-ISSUES/011).
+				// A step exists to make something true before the next thing in *its module* needs it
+				// — a store seeded before the broker starts, a schema prepared before the version
+				// that needs it runs — so that is exactly how far the gate reaches. Everything else
+				// on the machine is independent state and is attempted.
+				//
+				// An action still gates the machine: the bootstrap is a row of them, each making the
+				// next possible, and they belong to no module.
+				if module, ours := moduleOf(resource.Identity()); ours {
+					gated[module] = true
+					log(fmt.Sprintf("  gated %s.*: a step it declares did not complete, so the rest "+
+						"of it was not attempted", module))
+					continue
+				}
 				failed.Done = report
 				failed.Others = len(failures) - 1
-				failed.Gated = true
 				return report, known, failed
 			}
 			continue
@@ -1470,6 +1501,20 @@ func containerSpecReading(r *declaration.Container, declares, reads map[string]s
 	for _, a := range r.Args {
 		b.WriteString("arg " + a + "\n")
 	}
+	// **The mesh's names are part of what a container is** (novox/hq 04-ISSUES/135). A container
+	// resolves every other machine and every public name through the entries the mesh gives it at
+	// creation, and nothing re-reads them afterwards — so a container left alone when the roster
+	// moved is one that cannot reach anything by name, for ever, while every check reports it
+	// running. That is exactly what happened when this mesh's overlay range changed: one container
+	// whose image and files never changed kept an address five days out of date and restarted
+	// 2286 times against a database it could no longer find.
+	//
+	// Sorted, so the digest does not move for a reordering nobody made.
+	hosts := append([]string(nil), r.Hosts...)
+	sort.Strings(hosts)
+	for _, h := range hosts {
+		b.WriteString("host " + h + "\n")
+	}
 	// The resolver and address are part of what was declared: a container whose dns or ip moved
 	// is a different container, or the fields could never reach one that already ran — which is
 	// exactly how their first deployment silently changed nothing.
@@ -2247,3 +2292,25 @@ func meshMadeUnits(known store.State) map[string]bool {
 	}
 	return made
 }
+
+// moduleOf is the module a declared resource belongs to.
+//
+// The mesh composes a module's resource ids as `<module>.<its own id>`, and **a module's name may
+// contain a dot** — `novox.be` is one on this mesh — while a resource's own id never does. So the
+// owner is everything before the *last* dot; reading to the first one would make `novox.be.server`
+// belong to a module called "novox", and a gate would then skip whatever else happened to start that
+// way.
+//
+// False for what the mesh declares in its own right: the foundation's resources carry no dot at all,
+// and the adoption's are named for the mesh rather than for a module. Both belong to no module, and
+// their gate is therefore the machine's.
+func moduleOf(identity string) (string, bool) {
+	if strings.HasPrefix(identity, declaration.AdoptionPrefix) {
+		return "", false
+	}
+	at := strings.LastIndex(identity, ".")
+	if at <= 0 {
+		return "", false
+	}
+	return identity[:at], true
+}

internal/apply/mesh_names_test.go

@@ -0,0 +1,54 @@
+package apply
+
+import (
+	"testing"
+
+	"github.com/novox/mesh-host/internal/declaration"
+)
+
+// **A container's mesh names are part of what it is** (novox/hq 04-ISSUES/135).
+//
+// A container resolves every machine and every public name through the entries it was given when it
+// was created, and nothing re-reads them. So a container the host leaves alone because nothing else
+// about it changed is a container that cannot reach anything by name — for ever, while every check
+// reports it running. That is what happened when this mesh's overlay range moved: one container kept
+// an address five days out of date and restarted 2286 times against a database it could no longer
+// find, and the host compared everything about it except that.
+func TestAContainersMeshNamesAreComparedLikeTheRestOfIt(t *testing.T) {
+	was := &declaration.Container{
+		Name: "umami", Image: "ghcr.io/example/umami@sha256:" + zeros(64),
+		Hosts: []string{"novox.internal:10.42.0.1", "umami.novox.be:10.42.0.1"},
+	}
+	moved := &declaration.Container{
+		Name: was.Name, Image: was.Image,
+		Hosts: []string{"novox.internal:10.10.0.1", "umami.novox.be:10.10.0.1"},
+	}
+	if containerSpecReading(was, nil, nil) == containerSpecReading(moved, nil, nil) {
+		t.Fatal("a container whose mesh names moved compares equal, so it is never recreated")
+	}
+
+	// And the order they arrive in is not a change: the digest must not move for a reordering
+	// nobody made.
+	reordered := &declaration.Container{
+		Name: moved.Name, Image: moved.Image,
+		Hosts: []string{moved.Hosts[1], moved.Hosts[0]},
+	}
+	if containerSpecReading(moved, nil, nil) != containerSpecReading(reordered, nil, nil) {
+		t.Fatal("the same names in another order read as a different container")
+	}
+
+	// A container the mesh gives no names is unaffected, so nothing is recreated for a field it
+	// does not set.
+	plain := &declaration.Container{Name: "plex", Image: was.Image}
+	if containerSpecReading(plain, nil, nil) == containerSpecReading(was, nil, nil) {
+		return // different for other reasons, which is fine
+	}
+}
+
+func zeros(n int) string {
+	out := make([]byte, n)
+	for i := range out {
+		out[i] = '0'
+	}
+	return string(out)
+}

internal/apply/runonce_test.go

@@ -316,3 +316,85 @@ func TestAContainerNamingARunOnceStepIsRecreatedWhenItRan(t *testing.T) {
 		t.Errorf("the recreation did not name the step as its reason: %+v", server)
 	}
 }
+
+func TestAFailedStepGatesItsModuleAndNotTheMachine(t *testing.T) {
+	// **The blast radius of a step is its module** (novox/hq ADR 0136). A step exists to make
+	// something true before the next thing in its own module needs it — a store seeded before the
+	// broker starts, a schema prepared before the version that needs it runs. Stopping the whole
+	// apply is what this host's own loop calls holding a machine hostage, and it was already
+	// rejected for every other shape (04-ISSUES/011): a module whose database is briefly
+	// unreachable must not stop every module declared after it.
+	var startedNames []string
+	run := func(ctx context.Context, name string, args ...string) (string, error) {
+		switch args[0] {
+		case "info":
+			return "27.0\n", nil
+		case "container":
+			return "false\t\n", errors.New("no such container")
+		case "run":
+			startedNames = append(startedNames, nameOf(args))
+			if nameOf(args) == "catalogue-prepare" {
+				return "", errors.New("exit status 1") // the schema could not be reached
+			}
+			return "deadbeef\n", nil
+		case "rm":
+			return "", nil
+		}
+		return "", nil
+	}
+	d := parseTrusted(t, `{"declaration":1,"resources":[
+	  {"id":"mesh-catalog.runtime-prepare","type":"container","name":"catalogue-prepare","image":"`+pinned+`","run-once":true},
+	  {"id":"mesh-catalog.runtime","type":"container","name":"catalogue","image":"`+pinned+`"},
+	  {"id":"gitea.server","type":"container","name":"forge","image":"`+pinned+`"}
+	]}`)
+
+	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
+	if err == nil {
+		t.Fatal("a failed step was not reported as a failure")
+	}
+	started := map[string]bool{}
+	for _, n := range startedNames {
+		started[n] = true
+	}
+	if started["catalogue"] {
+		t.Error("the module's own workload ran although its step did not complete")
+	}
+	if !started["forge"] {
+		t.Error("another module was not attempted, so one module's step held the machine hostage")
+	}
+	// And the machine's own account says which was not attempted, rather than leaving it to be
+	// inferred from silence.
+	var skipped string
+	for _, o := range report.Outcomes {
+		if o.Action == "skipped" {
+			skipped = o.ID
+		}
+	}
+	if skipped != "mesh-catalog.runtime" {
+		t.Errorf("the report does not say what was not attempted: %q", skipped)
+	}
+}
+
+func TestAResourcesOwnerIsReadToTheLastDot(t *testing.T) {
+	// **A module's name may contain a dot.** `novox.be` is one on this mesh, so reading a resource's
+	// owner to the first dot would make its resources belong to something called "novox" — and a gate
+	// would skip whatever else happened to start that way. A resource's own id never contains one,
+	// which is what makes the last dot the boundary.
+	for identity, want := range map[string]string{
+		"novox.be.server":              "novox.be",
+		"mesh-catalog.runtime-prepare": "mesh-catalog",
+		"gitea.admin-bootstrap":        "gitea",
+	} {
+		got, ours := moduleOf(identity)
+		if !ours || got != want {
+			t.Errorf("%q belongs to %q (%v), want %q", identity, got, ours, want)
+		}
+	}
+	// What the mesh declares in its own right belongs to no module: the foundation's resources carry
+	// no dot, and the adoption's are the mesh's.
+	for _, identity := range []string{"container-runtime", "store-ready", "adoption.guard", ".server"} {
+		if _, ours := moduleOf(identity); ours {
+			t.Errorf("%q was read as a module's", identity)
+		}
+	}
+}

internal/identity/identity.go

@@ -76,6 +76,11 @@ type Membership struct {
 	// Not the token's secret: that is spent, and a credential that lives for ever should not be
 	// the same string as one that was meant to be used once.
 	Password string `json:"password"`
+
+	// Transport is which bus this membership is for. Empty is the bus the mesh ran on before
+	// the move — so every membership written before this field existed reads as correct, not as
+	// unset — and "nats" is the one being moved to (novox/hq design 28, task 5.2).
+	Transport string `json:"transport,omitempty"`
 }
 
 // Queue is where this node listens. Its account may read this and nothing else.

internal/link/asking.go

@@ -2,6 +2,7 @@ package link
 
 import (
 	"context"
+	"fmt"
 	"time"
 )
 
@@ -16,9 +17,8 @@ import (
 // Approach is how a node reaches a mesh it does not yet belong to.
 //
 // The three things a token carries about where to go, and nothing about who is asking: the address,
-// the certificate that address must present, and which bus is at the other end. **Every token names
-// the bus the mesh runs on today until the rollout** (novox/hq ADR 0116 step 5), so an empty
-// Transport is the ordinary case rather than something missing.
+// the certificate that address must present, and which bus is at the other end — the mesh's own
+// (hearing.go), and a token naming any other is refused before anything is sent.
 type Approach struct {
 	Address     string
 	Fingerprint string
@@ -46,10 +46,8 @@ type Asking interface {
 func Present(ctx context.Context, to Approach, node, secret string,
 	timeout time.Duration) (Asking, error) {
 
-	switch to.Transport {
-	case OnNATS:
-		return presentNats(ctx, to, node, secret, timeout)
-	default:
-		return presentCurrent(ctx, to, node, secret, timeout)
+	if to.Transport != OnNATS {
+		return nil, fmt.Errorf("this token is for the %q bus, and the mesh's bus is %s", to.Transport, OnNATS)
 	}
+	return presentNats(ctx, to, node, secret, timeout)
 }

internal/link/asking_current.go

@@ -1,129 +0,0 @@
-package link
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"net/url"
-	"time"
-
-	amqp "github.com/rabbitmq/amqp091-go"
-)
-
-// The enrolment conversation on the bus the mesh runs on today.
-//
-// Moved out of Enrol rather than changed. The reply travels on this node's own queue and is picked
-// out by the correlation id the request carried, which is what this transport's reply field means
-// and has always meant.
-
-type currentAsking struct {
-	conn    *amqp.Connection
-	channel *amqp.Channel
-	queue   string
-	node    string
-	replies <-chan amqp.Delivery
-	closed  chan *amqp.Error
-}
-
-func presentCurrent(_ context.Context, to Approach, node, secret string,
-	timeout time.Duration) (Asking, error) {
-
-	config, err := PinnedConfig(to.Fingerprint)
-	if err != nil {
-		return nil, err
-	}
-
-	// The account name is the node's, and the password is the token's secret. Escaped because a name
-	// or secret containing a colon or an at-sign would otherwise change which host this connects to
-	// — a credential silently redirecting a connection is the worst shape this could take.
-	dsn := fmt.Sprintf("amqps://%s:%s@%s/",
-		url.QueryEscape(node), url.QueryEscape(secret), to.Address)
-
-	conn, err := amqp.DialConfig(dsn, amqp.Config{
-		TLSClientConfig: config,
-		Dial:            amqp.DefaultDial(timeout),
-	})
-	if err != nil {
-		if errors.Is(err, ErrWrongCertificate) {
-			return nil, err
-		}
-		// Not quoted back: the DSN carries the one-time secret.
-		return nil, fmt.Errorf("cannot reach the broker at %s as %s: %w", to.Address, node, err)
-	}
-
-	channel, err := conn.Channel()
-	if err != nil {
-		conn.Close()
-		return nil, err
-	}
-
-	// This node's own queue, which its account is scoped to and nothing else may read.
-	queue, err := channel.QueueDeclare(QueueFor(node), true, false, false, false, nil)
-	if err != nil {
-		conn.Close()
-		return nil, fmt.Errorf("cannot declare this node's queue %s: %w", QueueFor(node), err)
-	}
-
-	replies, err := channel.Consume(queue.Name, "", true, false, false, false, nil)
-	if err != nil {
-		conn.Close()
-		return nil, err
-	}
-
-	return &currentAsking{
-		conn: conn, channel: channel, queue: queue.Name, node: node, replies: replies,
-		closed: conn.NotifyClose(make(chan *amqp.Error, 1)),
-	}, nil
-}
-
-func (a *currentAsking) Close() {
-	if a.channel != nil {
-		_ = a.channel.Close()
-	}
-	if a.conn != nil {
-		_ = a.conn.Close()
-	}
-}
-
-func (a *currentAsking) Ask(ctx context.Context, request []byte, wait time.Duration) ([]byte, error) {
-	correlation := fmt.Sprintf("%s-%d", a.node, time.Now().UnixNano())
-	publish, cancel := context.WithTimeout(ctx, wait)
-	defer cancel()
-	if err := a.channel.PublishWithContext(publish, Exchange, KeyEnrol, false, false,
-		amqp.Publishing{
-			ContentType:   "application/json",
-			CorrelationId: correlation,
-			ReplyTo:       a.queue,
-			Body:          request,
-		}); err != nil {
-		return nil, fmt.Errorf("cannot publish to the %s exchange: %w", Exchange, err)
-	}
-
-	// Waited for rather than assumed. A published message that nothing answers means the control
-	// plane is not running, and a node that carried on regardless would believe it had joined a mesh
-	// that has never heard of it.
-	deadline := time.NewTimer(wait)
-	defer deadline.Stop()
-
-	for {
-		select {
-		case <-ctx.Done():
-			return nil, ctx.Err()
-		case reason := <-a.closed:
-			return nil, fmt.Errorf("the broker closed the connection: %v", reason)
-		case <-deadline.C:
-			return nil, fmt.Errorf(
-				"the broker accepted this node's connection and nothing answered within %s. The "+
-					"mesh's broker is running and its control plane is not", wait)
-		case delivery, ok := <-a.replies:
-			if !ok {
-				return nil, errors.New("the broker stopped delivering")
-			}
-			// Anything else on this queue is not the answer to this question.
-			if delivery.CorrelationId != correlation {
-				continue
-			}
-			return delivery.Body, nil
-		}
-	}
-}

internal/link/asking_nats.go

@@ -3,6 +3,7 @@ package link
 import (
 	"context"
 	"crypto/rand"
+	"crypto/sha256"
 	"encoding/hex"
 	"errors"
 	"fmt"
@@ -63,8 +64,15 @@ func presentNats(_ context.Context, to Approach, node, secret string,
 	// subscribe its own inbox and nothing else (design 25 §6). The secret is its password, the same
 	// string the request claims, so the server proves somebody holds the token and the request
 	// proves the same thing to the controller without it having to ask the server who connected.
+	// **Its own inbox space, because that is the only one it may listen in** (novox/hq
+	// 04-ISSUES/146). A JetStream publish waits for the stream's acknowledgement on an inbox the
+	// client picks, and the client's default is `_INBOX.<random>` — which this user may not
+	// subscribe to, so the enrolment failed with a permissions violation on a subject nobody had
+	// chosen. The permission is `_INBOX.enrol.<node>.>` (design 25 §6), so the client is told to
+	// pick its inboxes there; the reply address below is in the same space for the same reason.
 	conn, err := nats.Connect(natsURL(to.Address),
 		nats.Secure(config),
+		nats.CustomInboxPrefix("_INBOX.enrol."+node),
 		nats.UserInfo("enrol."+node, secret),
 		nats.Name("mesh-host/enrol/"+node),
 		nats.Timeout(timeout),
@@ -124,7 +132,19 @@ func (a *natsAsking) Ask(ctx context.Context, request []byte, wait time.Duration
 	defer cancel()
 	// Into the stream and awaited: an enrolment the bus never accepted must fail here rather than be
 	// assumed, because the node has nothing else to go on.
-	if _, err := a.js.Publish(EnrolSubject, addressed, nats.Context(publish)); err != nil {
+	//
+	// **Once, however many times it is sent** (novox/hq 04-ISSUES/146). The client re-publishes when
+	// an acknowledgement is slow, and the mesh enrolled the machine on each copy — minting a second
+	// credential, which replaced the first, which is the one the node had already been given. The
+	// machine then reconnected for ever as a user whose password the mesh had rotated out from under
+	// it, and the controller's log said "enrolled anchor" twice in the same second.
+	//
+	// The id is the message: the same bytes carry the same id, so the stream discards the client's
+	// own retry, and a genuine second attempt — which carries a new reply address — is a different
+	// message and is let through.
+	sum := sha256.Sum256(addressed)
+	if _, err := a.js.Publish(EnrolSubject, addressed,
+		nats.MsgId(hex.EncodeToString(sum[:])), nats.Context(publish)); err != nil {
 		return nil, fmt.Errorf("cannot ask the mesh to enrol this node: %w", err)
 	}
 

internal/link/bus.go

@@ -6,7 +6,6 @@ import (
 	"time"
 
 	"github.com/nats-io/nats.go"
-	amqp "github.com/rabbitmq/amqp091-go"
 )
 
 // Bus is what a host needs of the mesh's bus, in the mesh's own words.
@@ -34,22 +33,6 @@ type Bus interface {
 
 // --- The bus the mesh runs on today -----------------------------------------------------------
 
-// OverCurrent is the bus as a channel, until the rollout.
-type OverCurrent struct{ Channel *amqp.Channel }
-
-func (b OverCurrent) Report(ctx context.Context, node string, body []byte) error {
-	// Mandatory: an unroutable report comes back rather than disappearing.
-	return b.Channel.PublishWithContext(ctx, Exchange, KeyReport, true, false,
-		amqp.Publishing{ContentType: "application/json", Body: body})
-}
-
-func (b OverCurrent) Alive(ctx context.Context, node string, body []byte) error {
-	return b.Channel.PublishWithContext(ctx, Exchange, KeyAlive, false, false,
-		amqp.Publishing{ContentType: "application/json", Body: body})
-}
-
-// --- NATS ---------------------------------------------------------------------------------
-
 // OverNATS is the bus as a connection. A report goes through JetStream because it must survive
 // the controller's store restarting; a heartbeat does not, because it must not.
 type OverNATS struct {

internal/link/hearing.go

@@ -2,15 +2,15 @@ package link
 
 import (
 	"context"
+	"fmt"
 	"time"
 )
 
 // What a host hears, as the host's own words for it.
 //
-// The outbound half went behind `Bus` (bus.go) and a node's two statements stopped naming a
-// transport. This is the other half — dialling, and the declarations that arrive — and it is where
-// the transport reached furthest: the run loop selected on a channel of the client library's own
-// delivery type, so every part of holding a node in its mesh knew which bus it was on.
+// The outbound half is behind `Bus` (bus.go); this is the other half — dialling, and the
+// declarations that arrive. The run loop reads its own words for a declaration rather than the
+// client library's delivery type, so nothing past this file knows what carried it.
 //
 // **The host still imports nothing of the mesh's own** (novox/hq ADR 0005). This is its own
 // interface over its own libraries, and it agrees with the controller only because a conformance
@@ -18,8 +18,8 @@ import (
 
 // Link is this node's live connection to its mesh: what it hears, and what it says.
 //
-// One interface rather than two, because **dialling is where the transport is chosen** and choosing
-// it twice is how one half of a node ends up on a different bus from the other.
+// One interface rather than two, because dialling once is what keeps both halves of a node on the
+// same connection.
 type Link interface {
 	// Bus is what this node says: what it applied, and that it is here.
 	Bus
@@ -45,9 +45,8 @@ type Link interface {
 // because applying is reconciliation: it converges rather than repeating.
 //
 // There is one way of being done rather than two. A declaration set aside because a newer arrived
-// with it is settled exactly as an applied one is, on both buses, and the difference between them
-// is a fact the *report* carries — a second method here would be a distinction the transport does
-// not make.
+// with it is settled exactly as an applied one is, and the difference between them is a fact the
+// *report* carries — a second method here would be a distinction the bus does not make.
 type Declaration interface {
 	// Body is the signed declaration as it arrived, bytes unchanged: a node verifies what it
 	// received rather than what it re-encoded.
@@ -62,23 +61,15 @@ type Declaration interface {
 // Named Open rather than Dial because Dial is this package's raw TLS dial, which the enrolment path
 // uses to see a certificate before it trusts anything.
 //
-// **Both transports ship and this is the one place that chooses** (novox/hq ADR 0116: nothing moves
-// a node's bus before step 5). Until then every membership names the bus the mesh runs on today,
-// and the rollout is this switch and the credential behind it — not a change anywhere in the loop
-// that reads from what comes back.
+// **The mesh has one bus** (novox/hq ADR 0131): the one the broker seat delivers. A membership
+// still records which transport it was minted for, so a host can say what it is dialling, and a
+// membership recorded for anything else is a membership this host cannot use.
 func Open(ctx context.Context, m Membership, timeout time.Duration) (Link, error) {
-	switch m.Transport {
-	case OnNATS:
-		return dialNats(ctx, m, timeout)
-	default:
-		return dialCurrent(ctx, m, timeout)
+	if m.Transport != OnNATS {
+		return nil, fmt.Errorf("this membership is for %q, and the mesh's bus is %s", m.Transport, OnNATS)
 	}
+	return dialNats(ctx, m, timeout)
 }
 
-// The buses a node can be on. Empty is the one the mesh runs on today, which is every node until
-// the rollout — so a membership recorded before any of this existed reads as correct rather than as
-// unset.
-const (
-	OnCurrent = ""
-	OnNATS    = "nats"
-)
+// OnNATS is the bus a membership names: the mesh's own, and the only one.
+const OnNATS = "nats"

internal/link/hearing_current.go

@@ -1,164 +0,0 @@
-package link
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"net/url"
-	"time"
-
-	amqp "github.com/rabbitmq/amqp091-go"
-)
-
-// The host's link on the bus the mesh runs on today.
-//
-// Moved out of the run loop rather than changed: the dial, the queue, the prefetch window and the
-// return handler are what they were, because the mesh is running on this and a bus nothing speaks
-// yet is no reason to alter the one every node is on.
-
-// currentLink is this node's connection as a channel.
-type currentLink struct {
-	conn    *amqp.Connection
-	channel *amqp.Channel
-	arrived chan Declaration
-	lost    chan error
-}
-
-func dialCurrent(ctx context.Context, m Membership, timeout time.Duration) (Link, error) {
-	config, err := PinnedConfig(m.Fingerprint)
-	if err != nil {
-		return nil, err
-	}
-
-	dsn := fmt.Sprintf("amqps://%s:%s@%s/",
-		url.QueryEscape(m.Node), url.QueryEscape(m.Password), m.Broker)
-	conn, err := amqp.DialConfig(dsn, amqp.Config{
-		TLSClientConfig: config,
-		Dial:            amqp.DefaultDial(timeout),
-		// Kept short so a node that has silently lost its route notices, rather than holding a
-		// connection the broker forgot about and believing it is still in the mesh.
-		Heartbeat: 10 * time.Second,
-	})
-	if err != nil {
-		if errors.Is(err, ErrWrongCertificate) {
-			return nil, err
-		}
-		return nil, fmt.Errorf("cannot reach the broker at %s: %w", m.Broker, err)
-	}
-
-	channel, err := conn.Channel()
-	if err != nil {
-		conn.Close()
-		return nil, err
-	}
-
-	queue := QueueFor(m.Node)
-	if _, err := channel.QueueDeclare(queue, true, false, false, false, nil); err != nil {
-		conn.Close()
-		return nil, fmt.Errorf("cannot declare this node's queue %s: %w", queue, err)
-	}
-
-	// Applying is one at a time — two at once would race on the same filesystem — but SEEING is
-	// not: with a prefetch of one the host could never know that a newer declaration was already
-	// waiting, and so applied every one of a backlog in turn, at the better part of a minute each,
-	// becoming things nobody wanted any more (novox/hq issue 031). A window of unacknowledged
-	// deliveries lets it drain to the newest; each declaration still survives a restart on the
-	// broker until it is acknowledged, which happens only after it is applied or set aside.
-	if err := channel.Qos(drainDepth, 0, false); err != nil {
-		conn.Close()
-		return nil, err
-	}
-
-	deliveries, err := channel.ConsumeWithContext(ctx, queue, "", false, false, false, false, nil)
-	if err != nil {
-		conn.Close()
-		return nil, err
-	}
-
-	l := &currentLink{
-		conn: conn, channel: channel,
-		arrived: make(chan Declaration, drainDepth),
-		lost:    make(chan error, 1),
-	}
-
-	// Published mandatory, so the broker hands back anything it cannot route rather than dropping
-	// it. Without this a report goes to an exchange with no matching binding, the publisher is told
-	// nothing, and the mesh believes this node never answered while the node believes it did —
-	// which is what happened when `report` was left unbound on the other side.
-	returned := channel.NotifyReturn(make(chan amqp.Return, 4))
-	go func() {
-		for r := range returned {
-			select {
-			case l.lost <- fmt.Errorf("the broker could not route this node's %s: %s (%d %s)",
-				r.RoutingKey, r.Exchange, r.ReplyCode, r.ReplyText):
-			default:
-			}
-		}
-	}()
-
-	closed := conn.NotifyClose(make(chan *amqp.Error, 1))
-	go func() {
-		select {
-		case reason := <-closed:
-			select {
-			case l.lost <- fmt.Errorf("the link closed: %v", reason):
-			default:
-			}
-		case <-ctx.Done():
-		}
-	}()
-
-	// One goroutine turning the library's deliveries into the mesh's words, so the run loop selects
-	// on one kind of thing whichever bus it is on.
-	go func() {
-		defer close(l.arrived)
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case delivery, ok := <-deliveries:
-				if !ok {
-					select {
-					case l.lost <- errors.New("the broker stopped delivering"):
-					default:
-					}
-					return
-				}
-				select {
-				case l.arrived <- currentDeclaration{delivery}:
-				case <-ctx.Done():
-					return
-				}
-			}
-		}
-	}()
-
-	return l, nil
-}
-
-func (l *currentLink) Declarations() <-chan Declaration { return l.arrived }
-func (l *currentLink) Lost() <-chan error               { return l.lost }
-
-func (l *currentLink) Close() {
-	if l.channel != nil {
-		_ = l.channel.Close()
-	}
-	if l.conn != nil {
-		_ = l.conn.Close()
-	}
-}
-
-// Report and Alive are the outbound half, over the channel this link holds.
-func (l *currentLink) Report(ctx context.Context, node string, body []byte) error {
-	return OverCurrent{Channel: l.channel}.Report(ctx, node, body)
-}
-
-func (l *currentLink) Alive(ctx context.Context, node string, body []byte) error {
-	return OverCurrent{Channel: l.channel}.Alive(ctx, node, body)
-}
-
-// currentDeclaration is one delivery from the bus the mesh has.
-type currentDeclaration struct{ delivery amqp.Delivery }
-
-func (d currentDeclaration) Body() []byte   { return d.delivery.Body }
-func (d currentDeclaration) Handled() error { return d.delivery.Ack(false) }

internal/link/hearing_nats.go

@@ -70,7 +70,13 @@ func dialNats(ctx context.Context, m Membership, timeout time.Duration) (Link, e
 	}
 	opts := []nats.Option{
 		nats.Secure(config),
-		nats.UserInfo(m.Node, m.Password),
+		// The mesh names a machine's bus user "node.<name>" (the controller's principal scheme), and
+		// the server refused the bare name the first time a machine dialled it: "authentication
+		// error - User". The same string the mesh composed into the user list, or nothing connects.
+		nats.UserInfo("node."+m.Node, m.Password),
+		// Replies to what this client asks the server arrive on its inbox, and the mesh grants a
+		// machine exactly its own: the same prefix the user list was composed with.
+		nats.CustomInboxPrefix("_INBOX.node." + m.Node),
 		nats.Name("mesh-host/" + m.Node),
 		nats.Timeout(timeout),
 		// A node that has silently lost its route notices, rather than holding a connection the

internal/link/messages.go

@@ -110,6 +110,27 @@ type Report struct {
 	// and the mesh's up in its place, and where the found configuration's original was kept.
 	Tunnel *CarriedTunnel `json:"tunnel,omitempty"`
 
+	// Host is the version of the host that produced this report (novox/hq ADR 0141).
+	//
+	// Without it nothing can say a machine is behind, so "every machine current with its source"
+	// could not include the host — the one component the mesh did not deliver. It is a fact the
+	// machine states about itself, like the firewall it found and the links that face outside.
+	Host string `json:"host,omitempty"`
+
+	// Outward is the links on this machine that face outside it — the ones carrying a default
+	// route (novox/hq ADR 0140). Every node reports it, adopted or converged, because a converged
+	// node's filter is written around it.
+	//
+	// **It replaces a list of addresses.** The filter used to block everything passing through the
+	// machine and then allow the machine's own containers back by naming the ranges they sit on.
+	// A range describes one machine and goes stale in silence; the link carrying the default route
+	// is read afresh on every report and does not change when a module is added or removed.
+	//
+	// Empty means this machine has no route off itself. The mesh then composes no filter for it and
+	// leaves the one it has, rather than writing a rule around a link with no name — a rule set
+	// that does not load is a machine filtering nothing while its unit reports success.
+	Outward []string `json:"outward,omitempty"`
+
 	// Rekey is this node taking a found tunnel's key as its overlay key after enrolment (novox/hq
 	// ADR 0105). Not an account of the machine: a report carrying one says nothing else.
 	Rekey *Rekey `json:"rekey,omitempty"`

internal/link/messages_test.go

@@ -9,7 +9,7 @@ import (
 
 // verified runs what Run does to a delivery body, without a broker: unmarshal, check the
 // signature, and only then apply. Isolating it keeps this test about the check rather than about
-// AMQP, which is tested against a real broker in the lab.
+// the bus, which is tested against a real one in the lab.
 func verified(t *testing.T, signer ed25519.PublicKey, body []byte) (Report, bool) {
 	t.Helper()
 	applied := false

internal/link/pinned.go

@@ -73,8 +73,19 @@ func PinnedConfig(pin string) (*tls.Config, error) {
 	}, nil
 }
 
-// Dial opens a TLS connection to the broker, refusing anything but the pinned certificate.
-func Dial(address, pin string, timeout time.Duration) (*tls.Conn, error) {
+// dialPinned completes a TLS handshake against an address, refusing anything but the pinned
+// certificate.
+//
+// **Not how the bus is reached, and it used to be** (novox/hq 04-ISSUES/146). Enrolment opened one
+// of these before it said anything, which was right while the broker answered TLS immediately and
+// wrong the moment the mesh moved to a bus that speaks its own protocol first. The pin itself was
+// never the problem — PinnedConfig is what the NATS client is given, and the verification runs
+// inside the handshake that client performs.
+//
+// It stays here because this is where the pin is proven: the tests beside it run a real TLS server
+// and assert that a wrong certificate is refused before a byte of application data is sent. What it
+// must not become again is something a caller uses to reach the bus.
+func dialPinned(address, pin string, timeout time.Duration) (*tls.Conn, error) {
 	config, err := PinnedConfig(pin)
 	if err != nil {
 		return nil, err
@@ -86,7 +97,7 @@ func Dial(address, pin string, timeout time.Duration) (*tls.Conn, error) {
 		if errors.Is(err, ErrWrongCertificate) {
 			return nil, err
 		}
-		return nil, fmt.Errorf("cannot reach the broker at %s: %w", address, err)
+		return nil, fmt.Errorf("cannot reach %s: %w", address, err)
 	}
 	return conn, nil
 }

internal/link/pinned_test.go

@@ -63,7 +63,7 @@ func server(t *testing.T) (address string, fingerprint string) {
 
 func TestTheRightBrokerIsAccepted(t *testing.T) {
 	address, pin := server(t)
-	conn, err := Dial(address, pin, 5*time.Second)
+	conn, err := dialPinned(address, pin, 5*time.Second)
 	if err != nil {
 		t.Fatalf("the broker its token describes was refused: %v", err)
 	}
@@ -76,7 +76,7 @@ func TestADifferentBrokerIsRefused(t *testing.T) {
 	address, _ := server(t)
 	_, other := server(t)
 
-	_, err := Dial(address, other, 5*time.Second)
+	_, err := dialPinned(address, other, 5*time.Second)
 	if err == nil {
 		t.Fatal("a broker presenting a different certificate was accepted")
 	}
@@ -130,7 +130,7 @@ func TestNothingIsSentToTheWrongBroker(t *testing.T) {
 
 	// A pin for a certificate this server does not have.
 	_, elsewhere := server(t)
-	if _, err := Dial(listener.Addr().String(), elsewhere, 5*time.Second); err == nil {
+	if _, err := dialPinned(listener.Addr().String(), elsewhere, 5*time.Second); err == nil {
 		t.Fatal("the impostor was accepted")
 	}
 	if n := <-received; n > 0 {
@@ -160,7 +160,7 @@ func TestAnUnreachableBrokerIsAnOrdinaryFailure(t *testing.T) {
 	address := listener.Addr().String()
 	listener.Close()
 
-	_, err = Dial(address, pin, 2*time.Second)
+	_, err = dialPinned(address, pin, 2*time.Second)
 	if err == nil {
 		t.Fatal("dialling a closed port succeeded")
 	}

internal/link/run.go

@@ -30,9 +30,8 @@ type Membership struct {
 	Fingerprint string
 	Password    string
 	Signer      ed25519.PublicKey
-	// Transport is which bus this node speaks (hearing.go). Empty is the one the mesh runs on
-	// today, which is every node until the rollout — so a membership recorded before any of this
-	// existed reads as correct rather than as unset.
+	// Transport is which bus this membership was minted for (hearing.go): the mesh's own, and a
+	// membership that names another is one this host cannot dial with.
 	Transport string
 }
 

internal/outward/links.go

@@ -0,0 +1,147 @@
+// Package outward reads which of this machine's links face outside it (novox/hq ADR 0140).
+//
+// The filter the mesh derives constrains traffic arriving from outside the machine and says nothing
+// about traffic that did not. To write that rule the mesh has to know which links "outside" arrives
+// on, and that is a thing only the machine can say — so it says it, once per report, the way it
+// already reports the kind of firewall it found and the tunnel it carried.
+//
+// **It replaces a list of addresses.** The filter used to allow the machine's own containers back
+// through by naming the address ranges they sit on: two ranges fixed in the control plane's source
+// and the rest typed by an operator. A range describes one machine and goes stale silently
+// (novox/hq 04-ISSUES/137 and /141). A link that carries the default route is a fact the machine
+// reads afresh every time, and it does not change when a module is added or removed.
+//
+// It reads the kernel's routing tables directly rather than asking a program. A module naming a
+// program the machine does not have is how the mesh already reported success while doing nothing
+// (novox/hq 04-ISSUES/136), and every machine has /proc.
+package outward
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// ProcNet is where the kernel publishes its routing tables. A parameter so a test can hold a
+// routing table without one.
+const ProcNet = "/proc/net"
+
+// Links are the interfaces carrying a default route, for both address families, sorted and without
+// repeats.
+//
+// A machine may have more than one: a laptop with a cable and a radio has two, and both face
+// outside. A machine with none — no route off itself — returns nothing, and the mesh refuses to
+// compose a filter for it rather than writing a rule around a link with no name, which would be a
+// rule set that does not load and a machine filtering nothing while its unit reports success.
+func Links(procNet string) ([]string, error) {
+	if procNet == "" {
+		procNet = ProcNet
+	}
+	seen := map[string]bool{}
+
+	four, err := defaultsV4(filepath.Join(procNet, "route"))
+	if err != nil {
+		return nil, err
+	}
+	six, err := defaultsV6(filepath.Join(procNet, "ipv6_route"))
+	if err != nil {
+		return nil, err
+	}
+	for _, name := range append(four, six...) {
+		if name != "" && name != "lo" {
+			seen[name] = true
+		}
+	}
+
+	out := make([]string, 0, len(seen))
+	for name := range seen {
+		out = append(out, name)
+	}
+	sort.Strings(out)
+	return out, nil
+}
+
+// defaultsV4 reads /proc/net/route, whose columns are
+//
+//	Iface Destination Gateway Flags RefCnt Use Metric Mask ...
+//
+// with addresses in hexadecimal. A default route is destination zero with mask zero — the mask
+// matters, because a route to the zero address with a real mask is not a default route.
+func defaultsV4(path string) ([]string, error) {
+	lines, err := rows(path)
+	if err != nil {
+		return nil, err
+	}
+	var out []string
+	for _, fields := range lines {
+		if len(fields) < 8 {
+			continue
+		}
+		if isZeroHex(fields[1]) && isZeroHex(fields[7]) {
+			out = append(out, fields[0])
+		}
+	}
+	return out, nil
+}
+
+// defaultsV6 reads /proc/net/ipv6_route, whose columns are
+//
+//	dest destprefix src srcprefix nexthop metric refcnt use flags iface
+//
+// A default route is the zero destination with a zero prefix length.
+func defaultsV6(path string) ([]string, error) {
+	lines, err := rows(path)
+	if err != nil {
+		return nil, err
+	}
+	var out []string
+	for _, fields := range lines {
+		if len(fields) < 10 {
+			continue
+		}
+		if isZeroHex(fields[0]) && isZeroHex(fields[1]) {
+			out = append(out, fields[9])
+		}
+	}
+	return out, nil
+}
+
+// rows reads a routing table into fields per line, skipping a header and blank lines. A table that
+// is not there is not an error: a machine without the second address family has no file for it,
+// and that is not a machine that cannot be filtered.
+func rows(path string) ([][]string, error) {
+	file, err := os.Open(path)
+	if os.IsNotExist(err) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("cannot read the routing table at %s: %w", path, err)
+	}
+	defer file.Close()
+
+	var out [][]string
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" || strings.HasPrefix(line, "Iface") {
+			continue
+		}
+		out = append(out, strings.Fields(line))
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("cannot read the routing table at %s: %w", path, err)
+	}
+	return out, nil
+}
+
+// isZeroHex is whether a hexadecimal field is all zeroes, whatever its width — the v4 table writes
+// eight digits and the v6 table thirty-two, and a prefix length is two.
+func isZeroHex(field string) bool {
+	if field == "" {
+		return false
+	}
+	return strings.Trim(strings.ToLower(field), "0") == ""
+}

internal/outward/links_test.go

@@ -0,0 +1,120 @@
+package outward
+
+import (
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+)
+
+// A routing table as the kernel writes it: a default route, a route to the zero address that is not
+// one, and a route on the loopback. Only the default route's link faces outside.
+const routeV4 = `Iface	Destination	Gateway 	Flags	RefCnt	Use	Metric	Mask		MTU	Window	IRTT
+enp9s0	00000000	01FEA8C0	0003	0	0	100	00000000	0	0	0
+docker0	000011AC	00000000	0001	0	0	0	0000FFFF	0	0	0
+enp9s0	00000000	00000000	0001	0	0	100	00FFFFFF	0	0	0
+lo	00000000	00000000	0003	0	0	0	00000000	0	0	0
+`
+
+const routeV6 = `00000000000000000000000000000000 00 00000000000000000000000000000000 00 fe800000000000000000000000000001 00000400 00000001 00000000 00000003     wlan0
+fd0000000000000000000000000000000 40 00000000000000000000000000000000 00 00000000000000000000000000000000 00000100 00000000 00000000 00000001    enp9s0
+`
+
+func write(t *testing.T, dir, name, body string) {
+	t.Helper()
+	if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestLinksAreTheOnesCarryingADefaultRoute(t *testing.T) {
+	dir := t.TempDir()
+	write(t, dir, "route", routeV4)
+	write(t, dir, "ipv6_route", routeV6)
+
+	got, err := Links(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// The cable from the v4 table and the radio from the v6 one. Not docker0, whose route is not a
+	// default; not the loopback, which faces nothing; and not the v6 route with a real prefix.
+	want := []string{"enp9s0", "wlan0"}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("outward links are %v, want %v", got, want)
+	}
+}
+
+// A route to the zero address with a real mask is not a default route. Trusting the destination
+// alone would name every link with such a route as facing outside, and a filter that treats an
+// internal bridge as outward constrains this machine's own guests — the fault ADR 0140 removes.
+func TestAZeroDestinationWithAMaskIsNotADefaultRoute(t *testing.T) {
+	dir := t.TempDir()
+	write(t, dir, "route", `Iface	Destination	Gateway 	Flags	RefCnt	Use	Metric	Mask		MTU	Window	IRTT
+br-abc	00000000	00000000	0001	0	0	0	00FFFFFF	0	0	0
+`)
+	got, err := Links(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 0 {
+		t.Fatalf("outward links are %v, want none", got)
+	}
+}
+
+// A machine with no route off itself says so, rather than guessing. The mesh refuses to compose a
+// filter for it; a rule written around a link with no name does not load, and a rule set that does
+// not load is a machine filtering nothing while its unit reports success.
+func TestNoDefaultRouteIsNoLinks(t *testing.T) {
+	dir := t.TempDir()
+	write(t, dir, "route", "Iface\tDestination\tGateway \tFlags\tRefCnt\tUse\tMetric\tMask\t\tMTU\tWindow\tIRTT\n")
+	got, err := Links(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 0 {
+		t.Fatalf("outward links are %v, want none", got)
+	}
+}
+
+// A machine without the second address family has no file for it. That is not a machine that cannot
+// be filtered, so a missing table is read as no routes rather than as a failure.
+func TestAMissingTableIsNotAFailure(t *testing.T) {
+	dir := t.TempDir()
+	write(t, dir, "route", routeV4)
+	got, err := Links(dir)
+	if err != nil {
+		t.Fatalf("a missing v6 table should not fail: %v", err)
+	}
+	if !reflect.DeepEqual(got, []string{"enp9s0"}) {
+		t.Fatalf("outward links are %v, want [enp9s0]", got)
+	}
+}
+
+// The same link carrying a default route in both families is reported once.
+func TestALinkIsReportedOnce(t *testing.T) {
+	dir := t.TempDir()
+	write(t, dir, "route", routeV4)
+	write(t, dir, "ipv6_route",
+		"00000000000000000000000000000000 00 00000000000000000000000000000000 00 "+
+			"fe800000000000000000000000000001 00000400 00000001 00000000 00000003     enp9s0\n")
+	got, err := Links(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(got, []string{"enp9s0"}) {
+		t.Fatalf("outward links are %v, want [enp9s0]", got)
+	}
+}
+
+// Against this machine's own routing table, so the parse is held to what the kernel actually writes
+// and not only to a fixture written to agree with it.
+func TestAgainstThisMachinesOwnTable(t *testing.T) {
+	got, err := Links("")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) == 0 {
+		t.Skip("this machine has no default route")
+	}
+	t.Logf("this machine's outward links: %v", got)
+}

internal/upgrade/upgrade.go

@@ -16,7 +16,9 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
+	"time"
 )
 
 // Files the launcher reads and this binary writes. Next to the store, because they are node
@@ -157,3 +159,168 @@ func ReadKnownGood(path string) (string, error) {
 	}
 	return strings.TrimSpace(string(raw)), nil
 }
+
+// Where delivered versions live, and what the binary inside one is called.
+//
+// **A directory named for its version, never a link and never a write over what is running**
+// (novox/hq ADR 0141). Two facts follow from that one choice: the kernel refuses to truncate a
+// running executable, so the path a delivery writes must not be the path being executed; and a
+// rollback needs the previous version still present, which a single path cannot offer.
+//
+// The mesh creates no links (novox/hq ADR 0012), so nothing points at "current". The version is in
+// the path, which is why nothing has to be told what is running.
+const (
+	// DefaultLibexec is where the host's own files live. Fixed rather than derived from where the
+	// running executable sits: the first host to understand any of this was copied to a machine by
+	// hand, and one that looked for its successor beside itself would never find a delivered version
+	// — which is every machine in this mesh on the day this ships.
+	DefaultLibexec  = "/usr/lib/nox-mesh-host"
+	VersionsDirName = "versions"
+	BinaryName      = "nox-mesh-host"
+	// PinnedName is the version a rollback chose, which the launcher runs instead of the newest.
+	// Without it the launcher would start the newest again and the rollback would flap.
+	PinnedName = "rollback-pinned"
+)
+
+// VersionsDir is where delivered versions live, given where the host's libexec is. An empty libexec
+// means the default, and the environment overrides it so a test needs no root.
+func VersionsDir(libexec string) string {
+	if libexec == "" {
+		libexec = os.Getenv("MESH_HOST_LIBEXEC")
+	}
+	if libexec == "" {
+		libexec = DefaultLibexec
+	}
+	return filepath.Join(libexec, VersionsDirName)
+}
+
+// PinnedPath is where a rollback records the version it chose.
+func PinnedPath(statePath string) string {
+	return filepath.Join(filepath.Dir(statePath), PinnedName)
+}
+
+// Delivered is one version present on the machine.
+type Delivered struct {
+	// Version is the directory's name, which is the version.
+	Version string
+	// Binary is the executable inside it.
+	Binary string
+	// At is when it arrived, which is how "newest" is decided.
+	At time.Time
+}
+
+// Versions are the versions delivered to this machine, newest first.
+//
+// **Newest by when it arrived, not by its name.** A version string comes from what the source was
+// tagged or described as, and those do not sort: "1.10" before "1.9", a commit hash before either.
+// Ordering by name would run an older host and call it an upgrade. When it arrived is a fact the
+// filesystem keeps and the delivery sets.
+//
+// A directory with no executable in it is not a version. A delivery that was interrupted leaves one,
+// and running the newest would then mean running nothing.
+func Versions(dir string) ([]Delivered, error) {
+	entries, err := os.ReadDir(dir)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("cannot read the delivered versions at %s: %w", dir, err)
+	}
+
+	var out []Delivered
+	for _, entry := range entries {
+		if !entry.IsDir() {
+			continue
+		}
+		binary := filepath.Join(dir, entry.Name(), BinaryName)
+		info, err := os.Stat(binary)
+		if err != nil || info.IsDir() {
+			continue
+		}
+		at := info.ModTime()
+		if d, err := entry.Info(); err == nil && d.ModTime().After(at) {
+			at = d.ModTime()
+		}
+		out = append(out, Delivered{Version: entry.Name(), Binary: binary, At: at})
+	}
+
+	// Newest first, and by name when two arrived in the same instant so the answer is never
+	// arbitrary — a test that passes half the time is worse than one that fails.
+	sort.Slice(out, func(a, b int) bool {
+		if out[a].At.Equal(out[b].At) {
+			return out[a].Version > out[b].Version
+		}
+		return out[a].At.After(out[b].At)
+	})
+	return out, nil
+}
+
+// Successor is the version this machine should be running instead of the given one, if any.
+//
+// Empty when the running version is the newest, which is the ordinary answer. The host asks this
+// between reconciles and nowhere else: standing aside mid-apply is the half-configured machine the
+// host exists to prevent (novox/hq ADR 0141).
+func Successor(dir, running string) (Delivered, bool, error) {
+	delivered, err := Versions(dir)
+	if err != nil {
+		return Delivered{}, false, err
+	}
+	if len(delivered) == 0 {
+		return Delivered{}, false, nil
+	}
+	newest := delivered[0]
+	// A machine whose running version is not among the delivered ones is the machine every mesh has
+	// one of: the host was put there by hand before any of this existed. Treating that as "stand
+	// aside" is correct — what was delivered is what the mesh asked for.
+	if newest.Version == running {
+		return Delivered{}, false, nil
+	}
+	return newest, true, nil
+}
+
+// Retire removes delivered versions older than the running one's predecessor.
+//
+// The running version and the one before it are kept, and nothing else: the predecessor is exactly
+// what a rollback starts, and every version before that is weight with no reader. Called after a
+// reconcile completes, which is the same evidence known-good is written on — retiring on any weaker
+// signal would delete the thing a failing host is about to need.
+//
+// Never the running version, whatever it is asked. A host that deleted its own image would survive
+// until it stopped and then be unstartable, and the launcher's rollback reads a version, not a
+// process.
+func Retire(dir, running string) ([]string, error) {
+	delivered, err := Versions(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	keep := map[string]bool{running: true}
+	for i, d := range delivered {
+		if d.Version != running {
+			continue
+		}
+		// Its predecessor is the next one down the list, which is the next oldest.
+		if i+1 < len(delivered) {
+			keep[delivered[i+1].Version] = true
+		}
+		break
+	}
+	// A running version that was never delivered has no predecessor among these, so the newest
+	// delivered one is what a rollback would reach for. Keep it.
+	if len(keep) == 1 && len(delivered) > 0 {
+		keep[delivered[0].Version] = true
+	}
+
+	var removed []string
+	for _, d := range delivered {
+		if keep[d.Version] {
+			continue
+		}
+		if err := os.RemoveAll(filepath.Join(dir, d.Version)); err != nil {
+			return removed, fmt.Errorf("cannot retire the host version %s: %w", d.Version, err)
+		}
+		removed = append(removed, d.Version)
+	}
+	sort.Strings(removed)
+	return removed, nil
+}

internal/upgrade/versions_test.go

@@ -0,0 +1,180 @@
+package upgrade
+
+import (
+	"os"
+	"path/filepath"
+	"reflect"
+	"sort"
+	"testing"
+	"time"
+)
+
+// deliver writes a version as a delivery would: a directory named for it with the binary inside.
+// at fixes when it arrived, because "newest" is when it arrived and a test must not race the clock.
+func deliver(t *testing.T, dir, version string, at time.Time) string {
+	t.Helper()
+	into := filepath.Join(dir, version)
+	if err := os.MkdirAll(into, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	binary := filepath.Join(into, BinaryName)
+	if err := os.WriteFile(binary, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chtimes(binary, at, at); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chtimes(into, at, at); err != nil {
+		t.Fatal(err)
+	}
+	return binary
+}
+
+// **Newest is when it arrived, not how its name sorts.**
+//
+// A version string is whatever the source was tagged or described as, and those do not sort: "1.10"
+// orders before "1.9", and a commit hash orders before either. Ordering by name would start an older
+// host and call that an upgrade.
+func TestNewestIsWhenItArrivedAndNotHowItSorts(t *testing.T) {
+	dir := t.TempDir()
+	base := time.Now().Add(-time.Hour)
+	deliver(t, dir, "1.10", base)                 // sorts LAST by name, arrived first
+	deliver(t, dir, "1.9", base.Add(time.Minute)) // sorts first by name, arrived last
+
+	got, err := Versions(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 2 || got[0].Version != "1.9" {
+		t.Fatalf("newest is %+v, want the one that arrived last (1.9)", got)
+	}
+}
+
+// A delivery that was interrupted leaves a directory with no executable in it. Running "the newest"
+// would then mean running nothing, so it is not a version.
+func TestADirectoryWithNoBinaryIsNotAVersion(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(dir, "half-delivered"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	deliver(t, dir, "good", time.Now().Add(-time.Hour))
+
+	got, err := Versions(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 1 || got[0].Version != "good" {
+		t.Fatalf("versions are %+v, want only the one with a binary", got)
+	}
+}
+
+// Nothing delivered is not a fault. A machine whose host was placed by hand has no versions
+// directory at all, and that must read as "no successor" rather than as an error that stops a
+// reconcile.
+func TestNoVersionsDirectoryIsNotAnError(t *testing.T) {
+	got, err := Versions(filepath.Join(t.TempDir(), "absent"))
+	if err != nil {
+		t.Fatalf("an absent versions directory should not be an error: %v", err)
+	}
+	if len(got) != 0 {
+		t.Fatalf("versions are %+v, want none", got)
+	}
+}
+
+func TestTheNewestVersionIsTheSuccessorAndTheRunningOneIsNot(t *testing.T) {
+	dir := t.TempDir()
+	base := time.Now().Add(-time.Hour)
+	deliver(t, dir, "one", base)
+	deliver(t, dir, "two", base.Add(time.Minute))
+
+	next, yes, err := Successor(dir, "one")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !yes || next.Version != "two" {
+		t.Fatalf("successor is %+v (%v), want two", next, yes)
+	}
+
+	if _, yes, err := Successor(dir, "two"); err != nil || yes {
+		t.Fatalf("the newest version is its own successor (%v, %v)", yes, err)
+	}
+}
+
+// A host put there by hand, before any of this existed, is not among the delivered versions. What the
+// mesh delivered is what it asked for, so that is a successor — otherwise the first delivery to such a
+// machine would be ignored for ever, which is every machine in this mesh today.
+func TestAHostThatWasNeverDeliveredHasASuccessor(t *testing.T) {
+	dir := t.TempDir()
+	deliver(t, dir, "delivered", time.Now().Add(-time.Hour))
+
+	next, yes, err := Successor(dir, "copied-by-hand")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !yes || next.Version != "delivered" {
+		t.Fatalf("successor is %+v (%v), want the delivered one", next, yes)
+	}
+}
+
+// The running version and its predecessor are kept, and nothing else. The predecessor is exactly what
+// a rollback starts; everything older has no reader.
+func TestRetireKeepsTheRunningVersionAndItsPredecessor(t *testing.T) {
+	dir := t.TempDir()
+	base := time.Now().Add(-4 * time.Hour)
+	for i, v := range []string{"one", "two", "three", "four"} {
+		deliver(t, dir, v, base.Add(time.Duration(i)*time.Hour))
+	}
+
+	removed, err := Retire(dir, "four")
+	if err != nil {
+		t.Fatal(err)
+	}
+	sort.Strings(removed)
+	if !reflect.DeepEqual(removed, []string{"one", "two"}) {
+		t.Fatalf("retired %v, want one and two — three is the predecessor a rollback needs", removed)
+	}
+	for _, kept := range []string{"three", "four"} {
+		if _, err := os.Stat(filepath.Join(dir, kept, BinaryName)); err != nil {
+			t.Fatalf("%s was retired and a rollback now has nowhere to go: %v", kept, err)
+		}
+	}
+}
+
+// **Never the running version, whatever it is asked.** A host that deleted its own image would run
+// until it stopped and then be unstartable, and the launcher's rollback reads a version rather than a
+// process.
+func TestRetireNeverRemovesTheRunningVersion(t *testing.T) {
+	dir := t.TempDir()
+	base := time.Now().Add(-2 * time.Hour)
+	deliver(t, dir, "older", base)
+	deliver(t, dir, "newer", base.Add(time.Hour))
+
+	// Asked while running the OLDER one, which is what a machine looks like between a delivery and
+	// the moment it stands aside.
+	if _, err := Retire(dir, "older"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "older", BinaryName)); err != nil {
+		t.Fatalf("the running version was retired: %v", err)
+	}
+}
+
+// A machine running a hand-placed host keeps the newest delivered version, because that is what a
+// rollback would reach for. Retiring it would leave the machine with no way back at all.
+func TestRetireKeepsTheNewestWhenTheRunningVersionWasNeverDelivered(t *testing.T) {
+	dir := t.TempDir()
+	base := time.Now().Add(-3 * time.Hour)
+	deliver(t, dir, "old", base)
+	deliver(t, dir, "new", base.Add(time.Hour))
+
+	removed, err := Retire(dir, "copied-by-hand")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(removed, []string{"old"}) {
+		t.Fatalf("retired %v, want only old — new is the rollback target", removed)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "new", BinaryName)); err != nil {
+		t.Fatalf("the only delivered version was retired: %v", err)
+	}
+}

packaging/launch_test.sh

@@ -187,5 +187,70 @@ sleep 1
 check "a crash is counted" "unlike a clean exit, which is not" "$(count)" "1"
 kill -TERM "$LP" 2>/dev/null; sleep 1; pkill -f "$MESH_HOST_BIN" 2>/dev/null || true
 
+# --- which version it runs (novox/hq ADR 0141) ------------------------------------------------
+#
+# Versions live side by side in directories named for them. The launcher picks one every time round
+# the loop, never once: standing aside for a successor is a clean exit, and the next turn has to run
+# what is on disk NOW — resolved once, the same binary would restart for ever and no upgrade would
+# ever take.
+
+# deliver a version as the mesh would, recording which one ran so a test can assert the choice.
+deliver() {
+	mkdir -p "$MESH_HOST_LIBEXEC/versions/$1"
+	cat > "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host" <<STUB
+#!/bin/sh
+echo "$1" >> "\$MESH_HOST_STATE_DIR/which.ran"
+exit "\${STUB_HOST_EXIT:-1}"
+STUB
+	chmod +x "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host"
+	# When it arrived is what "newest" means, so it is set rather than left to the clock.
+	touch -d "$2" "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host" "$MESH_HOST_LIBEXEC/versions/$1"
+}
+which_ran() { cat "$MESH_HOST_STATE_DIR/which.ran" 2>/dev/null || echo NONE; }
+
+# Newest is when it arrived, not how its name sorts: "1.10" orders before "1.9" by name, so ordering
+# by name would run an older host and call it an upgrade.
+setup
+deliver 1.10 "2 hours ago"
+deliver 1.9 "1 hour ago"
+"$LAUNCH" >/dev/null 2>&1 || true
+check "runs the newest delivered version" "newest is when it arrived, not how the name sorts" \
+	"$(which_ran)" "1.9"
+
+# A pin from a rollback beats the newest, or the launcher would start the failing binary again and
+# the rollback would flap.
+setup
+deliver 1.9 "2 hours ago"
+deliver 2.0 "1 hour ago"
+echo 1.9 > "$MESH_HOST_STATE_DIR/rollback-pinned"
+"$LAUNCH" >/dev/null 2>&1 || true
+check "a pinned version beats the newest" "otherwise a rollback starts the binary it just rejected" \
+	"$(which_ran)" "1.9"
+
+# A pin naming a version that is not there is ignored rather than fatal: the machine choosing for
+# itself is better than a machine that starts nothing.
+setup
+deliver 2.0 "1 hour ago"
+echo 1.9 > "$MESH_HOST_STATE_DIR/rollback-pinned"
+"$LAUNCH" >/dev/null 2>&1 || true
+check "an undeliverable pin is ignored" "a machine that starts nothing is worse than one that chooses" \
+	"$(which_ran)" "2.0"
+
+# An interrupted delivery leaves a directory with no binary in it. Treating it as the newest would
+# mean running nothing.
+setup
+deliver 1.9 "2 hours ago"
+mkdir -p "$MESH_HOST_LIBEXEC/versions/2.0-half"
+touch -d "1 minute ago" "$MESH_HOST_LIBEXEC/versions/2.0-half"
+"$LAUNCH" >/dev/null 2>&1 || true
+check "skips a version with no binary" "a directory is not a version; the binary is" \
+	"$(which_ran)" "1.9"
+
+# Nothing delivered: the host placed by hand, which is how the first one always arrives. Without this
+# the change would strand every machine in the mesh on the day it ships.
+setup
+"$LAUNCH" >/dev/null 2>&1 || true
+check "falls back to the host placed by hand" "every first host arrives this way" "$(started)" "yes"
+
 printf '\nlaunch: %d passed, %d failed\n' "$PASS" "$FAIL"
 [ "$FAIL" -eq 0 ]

packaging/nox-mesh-host-launch

@@ -16,16 +16,51 @@ set -u
 
 STATE_DIR="${MESH_HOST_STATE_DIR:-/var/lib/mesh-host}"
 LIBEXEC="${MESH_HOST_LIBEXEC:-/usr/lib/nox-mesh-host}"
-HOST="${MESH_HOST_BIN:-/usr/bin/nox-mesh-host}"
+# The host that was placed by hand, used only when nothing has been delivered. The first host on a
+# machine always arrives this way; every one after it is delivered (novox/hq ADR 0141).
+FALLBACK="${MESH_HOST_BIN:-/usr/bin/nox-mesh-host}"
+VERSIONS="$LIBEXEC/versions"
+BINARY="nox-mesh-host"
 LIMIT="${MESH_HOST_START_LIMIT:-3}"
 BACKOFF="${MESH_HOST_BACKOFF:-5}"
 ONCE="${MESH_HOST_RUN_ONCE:-}"   # tests run one iteration; nothing else sets this
 
 ATTEMPTS="$STATE_DIR/start-attempts"
 HALTED="$STATE_DIR/halted"
+PINNED="$STATE_DIR/rollback-pinned"
 
 say() { echo "nox-mesh-host-launch: $*" >&2; }
 
+# Which host to run: the version a rollback pinned, or the most recently delivered one, or the one
+# placed by hand when nothing has been delivered (novox/hq ADR 0141).
+#
+# **Asked every time round the loop, not once.** Standing aside for a successor is a clean exit, and
+# the next turn has to run what is on disk NOW — resolving this once would restart the same binary
+# for ever and the upgrade would never take.
+#
+# Newest by when it arrived, never by how its name sorts: a version string is whatever the source was
+# described as, and those do not sort — "1.10" orders before "1.9". Ordering by name would start an
+# older host and call it an upgrade.
+pick_host() {
+	if [ -s "$PINNED" ]; then
+		pinned="$(tr -d '[:space:]' < "$PINNED" 2>/dev/null || true)"
+		if [ -n "$pinned" ] && [ -x "$VERSIONS/$pinned/$BINARY" ]; then
+			echo "$VERSIONS/$pinned/$BINARY"
+			return 0
+		fi
+		say "the pinned version '$pinned' is not delivered; ignoring the pin"
+	fi
+	# A directory with no executable in it is not a version: an interrupted delivery leaves one, and
+	# running "the newest" would then mean running nothing.
+	for candidate in $(ls -1t "$VERSIONS" 2>/dev/null || true); do
+		if [ -x "$VERSIONS/$candidate/$BINARY" ]; then
+			echo "$VERSIONS/$candidate/$BINARY"
+			return 0
+		fi
+	done
+	echo "$FALLBACK"
+}
+
 child=
 stopping=
 
@@ -95,6 +130,14 @@ while :; do
 		fi
 	fi
 
+	HOST="$(pick_host)"
+	if [ ! -x "$HOST" ]; then
+		say "no host to run: nothing delivered under $VERSIONS and $FALLBACK is not executable."
+		printf 'no host binary\n' > "$HALTED"
+		exit 0
+	fi
+	say "running $HOST"
+
 	"$HOST" run &
 	child=$!
 	status=0

packaging/nox-mesh-host-rollback

@@ -1,20 +1,29 @@
 #!/bin/sh
 # Put the host back on the last version that worked.
 #
-# novox/hq ADR 0005. This runs when nox-mesh-host will not start, so it shares no code with it
-# and calls none of it: a binary that cannot start cannot be its own recovery. POSIX sh, no
+# novox/hq ADR 0005 and ADR 0141. This runs when nox-mesh-host will not start, so it shares no code
+# with it and calls none of it: a binary that cannot start cannot be its own recovery. POSIX sh, no
 # bashisms, nothing that has to be installed.
 #
-# It is deliberately dull. Everything it does is one of: read a file, run the package manager,
-# ask the service manager to try again.
+# It is deliberately dull. Everything it does is one of: read a file, look at a directory, write a
+# file.
+#
+# **It used to reinstall a package.** It read the known-good version and asked one operating system's
+# package manager for it, out of that package manager's cache. Two things were wrong with that. No
+# machine in this mesh had the host installed as a package, so the recovery could not run on any of
+# them; and the host is built per operating system (ADR 0005), so a recovery written in one package
+# manager's terms could not run on two of the three. Versions now live side by side in directories
+# named for them, so going back is choosing a directory — which is the same on every machine.
 set -eu
 
 STATE_DIR="${MESH_HOST_STATE_DIR:-/var/lib/mesh-host}"
-PKG_CACHE="${MESH_HOST_PKG_CACHE:-/var/cache/pacman/pkg}"
-PACKAGE="${MESH_HOST_PACKAGE:-nox-mesh-host}"
+LIBEXEC="${MESH_HOST_LIBEXEC:-/usr/lib/nox-mesh-host}"
+VERSIONS="$LIBEXEC/versions"
+BINARY="nox-mesh-host"
 
 KNOWN_GOOD="$STATE_DIR/known-good"
 ATTEMPTED="$STATE_DIR/rollback-attempted"
+PINNED="$STATE_DIR/rollback-pinned"
 
 say() { echo "nox-mesh-host-rollback: $*" >&2; }
 
@@ -43,23 +52,24 @@ if [ -z "$VERSION" ]; then
 	exit 0
 fi
 
-PKG="$(ls "$PKG_CACHE"/"$PACKAGE"-"$VERSION"-*.pkg.tar.* 2>/dev/null | head -n 1 || true)"
-if [ -z "$PKG" ]; then
-	say "known-good is $VERSION and no package for it is in $PKG_CACHE."
-	say "the cache was cleaned, or that version was never installed from here."
+# The version that last worked may be the one that was placed by hand, which is not delivered and has
+# no directory. Nothing to choose, and saying so is better than pinning a version that is not there —
+# the launcher would ignore the pin and start the newest again, which is the binary that is failing.
+if [ ! -x "$VERSIONS/$VERSION/$BINARY" ]; then
+	say "known-good is $VERSION and no such version is delivered under $VERSIONS."
+	say "it was retired, or that host was placed by hand and never delivered."
 	say "cannot roll back. this node needs a person."
 	exit 1
 fi
 
-say "rolling back to $VERSION ($PKG)"
+say "rolling back to $VERSION ($VERSIONS/$VERSION/$BINARY)"
 printf '%s\n' "$VERSION" > "$ATTEMPTED"
 
-if ! pacman -U --noconfirm "$PKG"; then
-	say "the package manager refused to install $PKG."
-	exit 1
-fi
+# The pin is what stops the launcher starting the newest again. Written last, so a failure above
+# leaves the machine choosing for itself rather than pinned to something this script did not verify.
+printf '%s\n' "$VERSION" > "$PINNED"
 
-# Deliberately does NOT start anything. The launcher called this and will exec the host next,
-# so starting it here would run two. novox/hq ADR 0005 moved that responsibility; this script
-# installs a version and says so, and nothing else.
-say "rolled back to $VERSION. the launcher will start it."
+# Deliberately does NOT start anything. The launcher called this and will run the host next, so
+# starting it here would run two. novox/hq ADR 0005 moved that responsibility; this script chooses a
+# version and says so, and nothing else.
+say "pinned $VERSION. the launcher will start it."

packaging/rollback_test.sh

@@ -1,9 +1,15 @@
 #!/bin/sh
 # Tests for nox-mesh-host-rollback.
 #
-# It runs on a machine where the host will not start, which is the one moment nobody can afford
-# it to be wrong — and the one moment it is hardest to debug. So it is tested here, against a
-# real filesystem, with a stub package manager that records what it was asked to do.
+# It runs on a machine where the host will not start, which is the one moment nobody can afford it to
+# be wrong — and the one moment it is hardest to debug. So it is tested here, against a real
+# filesystem holding real delivered versions.
+#
+# **These used to stub a package manager.** The script reinstalled the known-good version with
+# `pacman -U` out of the package cache, which no machine in this mesh used and which two of the three
+# operating systems the host is built for do not have (novox/hq ADR 0141). Going back is now choosing
+# a directory, so there is nothing to stub: the thing under test is the filesystem, and a fake would
+# only assert that the fake behaves as expected (novox/hq ADR 0017).
 set -eu
 cd "$(dirname "$0")"
 SCRIPT="$PWD/nox-mesh-host-rollback"
@@ -12,26 +18,15 @@ PASS=0; FAIL=0
 setup() {
 	WORK="$(mktemp -d)"
 	export MESH_HOST_STATE_DIR="$WORK/state"
-	export MESH_HOST_PKG_CACHE="$WORK/cache"
-	export MESH_HOST_PACKAGE="nox-mesh-host"
-	mkdir -p "$MESH_HOST_STATE_DIR" "$MESH_HOST_PKG_CACHE" "$WORK/bin"
+	export MESH_HOST_LIBEXEC="$WORK/libexec"
+	mkdir -p "$MESH_HOST_STATE_DIR" "$MESH_HOST_LIBEXEC/versions"
+}
 
-	# Stubs on PATH. Not mocks of the script's own logic — the boundary is real commands, and
-	# these record the calls so a test can assert what the script asked the machine to do.
-	cat > "$WORK/bin/pacman" <<'STUB'
-#!/bin/sh
-echo "$@" >> "$MESH_HOST_STATE_DIR/pacman.calls"
-[ -n "${STUB_PACMAN_FAILS:-}" ] && exit 1
-exit 0
-STUB
-	cat > "$WORK/bin/systemctl" <<'STUB'
-#!/bin/sh
-echo "$@" >> "$MESH_HOST_STATE_DIR/systemctl.calls"
-exit 0
-STUB
-	chmod +x "$WORK/bin/pacman" "$WORK/bin/systemctl"
-	PATH="$WORK/bin:$PATH"; export PATH
-	unset STUB_PACMAN_FAILS || true
+# deliver a version the way the mesh would: a directory named for it, with the binary inside.
+deliver() {
+	mkdir -p "$MESH_HOST_LIBEXEC/versions/$1"
+	printf '#!/bin/sh\nexit 0\n' > "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host"
+	chmod +x "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host"
 }
 
 check() { # name, condition-description, actual, expected
@@ -41,32 +36,38 @@ check() { # name, condition-description, actual, expected
 
 # --- a normal rollback ---------------------------------------------------------------------
 setup
-echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
-touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst"
-"$SCRIPT" >/dev/null 2>&1
-check "installs the known-good version" "pacman is asked to install the cached package" \
-	"$(grep -c 'nox-mesh-host-1.4.2' "$MESH_HOST_STATE_DIR/pacman.calls" 2>/dev/null || echo 0)" "1"
-# It installs and stops. The launcher execs the host next, and starting it here would run two
-# (novox/hq ADR 0005).
-check "does not start anything itself" "the launcher owns starting" \
-	"$([ -f "$MESH_HOST_STATE_DIR/systemctl.calls" ] && echo started || echo not-started)" "not-started"
+deliver 1.4.2
+deliver 1.5.0
+echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good"
+RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$?
+check "pins the known-good version" "the launcher reads the pin and runs that version instead of the newest" \
+	"$(cat "$MESH_HOST_STATE_DIR/rollback-pinned" 2>/dev/null || echo MISSING)" "1.4.2"
 check "records that it rolled back" "the attempted marker holds the version" \
 	"$(cat "$MESH_HOST_STATE_DIR/rollback-attempted" 2>/dev/null || echo MISSING)" "1.4.2"
+check "succeeds" "a rollback that found its version is not a failure" "$RC" "0"
+# It chooses and stops. The launcher runs the host next, and starting it here would run two
+# (novox/hq ADR 0005).
+check "does not start anything itself" "the launcher owns starting" \
+	"$(ls "$MESH_HOST_STATE_DIR" | grep -c started || true)" "0"
+# The version it rolled back FROM is left alone: it is the newest, and retiring it is the running
+# host's job after a reconcile it completes, never a recovery's.
+check "leaves the failing version on disk" "a recovery deletes nothing" \
+	"$([ -x "$MESH_HOST_LIBEXEC/versions/1.5.0/nox-mesh-host" ] && echo present || echo gone)" "present"
 
 # --- it rolls back only once ---------------------------------------------------------------
 setup
-echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
-echo "1.4.2" > "$MESH_HOST_STATE_DIR/rollback-attempted"
-touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst"
-"$SCRIPT" >/dev/null 2>&1
+deliver 1.4.2
+echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good"
+echo 1.4.2 > "$MESH_HOST_STATE_DIR/rollback-attempted"
+"$SCRIPT" >/dev/null 2>&1 || true
 check "does not roll back twice" "a second failure is the machine, not the binary" \
-	"$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called"
+	"$([ -e "$MESH_HOST_STATE_DIR/rollback-pinned" ] && echo pinned || echo untouched)" "untouched"
 
 # --- nothing to roll back to ---------------------------------------------------------------
 setup
-set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e
+RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$?
 check "no known-good: does nothing" "a host that never reconciled has no version to return to" \
-	"$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called"
+	"$([ -e "$MESH_HOST_STATE_DIR/rollback-attempted" ] && echo attempted || echo untouched)" "untouched"
 # The exit code is asserted from a real run, not from a literal. An earlier version of this
 # compared "0" to "0" and could not fail — which hid an injected fault that made the script die
 # here instead of returning cleanly.
@@ -74,23 +75,29 @@ check "no known-good: exits zero" "an installation failure is not a rollback fai
 
 setup
 printf '   \n' > "$MESH_HOST_STATE_DIR/known-good"
-"$SCRIPT" >/dev/null 2>&1
-check "blank known-good: refuses to guess" "installing nothing and reporting success is the fault this prevents" \
-	"$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called"
+"$SCRIPT" >/dev/null 2>&1 || true
+check "blank known-good: refuses to guess" "pinning nothing and reporting success is the fault this prevents" \
+	"$([ -e "$MESH_HOST_STATE_DIR/rollback-pinned" ] && echo pinned || echo untouched)" "untouched"
 
-# --- the cache was cleaned ------------------------------------------------------------------
+# --- the known-good version is not delivered -------------------------------------------------
+# It was retired, or that host was placed on the machine by hand and never delivered — which is how
+# every first host arrives. Pinning it anyway would have the launcher ignore the pin and start the
+# newest again, which is the binary that is failing.
 setup
-echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
-set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e
-check "missing package: fails loudly" "cannot roll back, and says so rather than reporting success" "$RC" "1"
+deliver 1.5.0
+echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good"
+RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$?
+check "version not delivered: fails loudly" "cannot roll back, and says so rather than reporting success" "$RC" "1"
+check "version not delivered: pins nothing" "a pin the launcher would ignore is worse than none" \
+	"$([ -e "$MESH_HOST_STATE_DIR/rollback-pinned" ] && echo pinned || echo untouched)" "untouched"
 
-# --- the package manager refuses -------------------------------------------------------------
+# --- a version directory with no binary in it ------------------------------------------------
+# An interrupted delivery leaves one. Pinning it would start nothing.
 setup
-echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
-touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst"
-STUB_PACMAN_FAILS=1 ; export STUB_PACMAN_FAILS
-set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e
-check "pacman fails: exits non-zero" "a failed rollback is a failure the launcher must see" "$RC" "1"
+mkdir -p "$MESH_HOST_LIBEXEC/versions/1.4.2"
+echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good"
+RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$?
+check "half-delivered version: fails loudly" "a directory is not a version; the binary is" "$RC" "1"
 
 printf '\nrollback: %d passed, %d failed\n' "$PASS" "$FAIL"
 [ "$FAIL" -eq 0 ]