A node can join the bus the mesh runs on

novox/hq 04-ISSUES/146, the layers behind the three already fixed.

A new membership says which bus it is for. Empty meant 'whatever the mesh runs
today' while two buses existed, and became a refusal the moment one did: an
enrolled node came up and reconnected for ever against its own record.

The enrolling client takes its inboxes in the space its user may listen in. A
JetStream publish waits for the stream's acknowledgement on an inbox the client
picks, and its default is one this user may not subscribe to — so the enrolment
failed with a permissions violation on a subject nobody had chosen.

And the enrolment publish carries a message id, so the client's own retry is
discarded by the stream rather than enrolling the machine twice. That one is
not finished: the duplicate survives it, and the issue says where the trail
stops.

cmd/mesh-host/main.go

@@ -601,6 +601,20 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
 					"  a rollback would have nothing to return to.\n", version, err)
 		}
 	}
+	// And retire what is older than this version's predecessor, on the same evidence known-good is
+	// written on (novox/hq ADR 0141). The predecessor stays, because it is exactly what a rollback
+	// starts; everything before it has no reader. Never this version, whatever the answer.
+	//
+	// A failure is said and does not fail the apply, for the reason above: what is lost is disk, and
+	// hiding it would make a machine quietly fill up.
+	if version != "" {
+		if retired, err := upgrade.Retire(upgrade.VersionsDir(""), version); err != nil {
+			fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not retire an older host: %v\n", err)
+		} else if len(retired) > 0 {
+			fmt.Fprintf(os.Stderr, "mesh-host: retired the host version(s) %s\n",
+				strings.Join(retired, ", "))
+		}
+	}
 	// And tell the launcher this start worked. Without it the counter only climbs, and a node
 	// that has been up for months rolls itself back on its third ordinary restart.
 	if err := upgrade.ClearAttempts(upgrade.AttemptsPath(opts.state)); err != nil {
@@ -692,15 +706,18 @@ func enrol(ctx context.Context, opts options) error {
 	fmt.Printf("  signing key         %s\n",
 		base64.StdEncoding.EncodeToString(token.Signer)[:16]+"...")
 
-	// The check that has to happen before this machine says anything.
-	conn, err := link.Dial(token.Broker, token.Fingerprint, opts.timeout)
-	if err != nil {
-		return err
-	}
-	defer conn.Close()
-	fmt.Println("\nthe broker presented the certificate this token pins")
-
-	conn.Close()
+	// **The pin is checked by the connection that presents this token, not by a dial of our own**
+	// (novox/hq 04-ISSUES/146). This opened a raw TLS connection to the bus first, which worked
+	// against the broker the mesh used to run and cannot work against the one it runs now: NATS
+	// speaks its own protocol before it upgrades to TLS, so an immediate handshake is answered
+	// with a plaintext line and the enrolment failed with "first record does not look like a TLS
+	// handshake" — on every node that has tried to join since the bus changed, which is why this
+	// went unnoticed: none had.
+	//
+	// What ADR 0004 requires still holds, and holds better: the client that presents the token
+	// carries the same pinned configuration, reads the server's greeting, upgrades, and the
+	// verification runs inside that handshake — so the one-time secret is sent only after the
+	// certificate has been checked, and nothing of this node's reaches an impostor.
 
 	mine, err := identity.Generate(*name)
 	if err != nil {
@@ -774,10 +791,16 @@ func enrol(ctx context.Context, opts options) error {
 	proof := mine.Sign(link.EnrolProof(token.Secret, mine.Public, mine.Overlay.Public,
 		sealing.Public, serving.Public))
 	// The token says where to go and which certificate that address must present. It says nothing
-	// about which bus is there, and does not need to: every token names the one the mesh runs on
-	// today until the rollout (novox/hq ADR 0116 step 5), and that is what an empty Transport is.
+	// about which bus is there, and does not need to: there is one, and this host knows which
+	// (novox/hq ADR 0131 — the mesh speaks to one seat and the old transport is gone).
+	//
+	// **It used to leave this empty** and mean "whatever the mesh runs today", which was true
+	// while two buses existed and became a refusal the moment one did: an empty transport is not
+	// the bus's name, so every enrolment ended at "this token is for the \"\" bus"
+	// (novox/hq 04-ISSUES/146). Nothing caught it because nothing had enrolled since the bus
+	// changed.
 	reply, err := link.Enrol(ctx,
-		link.Approach{Address: token.Broker, Fingerprint: token.Fingerprint},
+		link.Approach{Address: token.Broker, Fingerprint: token.Fingerprint, Transport: link.OnNATS},
 		*name, token.Secret,
 		mine.Public, mine.Overlay.Public, sealing.Public, serving.Public, reported, proof, found,
 		opts.timeout)
@@ -793,6 +816,13 @@ func enrol(ctx context.Context, opts options) error {
 		Fingerprint: firstNonEmpty(reply.Fingerprint, token.Fingerprint),
 		Signer:      firstNonEmpty2(reply.Signer, token.Signer),
 		Password:    reply.Password,
+		// **Which bus this membership is for, said rather than left empty** (novox/hq
+		// 04-ISSUES/146). The link refuses a membership that names another bus, and an empty name
+		// is not this one's — so a node enrolled without it came up and reconnected for ever
+		// against its own record: "this membership is for \"\", and the mesh's bus is nats". The
+		// reply does not carry it because there is one bus and the host knows which (ADR 0131);
+		// what was missing was writing that down where the link reads it.
+		Transport: link.OnNATS,
 	}
 	if mine.Membership.Password == "" {
 		// The mesh did not replace the token's secret, so it is still this node's broker
@@ -979,12 +1009,31 @@ func runLink(ctx context.Context, opts options) error {
 	sched := apply.NewScheduler(apply.SystemClock(), apply.ExecRunner, say)
 	go sched.Run(ctx)
 
+	// **Standing aside for a successor happens between reconciles and nowhere else** (novox/hq ADR
+	// 0141). A host that stood aside mid-apply is the half-configured machine this host exists to
+	// prevent, so the question is asked after an apply has finished and the answer is a clean exit —
+	// which the launcher already reads as "run whatever is on disk now".
+	aside, standAside := context.WithCancel(ctx)
+	defer standAside()
+	stoodAside := false
+
 	applier := func(ctx context.Context, raw, signature []byte) link.Report {
 		report := applyAndKeep(ctx, opts, raw, &store.Declared{Declaration: raw, Signature: signature}, sched, say)
 		// **A declaration may carry this machine's membership for another bus.** It arrives as a
 		// sealed file like any secret, and is read after the rest has applied so the bus it names is
 		// standing before this machine leaves the one it is on (novox/hq design 28, task 5.2).
 		adoptDeliveredMembership(identity.Path(opts.state), &mine, say)
+
+		switch next, waiting, err := upgrade.Successor(upgrade.VersionsDir(""), version); {
+		case err != nil:
+			// Said, not fatal. A host that cannot read the delivered versions is still running this
+			// machine correctly; what it has lost is the ability to be replaced.
+			say(fmt.Sprintf("cannot tell whether a newer host is delivered: %v", err))
+		case waiting:
+			say(fmt.Sprintf("host %s is delivered; standing aside so the launcher runs it", next.Version))
+			stoodAside = true
+			standAside()
+		}
 		return report
 	}
 
@@ -1015,7 +1064,7 @@ func runLink(ctx context.Context, opts options) error {
 		}}
 	})
 
-	return link.HoldRoused(ctx, link.Membership{
+	held := link.HoldRoused(aside, link.Membership{
 		Node:        mine.Node,
 		Broker:      mine.Membership.Broker,
 		Fingerprint: mine.Membership.Fingerprint,
@@ -1023,6 +1072,13 @@ func runLink(ctx context.Context, opts options) error {
 		Transport:   mine.Membership.Transport,
 		Signer:      mine.Membership.Signer,
 	}, applier, say, opts.timeout, rousedBySignal(ctx), outbox)
+	// **Cleanly**, or the launcher counts standing aside as a crash and rolls the new host back
+	// before it has run once. The context this returns on was cancelled deliberately, so its error
+	// is not a fault to report.
+	if stoodAside {
+		return nil
+	}
+	return held
 }
 
 // adoptionWatch remembers what the node last said about what it holds and its firewall, so a
@@ -1037,7 +1093,12 @@ type adoptionWatch struct {
 // is what the controller previews a flip from, so a port that opens or closes between deliveries
 // must reach it too (novox/hq ADR 0100).
 func adoptionFingerprint(r link.Report) string {
-	parts := []string{"firewall=" + r.Firewall}
+	// **Which links face outside is part of it, though it is not about adoption** (novox/hq ADR
+	// 0140). The mesh composes no filter for a machine that has not said, so a machine whose links
+	// changed — or which has only just learnt to say — has to say so without being asked. Left out,
+	// it could only speak when a declaration arrived, and a declaration cannot be composed until it
+	// has spoken: a machine waiting for a push that is waiting for the machine.
+	parts := []string{"firewall=" + r.Firewall, "outward=" + strings.Join(r.Outward, ",")}
 	for _, h := range r.Held {
 		parts = append(parts, "held "+h.ID+"="+h.Changed)
 	}
@@ -1045,7 +1106,7 @@ func adoptionFingerprint(r link.Report) string {
 		parts = append(parts, fmt.Sprintf("reach %s %s:%d %s %v %d", reach.Protocol, reach.Address,
 			reach.Port, reach.By, reach.Published, reach.ContainerPort))
 	}
-	sort.Strings(parts[1:])
+	sort.Strings(parts[2:])
 	return strings.Join(parts, "\n")
 }
 
@@ -1153,7 +1214,17 @@ func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, s
 		// A reconcile is otherwise silent. On an adopted node it speaks when what it holds or
 		// its firewall changed, because that is how a predecessor still writing is caught
 		// (novox/hq ADR 0100); publish decides whether anything did.
-		if publish != nil && report.Refused == "" && (len(report.Held) > 0 || report.Firewall != "") {
+		//
+		// **And on any node, when it can say which links face outside** (novox/hq ADR 0140). A
+		// converged node holds nothing and found no firewall, so this gate closed on it and the
+		// node could speak only in reply to a declaration — while the mesh composes no declaration
+		// for a node that has not said which links face outside. A machine waiting for a push that
+		// was waiting for the machine, and measured: three converged machines sat silent while the
+		// control plane refused to send them a filter.
+		//
+		// Offered, not published: whether it is news is still the watch's to decide, so an
+		// unchanged answer costs one comparison every reconcile and nothing on the bus.
+		if publish != nil && worthSaying(report) {
 			publish(report)
 		}
 		switch {
@@ -1165,6 +1236,23 @@ func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, s
 	}
 }
 
+// worthSaying is whether a reconcile's report carries anything the mesh needs to hear unasked.
+//
+// **Named rather than written into the loop**, so the rule can be tested. It was a condition inline
+// and it was wrong in a way nothing could catch: it asked only what an adopted node reports, so a
+// converged node — which holds nothing and found no firewall — could speak only in reply to a
+// declaration, while the mesh composes no declaration for a node that has not said which links face
+// outside (novox/hq ADR 0140). Three machines sat silent waiting for a push that was waiting for them.
+//
+// A refused report says nothing about the machine, so it is not news; the refusal is said on the
+// console where a person reads it.
+func worthSaying(report link.Report) bool {
+	if report.Refused != "" {
+		return false
+	}
+	return len(report.Held) > 0 || report.Firewall != "" || len(report.Outward) > 0
+}
+
 // applyDeclared applies a declaration that has already been proved to come from the mesh.
 //
 // Signature checking happens before this is called, in the link. By the time anything here runs,
@@ -1174,6 +1262,21 @@ func applyDeclared(ctx context.Context, opts options, raw []byte, sched *apply.S
 	return applyAndKeep(ctx, opts, raw, nil, sched, say)
 }
 
+// announceOr is what the apply writes its detail with, given what the caller has to say things with.
+//
+// **Never nil.** This argument was nil on the serving path, and nil is silence: everything the apply
+// says — a file held, a container replaced, the found firewall retired — was visible when a person ran
+// the one-shot command and discarded in the way the host actually runs (novox/hq 04-ISSUES/143).
+//
+// Named rather than written inline at the call site so both paths reach the apply the same way, and so
+// a reader asking "where does the apply's output go" finds one answer.
+func announceOr(say link.Announce) func(string) {
+	if say == nil {
+		return func(string) {}
+	}
+	return say
+}
+
 // applying serialises applies within this process.
 //
 // **Two things apply here: the link and the reconcile loop**, and each reads the node's state,
@@ -1236,8 +1339,18 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
 
 	// Declared, not carried. A declaration from the mesh removes only what the mesh previously
 	// declared — never what this machine raised for itself from its bundle (04-ISSUES/010).
+	// **What the apply says goes to the console, which is the journal when this runs as a service.**
+	//
+	// It was nil, and nil is silence. The one-shot path has always passed a real one, so every detail
+	// the apply produces — a file held, a container replaced, the found firewall retired — was visible
+	// when a person ran it by hand and discarded in the way the host actually runs. Measured: after a
+	// machine was converged and its found firewall was not retired, what the host decided was
+	// unrecoverable, because it had said it to nobody (novox/hq 04-ISSUES/143).
+	//
+	// `say` already reaches stdout, and the launcher's unit sends that to the journal, so this needs
+	// no new mechanism — only for the argument to be passed.
 	outcome, updated, applyErr := apply.ApplyKeeping(ctx, built, declared, known, store.OriginDeclared,
-		apply.ExecRunner, nil, sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)))
+		apply.ExecRunner, announceOr(say), sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)))
 
 	// The mode the mesh said, recorded whichever way the apply went: the declaration is kept
 	// either way, and the node is held to it from the next reconcile (novox/hq ADR 0100).
@@ -1263,7 +1376,7 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
 		sched.Sync(declared, held)
 	}
 
-	report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw)}
+	report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw), Host: version}
 	// Which of this machine's links face outside, for the filter the mesh writes around them
 	// (novox/hq ADR 0140). Reported whatever the node's mode: a converged node's filter needs it,
 	// and an adopted one becomes converged without a further round trip. A machine that cannot read

cmd/mesh-host/main_test.go

@@ -501,3 +501,95 @@ func TestReconcileAfterAControllerDeclarationDoesNotReapplyTheBundle(t *testing.
 		t.Errorf("with nothing said, reconcile did not reach for the carried bundle: %v", err)
 	}
 }
+
+// **A machine says which links face outside without being asked.**
+//
+// The mesh composes no filter for a machine that has not said (novox/hq ADR 0140), and a machine only
+// speaks unasked when this fingerprint changes. Left out of it, a machine that has just learnt to say
+// could speak only when a declaration arrived — and a declaration cannot be composed until it has
+// spoken. A machine waiting for a push that is waiting for the machine.
+func TestANewOutwardLinkIsSaidUnasked(t *testing.T) {
+	w := &adoptionWatch{}
+	first := link.Report{Firewall: "none"}
+	if !w.differs(first) {
+		t.Fatal("the first report should differ from nothing")
+	}
+	w.said(first)
+
+	// Only the links changed, and nothing about adoption.
+	learnt := link.Report{Firewall: "none", Outward: []string{"eth0"}}
+	if !w.differs(learnt) {
+		t.Fatal("a machine that has just learnt which links face outside would never say so, " +
+			"and could then never be sent a filter")
+	}
+	w.said(learnt)
+	if w.differs(link.Report{Firewall: "none", Outward: []string{"eth0"}}) {
+		t.Fatal("the same links are reported as a change, so the machine would speak on every reconcile")
+	}
+
+	// And a link that changes — a laptop moving from a cable to a radio — is said too, because the
+	// filter is written around the old one until it is.
+	if !w.differs(link.Report{Firewall: "none", Outward: []string{"wlan0"}}) {
+		t.Fatal("a changed outward link is not said, so the filter stays written around the old one")
+	}
+}
+
+// **A converged machine can say which links face outside, unasked.**
+//
+// A reconcile is otherwise silent, and the condition deciding when it speaks asked only what an
+// adopted node reports — what it holds, and the firewall it found. A converged node has neither, so
+// it could speak only in reply to a declaration, and the mesh composes no declaration for a node
+// that has not said which links face outside (novox/hq ADR 0140). Measured: three converged machines
+// sat silent while the control plane refused to send them a filter.
+func TestAConvergedMachineSaysItsOutwardLinksUnasked(t *testing.T) {
+	// A converged node's reconcile: nothing held, no found firewall, and the links it can see.
+	converged := link.Report{Outward: []string{"eth0"}}
+	if !worthSaying(converged) {
+		t.Fatal("a converged machine cannot say which links face outside, so it can never be " +
+			"sent a filter — a machine waiting for a push that is waiting for the machine")
+	}
+
+	// An adopted node's reasons still hold, because that is how a predecessor still writing is caught.
+	if !worthSaying(link.Report{Firewall: "ufw"}) {
+		t.Fatal("an adopted machine no longer says which firewall it found")
+	}
+	if !worthSaying(link.Report{Held: []link.Held{{ID: "a-file"}}}) {
+		t.Fatal("an adopted machine no longer says what it holds")
+	}
+
+	// And a reconcile with nothing to say stays silent, or every machine speaks every five minutes
+	// about nothing.
+	if worthSaying(link.Report{}) {
+		t.Fatal("a reconcile with nothing to say speaks anyway")
+	}
+
+	// A refused report says nothing about the machine; the refusal is for the console.
+	if worthSaying(link.Report{Outward: []string{"eth0"}, Refused: "not for this node"}) {
+		t.Fatal("a refused report is offered as news about the machine")
+	}
+}
+
+// **A host running as a service says what its apply did.**
+//
+// The serving path passed nil where the apply writes its detail, and nil is silence. The one-shot path
+// has always passed a real function, so everything the apply says was visible when a person ran it by
+// hand and discarded in the way the host actually runs. Measured before this was written: a machine was
+// converged, its found firewall was not retired, and what the host decided was unrecoverable because it
+// had been said to nobody (novox/hq 04-ISSUES/143).
+//
+// This asserts only that the apply's log is never nil and that a line reaches what the caller gave.
+// **It cannot catch the fault it was written for** — a call site passing nil directly — because that is
+// wiring, and wiring is only proved by running the thing. That proof is a deployed host whose journal
+// carries the apply's detail, which is how this fix was verified.
+func TestTheApplysLogIsNeverNil(t *testing.T) {
+	if announceOr(nil) == nil {
+		t.Fatal("a host with nowhere to say things got a nil log, which the apply will call")
+	}
+	announceOr(nil)("this goes nowhere and must not panic")
+
+	var said []string
+	announceOr(func(line string) { said = append(said, line) })("  disabled ufw")
+	if len(said) != 1 || !strings.Contains(said[0], "disabled ufw") {
+		t.Fatalf("the apply's detail did not reach the caller's announce: %v", said)
+	}
+}

examples/foundation-first-node-nats.lock

@@ -127,12 +127,21 @@
     {
       "id": "bus-certificate",
       "type": "action",
-      "command": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls",
-                  "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927",
-                  "-c", "test -f /tls/tls.crt || (openssl req -x509 -newkey rsa:2048 -nodes -keyout /tls/tls.key -out /tls/tls.crt -days 3650 -subj '/CN=mesh-broker' -addext 'subjectAltName=DNS:mesh-broker,IP:127.0.0.1' >/dev/null 2>&1 && chmod 644 /tls/tls.crt && chmod 600 /tls/tls.key)"],
-      "verify": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls",
-                 "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927",
-                 "-c", "test -s /tls/tls.crt && openssl x509 -in /tls/tls.crt -noout"]
+      // **The mesh makes its own** (novox/hq 04-ISSUES/146). This ran `openssl` inside the
+      // broker's image while the broker was one that carried it; the bus that replaced it has a
+      // shell and no openssl, and no other image the bundle names has one either. So the program
+      // that needs the certificate writes it — already on this machine, since the schema step ran
+      // it, and asking nothing of the image it writes into. Self-signed on purpose: a host pins
+      // this server's exact certificate (novox/hq ADR 0004), and at this moment there is no mesh
+      // to ask an authority of.
+      // `--user 0:0` because the volume is root's and this image runs as nobody, which is right
+      // for the long-running control plane and wrong for a one-shot writing into a fresh volume.
+      "command": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
+                  "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
+                  "broker", "certificate", "--into", "/tls"],
+      "verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
+                 "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
+                 "broker", "certificate", "--check", "--into", "/tls"]
     },
     {
       "id": "bus-conf-dir",

internal/link/asking_nats.go

@@ -3,6 +3,7 @@ package link
 import (
 	"context"
 	"crypto/rand"
+	"crypto/sha256"
 	"encoding/hex"
 	"errors"
 	"fmt"
@@ -63,8 +64,15 @@ func presentNats(_ context.Context, to Approach, node, secret string,
 	// subscribe its own inbox and nothing else (design 25 §6). The secret is its password, the same
 	// string the request claims, so the server proves somebody holds the token and the request
 	// proves the same thing to the controller without it having to ask the server who connected.
+	// **Its own inbox space, because that is the only one it may listen in** (novox/hq
+	// 04-ISSUES/146). A JetStream publish waits for the stream's acknowledgement on an inbox the
+	// client picks, and the client's default is `_INBOX.<random>` — which this user may not
+	// subscribe to, so the enrolment failed with a permissions violation on a subject nobody had
+	// chosen. The permission is `_INBOX.enrol.<node>.>` (design 25 §6), so the client is told to
+	// pick its inboxes there; the reply address below is in the same space for the same reason.
 	conn, err := nats.Connect(natsURL(to.Address),
 		nats.Secure(config),
+		nats.CustomInboxPrefix("_INBOX.enrol."+node),
 		nats.UserInfo("enrol."+node, secret),
 		nats.Name("mesh-host/enrol/"+node),
 		nats.Timeout(timeout),
@@ -124,7 +132,19 @@ func (a *natsAsking) Ask(ctx context.Context, request []byte, wait time.Duration
 	defer cancel()
 	// Into the stream and awaited: an enrolment the bus never accepted must fail here rather than be
 	// assumed, because the node has nothing else to go on.
-	if _, err := a.js.Publish(EnrolSubject, addressed, nats.Context(publish)); err != nil {
+	//
+	// **Once, however many times it is sent** (novox/hq 04-ISSUES/146). The client re-publishes when
+	// an acknowledgement is slow, and the mesh enrolled the machine on each copy — minting a second
+	// credential, which replaced the first, which is the one the node had already been given. The
+	// machine then reconnected for ever as a user whose password the mesh had rotated out from under
+	// it, and the controller's log said "enrolled anchor" twice in the same second.
+	//
+	// The id is the message: the same bytes carry the same id, so the stream discards the client's
+	// own retry, and a genuine second attempt — which carries a new reply address — is a different
+	// message and is let through.
+	sum := sha256.Sum256(addressed)
+	if _, err := a.js.Publish(EnrolSubject, addressed,
+		nats.MsgId(hex.EncodeToString(sum[:])), nats.Context(publish)); err != nil {
 		return nil, fmt.Errorf("cannot ask the mesh to enrol this node: %w", err)
 	}
 

internal/link/messages.go

@@ -110,6 +110,13 @@ type Report struct {
 	// and the mesh's up in its place, and where the found configuration's original was kept.
 	Tunnel *CarriedTunnel `json:"tunnel,omitempty"`
 
+	// Host is the version of the host that produced this report (novox/hq ADR 0141).
+	//
+	// Without it nothing can say a machine is behind, so "every machine current with its source"
+	// could not include the host — the one component the mesh did not deliver. It is a fact the
+	// machine states about itself, like the firewall it found and the links that face outside.
+	Host string `json:"host,omitempty"`
+
 	// Outward is the links on this machine that face outside it — the ones carrying a default
 	// route (novox/hq ADR 0140). Every node reports it, adopted or converged, because a converged
 	// node's filter is written around it.

internal/link/pinned.go

@@ -73,8 +73,19 @@ func PinnedConfig(pin string) (*tls.Config, error) {
 	}, nil
 }
 
-// Dial opens a TLS connection to the broker, refusing anything but the pinned certificate.
-func Dial(address, pin string, timeout time.Duration) (*tls.Conn, error) {
+// dialPinned completes a TLS handshake against an address, refusing anything but the pinned
+// certificate.
+//
+// **Not how the bus is reached, and it used to be** (novox/hq 04-ISSUES/146). Enrolment opened one
+// of these before it said anything, which was right while the broker answered TLS immediately and
+// wrong the moment the mesh moved to a bus that speaks its own protocol first. The pin itself was
+// never the problem — PinnedConfig is what the NATS client is given, and the verification runs
+// inside the handshake that client performs.
+//
+// It stays here because this is where the pin is proven: the tests beside it run a real TLS server
+// and assert that a wrong certificate is refused before a byte of application data is sent. What it
+// must not become again is something a caller uses to reach the bus.
+func dialPinned(address, pin string, timeout time.Duration) (*tls.Conn, error) {
 	config, err := PinnedConfig(pin)
 	if err != nil {
 		return nil, err
@@ -86,7 +97,7 @@ func Dial(address, pin string, timeout time.Duration) (*tls.Conn, error) {
 		if errors.Is(err, ErrWrongCertificate) {
 			return nil, err
 		}
-		return nil, fmt.Errorf("cannot reach the broker at %s: %w", address, err)
+		return nil, fmt.Errorf("cannot reach %s: %w", address, err)
 	}
 	return conn, nil
 }

internal/link/pinned_test.go

@@ -63,7 +63,7 @@ func server(t *testing.T) (address string, fingerprint string) {
 
 func TestTheRightBrokerIsAccepted(t *testing.T) {
 	address, pin := server(t)
-	conn, err := Dial(address, pin, 5*time.Second)
+	conn, err := dialPinned(address, pin, 5*time.Second)
 	if err != nil {
 		t.Fatalf("the broker its token describes was refused: %v", err)
 	}
@@ -76,7 +76,7 @@ func TestADifferentBrokerIsRefused(t *testing.T) {
 	address, _ := server(t)
 	_, other := server(t)
 
-	_, err := Dial(address, other, 5*time.Second)
+	_, err := dialPinned(address, other, 5*time.Second)
 	if err == nil {
 		t.Fatal("a broker presenting a different certificate was accepted")
 	}
@@ -130,7 +130,7 @@ func TestNothingIsSentToTheWrongBroker(t *testing.T) {
 
 	// A pin for a certificate this server does not have.
 	_, elsewhere := server(t)
-	if _, err := Dial(listener.Addr().String(), elsewhere, 5*time.Second); err == nil {
+	if _, err := dialPinned(listener.Addr().String(), elsewhere, 5*time.Second); err == nil {
 		t.Fatal("the impostor was accepted")
 	}
 	if n := <-received; n > 0 {
@@ -160,7 +160,7 @@ func TestAnUnreachableBrokerIsAnOrdinaryFailure(t *testing.T) {
 	address := listener.Addr().String()
 	listener.Close()
 
-	_, err = Dial(address, pin, 2*time.Second)
+	_, err = dialPinned(address, pin, 2*time.Second)
 	if err == nil {
 		t.Fatal("dialling a closed port succeeded")
 	}

internal/upgrade/upgrade.go

@@ -16,7 +16,9 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
+	"time"
 )
 
 // Files the launcher reads and this binary writes. Next to the store, because they are node
@@ -157,3 +159,168 @@ func ReadKnownGood(path string) (string, error) {
 	}
 	return strings.TrimSpace(string(raw)), nil
 }
+
+// Where delivered versions live, and what the binary inside one is called.
+//
+// **A directory named for its version, never a link and never a write over what is running**
+// (novox/hq ADR 0141). Two facts follow from that one choice: the kernel refuses to truncate a
+// running executable, so the path a delivery writes must not be the path being executed; and a
+// rollback needs the previous version still present, which a single path cannot offer.
+//
+// The mesh creates no links (novox/hq ADR 0012), so nothing points at "current". The version is in
+// the path, which is why nothing has to be told what is running.
+const (
+	// DefaultLibexec is where the host's own files live. Fixed rather than derived from where the
+	// running executable sits: the first host to understand any of this was copied to a machine by
+	// hand, and one that looked for its successor beside itself would never find a delivered version
+	// — which is every machine in this mesh on the day this ships.
+	DefaultLibexec  = "/usr/lib/nox-mesh-host"
+	VersionsDirName = "versions"
+	BinaryName      = "nox-mesh-host"
+	// PinnedName is the version a rollback chose, which the launcher runs instead of the newest.
+	// Without it the launcher would start the newest again and the rollback would flap.
+	PinnedName = "rollback-pinned"
+)
+
+// VersionsDir is where delivered versions live, given where the host's libexec is. An empty libexec
+// means the default, and the environment overrides it so a test needs no root.
+func VersionsDir(libexec string) string {
+	if libexec == "" {
+		libexec = os.Getenv("MESH_HOST_LIBEXEC")
+	}
+	if libexec == "" {
+		libexec = DefaultLibexec
+	}
+	return filepath.Join(libexec, VersionsDirName)
+}
+
+// PinnedPath is where a rollback records the version it chose.
+func PinnedPath(statePath string) string {
+	return filepath.Join(filepath.Dir(statePath), PinnedName)
+}
+
+// Delivered is one version present on the machine.
+type Delivered struct {
+	// Version is the directory's name, which is the version.
+	Version string
+	// Binary is the executable inside it.
+	Binary string
+	// At is when it arrived, which is how "newest" is decided.
+	At time.Time
+}
+
+// Versions are the versions delivered to this machine, newest first.
+//
+// **Newest by when it arrived, not by its name.** A version string comes from what the source was
+// tagged or described as, and those do not sort: "1.10" before "1.9", a commit hash before either.
+// Ordering by name would run an older host and call it an upgrade. When it arrived is a fact the
+// filesystem keeps and the delivery sets.
+//
+// A directory with no executable in it is not a version. A delivery that was interrupted leaves one,
+// and running the newest would then mean running nothing.
+func Versions(dir string) ([]Delivered, error) {
+	entries, err := os.ReadDir(dir)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("cannot read the delivered versions at %s: %w", dir, err)
+	}
+
+	var out []Delivered
+	for _, entry := range entries {
+		if !entry.IsDir() {
+			continue
+		}
+		binary := filepath.Join(dir, entry.Name(), BinaryName)
+		info, err := os.Stat(binary)
+		if err != nil || info.IsDir() {
+			continue
+		}
+		at := info.ModTime()
+		if d, err := entry.Info(); err == nil && d.ModTime().After(at) {
+			at = d.ModTime()
+		}
+		out = append(out, Delivered{Version: entry.Name(), Binary: binary, At: at})
+	}
+
+	// Newest first, and by name when two arrived in the same instant so the answer is never
+	// arbitrary — a test that passes half the time is worse than one that fails.
+	sort.Slice(out, func(a, b int) bool {
+		if out[a].At.Equal(out[b].At) {
+			return out[a].Version > out[b].Version
+		}
+		return out[a].At.After(out[b].At)
+	})
+	return out, nil
+}
+
+// Successor is the version this machine should be running instead of the given one, if any.
+//
+// Empty when the running version is the newest, which is the ordinary answer. The host asks this
+// between reconciles and nowhere else: standing aside mid-apply is the half-configured machine the
+// host exists to prevent (novox/hq ADR 0141).
+func Successor(dir, running string) (Delivered, bool, error) {
+	delivered, err := Versions(dir)
+	if err != nil {
+		return Delivered{}, false, err
+	}
+	if len(delivered) == 0 {
+		return Delivered{}, false, nil
+	}
+	newest := delivered[0]
+	// A machine whose running version is not among the delivered ones is the machine every mesh has
+	// one of: the host was put there by hand before any of this existed. Treating that as "stand
+	// aside" is correct — what was delivered is what the mesh asked for.
+	if newest.Version == running {
+		return Delivered{}, false, nil
+	}
+	return newest, true, nil
+}
+
+// Retire removes delivered versions older than the running one's predecessor.
+//
+// The running version and the one before it are kept, and nothing else: the predecessor is exactly
+// what a rollback starts, and every version before that is weight with no reader. Called after a
+// reconcile completes, which is the same evidence known-good is written on — retiring on any weaker
+// signal would delete the thing a failing host is about to need.
+//
+// Never the running version, whatever it is asked. A host that deleted its own image would survive
+// until it stopped and then be unstartable, and the launcher's rollback reads a version, not a
+// process.
+func Retire(dir, running string) ([]string, error) {
+	delivered, err := Versions(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	keep := map[string]bool{running: true}
+	for i, d := range delivered {
+		if d.Version != running {
+			continue
+		}
+		// Its predecessor is the next one down the list, which is the next oldest.
+		if i+1 < len(delivered) {
+			keep[delivered[i+1].Version] = true
+		}
+		break
+	}
+	// A running version that was never delivered has no predecessor among these, so the newest
+	// delivered one is what a rollback would reach for. Keep it.
+	if len(keep) == 1 && len(delivered) > 0 {
+		keep[delivered[0].Version] = true
+	}
+
+	var removed []string
+	for _, d := range delivered {
+		if keep[d.Version] {
+			continue
+		}
+		if err := os.RemoveAll(filepath.Join(dir, d.Version)); err != nil {
+			return removed, fmt.Errorf("cannot retire the host version %s: %w", d.Version, err)
+		}
+		removed = append(removed, d.Version)
+	}
+	sort.Strings(removed)
+	return removed, nil
+}

internal/upgrade/versions_test.go

@@ -0,0 +1,180 @@
+package upgrade
+
+import (
+	"os"
+	"path/filepath"
+	"reflect"
+	"sort"
+	"testing"
+	"time"
+)
+
+// deliver writes a version as a delivery would: a directory named for it with the binary inside.
+// at fixes when it arrived, because "newest" is when it arrived and a test must not race the clock.
+func deliver(t *testing.T, dir, version string, at time.Time) string {
+	t.Helper()
+	into := filepath.Join(dir, version)
+	if err := os.MkdirAll(into, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	binary := filepath.Join(into, BinaryName)
+	if err := os.WriteFile(binary, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chtimes(binary, at, at); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chtimes(into, at, at); err != nil {
+		t.Fatal(err)
+	}
+	return binary
+}
+
+// **Newest is when it arrived, not how its name sorts.**
+//
+// A version string is whatever the source was tagged or described as, and those do not sort: "1.10"
+// orders before "1.9", and a commit hash orders before either. Ordering by name would start an older
+// host and call that an upgrade.
+func TestNewestIsWhenItArrivedAndNotHowItSorts(t *testing.T) {
+	dir := t.TempDir()
+	base := time.Now().Add(-time.Hour)
+	deliver(t, dir, "1.10", base)                 // sorts LAST by name, arrived first
+	deliver(t, dir, "1.9", base.Add(time.Minute)) // sorts first by name, arrived last
+
+	got, err := Versions(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 2 || got[0].Version != "1.9" {
+		t.Fatalf("newest is %+v, want the one that arrived last (1.9)", got)
+	}
+}
+
+// A delivery that was interrupted leaves a directory with no executable in it. Running "the newest"
+// would then mean running nothing, so it is not a version.
+func TestADirectoryWithNoBinaryIsNotAVersion(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(dir, "half-delivered"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	deliver(t, dir, "good", time.Now().Add(-time.Hour))
+
+	got, err := Versions(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 1 || got[0].Version != "good" {
+		t.Fatalf("versions are %+v, want only the one with a binary", got)
+	}
+}
+
+// Nothing delivered is not a fault. A machine whose host was placed by hand has no versions
+// directory at all, and that must read as "no successor" rather than as an error that stops a
+// reconcile.
+func TestNoVersionsDirectoryIsNotAnError(t *testing.T) {
+	got, err := Versions(filepath.Join(t.TempDir(), "absent"))
+	if err != nil {
+		t.Fatalf("an absent versions directory should not be an error: %v", err)
+	}
+	if len(got) != 0 {
+		t.Fatalf("versions are %+v, want none", got)
+	}
+}
+
+func TestTheNewestVersionIsTheSuccessorAndTheRunningOneIsNot(t *testing.T) {
+	dir := t.TempDir()
+	base := time.Now().Add(-time.Hour)
+	deliver(t, dir, "one", base)
+	deliver(t, dir, "two", base.Add(time.Minute))
+
+	next, yes, err := Successor(dir, "one")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !yes || next.Version != "two" {
+		t.Fatalf("successor is %+v (%v), want two", next, yes)
+	}
+
+	if _, yes, err := Successor(dir, "two"); err != nil || yes {
+		t.Fatalf("the newest version is its own successor (%v, %v)", yes, err)
+	}
+}
+
+// A host put there by hand, before any of this existed, is not among the delivered versions. What the
+// mesh delivered is what it asked for, so that is a successor — otherwise the first delivery to such a
+// machine would be ignored for ever, which is every machine in this mesh today.
+func TestAHostThatWasNeverDeliveredHasASuccessor(t *testing.T) {
+	dir := t.TempDir()
+	deliver(t, dir, "delivered", time.Now().Add(-time.Hour))
+
+	next, yes, err := Successor(dir, "copied-by-hand")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !yes || next.Version != "delivered" {
+		t.Fatalf("successor is %+v (%v), want the delivered one", next, yes)
+	}
+}
+
+// The running version and its predecessor are kept, and nothing else. The predecessor is exactly what
+// a rollback starts; everything older has no reader.
+func TestRetireKeepsTheRunningVersionAndItsPredecessor(t *testing.T) {
+	dir := t.TempDir()
+	base := time.Now().Add(-4 * time.Hour)
+	for i, v := range []string{"one", "two", "three", "four"} {
+		deliver(t, dir, v, base.Add(time.Duration(i)*time.Hour))
+	}
+
+	removed, err := Retire(dir, "four")
+	if err != nil {
+		t.Fatal(err)
+	}
+	sort.Strings(removed)
+	if !reflect.DeepEqual(removed, []string{"one", "two"}) {
+		t.Fatalf("retired %v, want one and two — three is the predecessor a rollback needs", removed)
+	}
+	for _, kept := range []string{"three", "four"} {
+		if _, err := os.Stat(filepath.Join(dir, kept, BinaryName)); err != nil {
+			t.Fatalf("%s was retired and a rollback now has nowhere to go: %v", kept, err)
+		}
+	}
+}
+
+// **Never the running version, whatever it is asked.** A host that deleted its own image would run
+// until it stopped and then be unstartable, and the launcher's rollback reads a version rather than a
+// process.
+func TestRetireNeverRemovesTheRunningVersion(t *testing.T) {
+	dir := t.TempDir()
+	base := time.Now().Add(-2 * time.Hour)
+	deliver(t, dir, "older", base)
+	deliver(t, dir, "newer", base.Add(time.Hour))
+
+	// Asked while running the OLDER one, which is what a machine looks like between a delivery and
+	// the moment it stands aside.
+	if _, err := Retire(dir, "older"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "older", BinaryName)); err != nil {
+		t.Fatalf("the running version was retired: %v", err)
+	}
+}
+
+// A machine running a hand-placed host keeps the newest delivered version, because that is what a
+// rollback would reach for. Retiring it would leave the machine with no way back at all.
+func TestRetireKeepsTheNewestWhenTheRunningVersionWasNeverDelivered(t *testing.T) {
+	dir := t.TempDir()
+	base := time.Now().Add(-3 * time.Hour)
+	deliver(t, dir, "old", base)
+	deliver(t, dir, "new", base.Add(time.Hour))
+
+	removed, err := Retire(dir, "copied-by-hand")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(removed, []string{"old"}) {
+		t.Fatalf("retired %v, want only old — new is the rollback target", removed)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "new", BinaryName)); err != nil {
+		t.Fatalf("the only delivered version was retired: %v", err)
+	}
+}

packaging/launch_test.sh

@@ -187,5 +187,70 @@ sleep 1
 check "a crash is counted" "unlike a clean exit, which is not" "$(count)" "1"
 kill -TERM "$LP" 2>/dev/null; sleep 1; pkill -f "$MESH_HOST_BIN" 2>/dev/null || true
 
+# --- which version it runs (novox/hq ADR 0141) ------------------------------------------------
+#
+# Versions live side by side in directories named for them. The launcher picks one every time round
+# the loop, never once: standing aside for a successor is a clean exit, and the next turn has to run
+# what is on disk NOW — resolved once, the same binary would restart for ever and no upgrade would
+# ever take.
+
+# deliver a version as the mesh would, recording which one ran so a test can assert the choice.
+deliver() {
+	mkdir -p "$MESH_HOST_LIBEXEC/versions/$1"
+	cat > "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host" <<STUB
+#!/bin/sh
+echo "$1" >> "\$MESH_HOST_STATE_DIR/which.ran"
+exit "\${STUB_HOST_EXIT:-1}"
+STUB
+	chmod +x "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host"
+	# When it arrived is what "newest" means, so it is set rather than left to the clock.
+	touch -d "$2" "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host" "$MESH_HOST_LIBEXEC/versions/$1"
+}
+which_ran() { cat "$MESH_HOST_STATE_DIR/which.ran" 2>/dev/null || echo NONE; }
+
+# Newest is when it arrived, not how its name sorts: "1.10" orders before "1.9" by name, so ordering
+# by name would run an older host and call it an upgrade.
+setup
+deliver 1.10 "2 hours ago"
+deliver 1.9 "1 hour ago"
+"$LAUNCH" >/dev/null 2>&1 || true
+check "runs the newest delivered version" "newest is when it arrived, not how the name sorts" \
+	"$(which_ran)" "1.9"
+
+# A pin from a rollback beats the newest, or the launcher would start the failing binary again and
+# the rollback would flap.
+setup
+deliver 1.9 "2 hours ago"
+deliver 2.0 "1 hour ago"
+echo 1.9 > "$MESH_HOST_STATE_DIR/rollback-pinned"
+"$LAUNCH" >/dev/null 2>&1 || true
+check "a pinned version beats the newest" "otherwise a rollback starts the binary it just rejected" \
+	"$(which_ran)" "1.9"
+
+# A pin naming a version that is not there is ignored rather than fatal: the machine choosing for
+# itself is better than a machine that starts nothing.
+setup
+deliver 2.0 "1 hour ago"
+echo 1.9 > "$MESH_HOST_STATE_DIR/rollback-pinned"
+"$LAUNCH" >/dev/null 2>&1 || true
+check "an undeliverable pin is ignored" "a machine that starts nothing is worse than one that chooses" \
+	"$(which_ran)" "2.0"
+
+# An interrupted delivery leaves a directory with no binary in it. Treating it as the newest would
+# mean running nothing.
+setup
+deliver 1.9 "2 hours ago"
+mkdir -p "$MESH_HOST_LIBEXEC/versions/2.0-half"
+touch -d "1 minute ago" "$MESH_HOST_LIBEXEC/versions/2.0-half"
+"$LAUNCH" >/dev/null 2>&1 || true
+check "skips a version with no binary" "a directory is not a version; the binary is" \
+	"$(which_ran)" "1.9"
+
+# Nothing delivered: the host placed by hand, which is how the first one always arrives. Without this
+# the change would strand every machine in the mesh on the day it ships.
+setup
+"$LAUNCH" >/dev/null 2>&1 || true
+check "falls back to the host placed by hand" "every first host arrives this way" "$(started)" "yes"
+
 printf '\nlaunch: %d passed, %d failed\n' "$PASS" "$FAIL"
 [ "$FAIL" -eq 0 ]

packaging/nox-mesh-host-launch

@@ -16,16 +16,51 @@ set -u
 
 STATE_DIR="${MESH_HOST_STATE_DIR:-/var/lib/mesh-host}"
 LIBEXEC="${MESH_HOST_LIBEXEC:-/usr/lib/nox-mesh-host}"
-HOST="${MESH_HOST_BIN:-/usr/bin/nox-mesh-host}"
+# The host that was placed by hand, used only when nothing has been delivered. The first host on a
+# machine always arrives this way; every one after it is delivered (novox/hq ADR 0141).
+FALLBACK="${MESH_HOST_BIN:-/usr/bin/nox-mesh-host}"
+VERSIONS="$LIBEXEC/versions"
+BINARY="nox-mesh-host"
 LIMIT="${MESH_HOST_START_LIMIT:-3}"
 BACKOFF="${MESH_HOST_BACKOFF:-5}"
 ONCE="${MESH_HOST_RUN_ONCE:-}"   # tests run one iteration; nothing else sets this
 
 ATTEMPTS="$STATE_DIR/start-attempts"
 HALTED="$STATE_DIR/halted"
+PINNED="$STATE_DIR/rollback-pinned"
 
 say() { echo "nox-mesh-host-launch: $*" >&2; }
 
+# Which host to run: the version a rollback pinned, or the most recently delivered one, or the one
+# placed by hand when nothing has been delivered (novox/hq ADR 0141).
+#
+# **Asked every time round the loop, not once.** Standing aside for a successor is a clean exit, and
+# the next turn has to run what is on disk NOW — resolving this once would restart the same binary
+# for ever and the upgrade would never take.
+#
+# Newest by when it arrived, never by how its name sorts: a version string is whatever the source was
+# described as, and those do not sort — "1.10" orders before "1.9". Ordering by name would start an
+# older host and call it an upgrade.
+pick_host() {
+	if [ -s "$PINNED" ]; then
+		pinned="$(tr -d '[:space:]' < "$PINNED" 2>/dev/null || true)"
+		if [ -n "$pinned" ] && [ -x "$VERSIONS/$pinned/$BINARY" ]; then
+			echo "$VERSIONS/$pinned/$BINARY"
+			return 0
+		fi
+		say "the pinned version '$pinned' is not delivered; ignoring the pin"
+	fi
+	# A directory with no executable in it is not a version: an interrupted delivery leaves one, and
+	# running "the newest" would then mean running nothing.
+	for candidate in $(ls -1t "$VERSIONS" 2>/dev/null || true); do
+		if [ -x "$VERSIONS/$candidate/$BINARY" ]; then
+			echo "$VERSIONS/$candidate/$BINARY"
+			return 0
+		fi
+	done
+	echo "$FALLBACK"
+}
+
 child=
 stopping=
 
@@ -95,6 +130,14 @@ while :; do
 		fi
 	fi
 
+	HOST="$(pick_host)"
+	if [ ! -x "$HOST" ]; then
+		say "no host to run: nothing delivered under $VERSIONS and $FALLBACK is not executable."
+		printf 'no host binary\n' > "$HALTED"
+		exit 0
+	fi
+	say "running $HOST"
+
 	"$HOST" run &
 	child=$!
 	status=0

packaging/nox-mesh-host-rollback

@@ -1,20 +1,29 @@
 #!/bin/sh
 # Put the host back on the last version that worked.
 #
-# novox/hq ADR 0005. This runs when nox-mesh-host will not start, so it shares no code with it
-# and calls none of it: a binary that cannot start cannot be its own recovery. POSIX sh, no
+# novox/hq ADR 0005 and ADR 0141. This runs when nox-mesh-host will not start, so it shares no code
+# with it and calls none of it: a binary that cannot start cannot be its own recovery. POSIX sh, no
 # bashisms, nothing that has to be installed.
 #
-# It is deliberately dull. Everything it does is one of: read a file, run the package manager,
-# ask the service manager to try again.
+# It is deliberately dull. Everything it does is one of: read a file, look at a directory, write a
+# file.
+#
+# **It used to reinstall a package.** It read the known-good version and asked one operating system's
+# package manager for it, out of that package manager's cache. Two things were wrong with that. No
+# machine in this mesh had the host installed as a package, so the recovery could not run on any of
+# them; and the host is built per operating system (ADR 0005), so a recovery written in one package
+# manager's terms could not run on two of the three. Versions now live side by side in directories
+# named for them, so going back is choosing a directory — which is the same on every machine.
 set -eu
 
 STATE_DIR="${MESH_HOST_STATE_DIR:-/var/lib/mesh-host}"
-PKG_CACHE="${MESH_HOST_PKG_CACHE:-/var/cache/pacman/pkg}"
-PACKAGE="${MESH_HOST_PACKAGE:-nox-mesh-host}"
+LIBEXEC="${MESH_HOST_LIBEXEC:-/usr/lib/nox-mesh-host}"
+VERSIONS="$LIBEXEC/versions"
+BINARY="nox-mesh-host"
 
 KNOWN_GOOD="$STATE_DIR/known-good"
 ATTEMPTED="$STATE_DIR/rollback-attempted"
+PINNED="$STATE_DIR/rollback-pinned"
 
 say() { echo "nox-mesh-host-rollback: $*" >&2; }
 
@@ -43,23 +52,24 @@ if [ -z "$VERSION" ]; then
 	exit 0
 fi
 
-PKG="$(ls "$PKG_CACHE"/"$PACKAGE"-"$VERSION"-*.pkg.tar.* 2>/dev/null | head -n 1 || true)"
-if [ -z "$PKG" ]; then
-	say "known-good is $VERSION and no package for it is in $PKG_CACHE."
-	say "the cache was cleaned, or that version was never installed from here."
+# The version that last worked may be the one that was placed by hand, which is not delivered and has
+# no directory. Nothing to choose, and saying so is better than pinning a version that is not there —
+# the launcher would ignore the pin and start the newest again, which is the binary that is failing.
+if [ ! -x "$VERSIONS/$VERSION/$BINARY" ]; then
+	say "known-good is $VERSION and no such version is delivered under $VERSIONS."
+	say "it was retired, or that host was placed by hand and never delivered."
 	say "cannot roll back. this node needs a person."
 	exit 1
 fi
 
-say "rolling back to $VERSION ($PKG)"
+say "rolling back to $VERSION ($VERSIONS/$VERSION/$BINARY)"
 printf '%s\n' "$VERSION" > "$ATTEMPTED"
 
-if ! pacman -U --noconfirm "$PKG"; then
-	say "the package manager refused to install $PKG."
-	exit 1
-fi
+# The pin is what stops the launcher starting the newest again. Written last, so a failure above
+# leaves the machine choosing for itself rather than pinned to something this script did not verify.
+printf '%s\n' "$VERSION" > "$PINNED"
 
-# Deliberately does NOT start anything. The launcher called this and will exec the host next,
-# so starting it here would run two. novox/hq ADR 0005 moved that responsibility; this script
-# installs a version and says so, and nothing else.
-say "rolled back to $VERSION. the launcher will start it."
+# Deliberately does NOT start anything. The launcher called this and will run the host next, so
+# starting it here would run two. novox/hq ADR 0005 moved that responsibility; this script chooses a
+# version and says so, and nothing else.
+say "pinned $VERSION. the launcher will start it."

packaging/rollback_test.sh

@@ -1,9 +1,15 @@
 #!/bin/sh
 # Tests for nox-mesh-host-rollback.
 #
-# It runs on a machine where the host will not start, which is the one moment nobody can afford
-# it to be wrong — and the one moment it is hardest to debug. So it is tested here, against a
-# real filesystem, with a stub package manager that records what it was asked to do.
+# It runs on a machine where the host will not start, which is the one moment nobody can afford it to
+# be wrong — and the one moment it is hardest to debug. So it is tested here, against a real
+# filesystem holding real delivered versions.
+#
+# **These used to stub a package manager.** The script reinstalled the known-good version with
+# `pacman -U` out of the package cache, which no machine in this mesh used and which two of the three
+# operating systems the host is built for do not have (novox/hq ADR 0141). Going back is now choosing
+# a directory, so there is nothing to stub: the thing under test is the filesystem, and a fake would
+# only assert that the fake behaves as expected (novox/hq ADR 0017).
 set -eu
 cd "$(dirname "$0")"
 SCRIPT="$PWD/nox-mesh-host-rollback"
@@ -12,26 +18,15 @@ PASS=0; FAIL=0
 setup() {
 	WORK="$(mktemp -d)"
 	export MESH_HOST_STATE_DIR="$WORK/state"
-	export MESH_HOST_PKG_CACHE="$WORK/cache"
-	export MESH_HOST_PACKAGE="nox-mesh-host"
-	mkdir -p "$MESH_HOST_STATE_DIR" "$MESH_HOST_PKG_CACHE" "$WORK/bin"
+	export MESH_HOST_LIBEXEC="$WORK/libexec"
+	mkdir -p "$MESH_HOST_STATE_DIR" "$MESH_HOST_LIBEXEC/versions"
+}
 
-	# Stubs on PATH. Not mocks of the script's own logic — the boundary is real commands, and
-	# these record the calls so a test can assert what the script asked the machine to do.
-	cat > "$WORK/bin/pacman" <<'STUB'
-#!/bin/sh
-echo "$@" >> "$MESH_HOST_STATE_DIR/pacman.calls"
-[ -n "${STUB_PACMAN_FAILS:-}" ] && exit 1
-exit 0
-STUB
-	cat > "$WORK/bin/systemctl" <<'STUB'
-#!/bin/sh
-echo "$@" >> "$MESH_HOST_STATE_DIR/systemctl.calls"
-exit 0
-STUB
-	chmod +x "$WORK/bin/pacman" "$WORK/bin/systemctl"
-	PATH="$WORK/bin:$PATH"; export PATH
-	unset STUB_PACMAN_FAILS || true
+# deliver a version the way the mesh would: a directory named for it, with the binary inside.
+deliver() {
+	mkdir -p "$MESH_HOST_LIBEXEC/versions/$1"
+	printf '#!/bin/sh\nexit 0\n' > "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host"
+	chmod +x "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host"
 }
 
 check() { # name, condition-description, actual, expected
@@ -41,32 +36,38 @@ check() { # name, condition-description, actual, expected
 
 # --- a normal rollback ---------------------------------------------------------------------
 setup
-echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
-touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst"
-"$SCRIPT" >/dev/null 2>&1
-check "installs the known-good version" "pacman is asked to install the cached package" \
-	"$(grep -c 'nox-mesh-host-1.4.2' "$MESH_HOST_STATE_DIR/pacman.calls" 2>/dev/null || echo 0)" "1"
-# It installs and stops. The launcher execs the host next, and starting it here would run two
-# (novox/hq ADR 0005).
-check "does not start anything itself" "the launcher owns starting" \
-	"$([ -f "$MESH_HOST_STATE_DIR/systemctl.calls" ] && echo started || echo not-started)" "not-started"
+deliver 1.4.2
+deliver 1.5.0
+echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good"
+RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$?
+check "pins the known-good version" "the launcher reads the pin and runs that version instead of the newest" \
+	"$(cat "$MESH_HOST_STATE_DIR/rollback-pinned" 2>/dev/null || echo MISSING)" "1.4.2"
 check "records that it rolled back" "the attempted marker holds the version" \
 	"$(cat "$MESH_HOST_STATE_DIR/rollback-attempted" 2>/dev/null || echo MISSING)" "1.4.2"
+check "succeeds" "a rollback that found its version is not a failure" "$RC" "0"
+# It chooses and stops. The launcher runs the host next, and starting it here would run two
+# (novox/hq ADR 0005).
+check "does not start anything itself" "the launcher owns starting" \
+	"$(ls "$MESH_HOST_STATE_DIR" | grep -c started || true)" "0"
+# The version it rolled back FROM is left alone: it is the newest, and retiring it is the running
+# host's job after a reconcile it completes, never a recovery's.
+check "leaves the failing version on disk" "a recovery deletes nothing" \
+	"$([ -x "$MESH_HOST_LIBEXEC/versions/1.5.0/nox-mesh-host" ] && echo present || echo gone)" "present"
 
 # --- it rolls back only once ---------------------------------------------------------------
 setup
-echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
-echo "1.4.2" > "$MESH_HOST_STATE_DIR/rollback-attempted"
-touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst"
-"$SCRIPT" >/dev/null 2>&1
+deliver 1.4.2
+echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good"
+echo 1.4.2 > "$MESH_HOST_STATE_DIR/rollback-attempted"
+"$SCRIPT" >/dev/null 2>&1 || true
 check "does not roll back twice" "a second failure is the machine, not the binary" \
-	"$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called"
+	"$([ -e "$MESH_HOST_STATE_DIR/rollback-pinned" ] && echo pinned || echo untouched)" "untouched"
 
 # --- nothing to roll back to ---------------------------------------------------------------
 setup
-set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e
+RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$?
 check "no known-good: does nothing" "a host that never reconciled has no version to return to" \
-	"$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called"
+	"$([ -e "$MESH_HOST_STATE_DIR/rollback-attempted" ] && echo attempted || echo untouched)" "untouched"
 # The exit code is asserted from a real run, not from a literal. An earlier version of this
 # compared "0" to "0" and could not fail — which hid an injected fault that made the script die
 # here instead of returning cleanly.
@@ -74,23 +75,29 @@ check "no known-good: exits zero" "an installation failure is not a rollback fai
 
 setup
 printf '   \n' > "$MESH_HOST_STATE_DIR/known-good"
-"$SCRIPT" >/dev/null 2>&1
-check "blank known-good: refuses to guess" "installing nothing and reporting success is the fault this prevents" \
-	"$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called"
+"$SCRIPT" >/dev/null 2>&1 || true
+check "blank known-good: refuses to guess" "pinning nothing and reporting success is the fault this prevents" \
+	"$([ -e "$MESH_HOST_STATE_DIR/rollback-pinned" ] && echo pinned || echo untouched)" "untouched"
 
-# --- the cache was cleaned ------------------------------------------------------------------
+# --- the known-good version is not delivered -------------------------------------------------
+# It was retired, or that host was placed on the machine by hand and never delivered — which is how
+# every first host arrives. Pinning it anyway would have the launcher ignore the pin and start the
+# newest again, which is the binary that is failing.
 setup
-echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
-set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e
-check "missing package: fails loudly" "cannot roll back, and says so rather than reporting success" "$RC" "1"
+deliver 1.5.0
+echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good"
+RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$?
+check "version not delivered: fails loudly" "cannot roll back, and says so rather than reporting success" "$RC" "1"
+check "version not delivered: pins nothing" "a pin the launcher would ignore is worse than none" \
+	"$([ -e "$MESH_HOST_STATE_DIR/rollback-pinned" ] && echo pinned || echo untouched)" "untouched"
 
-# --- the package manager refuses -------------------------------------------------------------
+# --- a version directory with no binary in it ------------------------------------------------
+# An interrupted delivery leaves one. Pinning it would start nothing.
 setup
-echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
-touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst"
-STUB_PACMAN_FAILS=1 ; export STUB_PACMAN_FAILS
-set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e
-check "pacman fails: exits non-zero" "a failed rollback is a failure the launcher must see" "$RC" "1"
+mkdir -p "$MESH_HOST_LIBEXEC/versions/1.4.2"
+echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good"
+RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$?
+check "half-delivered version: fails loudly" "a directory is not a version; the binary is" "$RC" "1"
 
 printf '\nrollback: %d passed, %d failed\n' "$PASS" "$FAIL"
 [ "$FAIL" -eq 0 ]