Merge pull request 'A delivered host knows it is the delivered one' (#58) from fix/163-a-delivered-host-knows-it-is-the-delivered-one into main

novox/hq 04-ISSUES/163. The host asks after every apply whether a newer
host is delivered than the one running, and asked with the link-time
version stamp — which every delivered host carries as "development
build", because the version comes from where the binary sits now (0142).
So a delivered host never matched the newest delivered version, stood
aside on every push for ever, and because standing aside cancels the
report, the mesh never heard from it again.

Measured on two machines: each push produced "host <v> is delivered;
standing aside" for the version already running, then "applied, and could
not tell the mesh: reporting: context canceled". A machine restarting its
host on every push and reporting nothing, reading as healthy.

Asked with the running version now. Half of 0142 was applied to the
report and the known-good record and not here; this is the other half.

cmd/mesh-host/main.go

@@ -51,6 +51,43 @@ var builtFor = ""
 
 var version = "development build"
 
+// runningVersion is this host's version: the directory it was delivered into, or the link-time stamp
+// for one placed by hand.
+//
+// **From where it sits, not from its linker** (novox/hq ADR 0142): "It is unpacked into a directory
+// named for its version, so it can read its own version from its path. The stamp goes, and with it the
+// need for a build to know what it will be called."
+//
+// The mesh's toolchain does not stamp a version, on purpose — a build does not know what it will be
+// called — so a delivered host read as "development build" and the mesh could not tell which host any
+// machine ran (novox/hq 04-ISSUES/161, and 087 for why that matters). The path knows: a delivered host
+// lives at `<libexec>/versions/<version>/<binary>`.
+//
+// A host placed by hand keeps its stamp, which is the honest answer for one the mesh did not deliver.
+func runningVersion() string {
+	self, err := os.Executable()
+	if err != nil {
+		return version
+	}
+	return versionAt(self, version)
+}
+
+// versionAt is runningVersion's decision, with the executable's path and the link-time stamp given —
+// so a test can ask it about a path without being that binary.
+func versionAt(self, stamped string) string {
+	// .../versions/<version>/<binary> — the parent is the version, and its parent is the versions
+	// directory. Checked rather than assumed, so a binary somewhere else does not read a directory
+	// name as a version.
+	dir := filepath.Dir(self)
+	if filepath.Base(filepath.Dir(dir)) != upgrade.VersionsDirName {
+		return stamped
+	}
+	if name := filepath.Base(dir); name != "" && name != "." && name != string(filepath.Separator) {
+		return name
+	}
+	return stamped
+}
+
 const usage = `mesh-host — the node host
 
   profile     what this machine can be asked to do
@@ -254,7 +291,7 @@ func run(ctx context.Context, command string, opts options) error {
 		return runLink(ctx, opts)
 
 	case "version":
-		fmt.Println(version)
+		fmt.Println(runningVersion())
 		return nil
 
 	case "", "help", "-h", "--help":
@@ -594,8 +631,8 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
 	//
 	// A failure to record is reported and does not fail the apply. The apply worked; what is
 	// lost is a rollback's ability to come back here, which is worse to hide than to say.
-	if version != "" {
+	if v := runningVersion(); v != "" {
-		if err := upgrade.RecordKnownGood(upgrade.KnownGoodPath(opts.state), version); err != nil {
+		if err := upgrade.RecordKnownGood(upgrade.KnownGoodPath(opts.state), v); err != nil {
 			fmt.Fprintf(os.Stderr,
 				"mesh-host: applied, but could not record %s as known-good: %v\n"+
 					"  a rollback would have nothing to return to.\n", version, err)
@@ -706,15 +743,18 @@ func enrol(ctx context.Context, opts options) error {
 	fmt.Printf("  signing key         %s\n",
 		base64.StdEncoding.EncodeToString(token.Signer)[:16]+"...")
 
-	// The check that has to happen before this machine says anything.
+	// **The pin is checked by the connection that presents this token, not by a dial of our own**
-	conn, err := link.Dial(token.Broker, token.Fingerprint, opts.timeout)
+	// (novox/hq 04-ISSUES/146). This opened a raw TLS connection to the bus first, which worked
-	if err != nil {
+	// against the broker the mesh used to run and cannot work against the one it runs now: NATS
-		return err
+	// speaks its own protocol before it upgrades to TLS, so an immediate handshake is answered
-	}
+	// with a plaintext line and the enrolment failed with "first record does not look like a TLS
-	defer conn.Close()
+	// handshake" — on every node that has tried to join since the bus changed, which is why this
-	fmt.Println("\nthe broker presented the certificate this token pins")
+	// went unnoticed: none had.
-
+	//
-	conn.Close()
+	// What ADR 0004 requires still holds, and holds better: the client that presents the token
+	// carries the same pinned configuration, reads the server's greeting, upgrades, and the
+	// verification runs inside that handshake — so the one-time secret is sent only after the
+	// certificate has been checked, and nothing of this node's reaches an impostor.
 
 	mine, err := identity.Generate(*name)
 	if err != nil {
@@ -788,10 +828,16 @@ func enrol(ctx context.Context, opts options) error {
 	proof := mine.Sign(link.EnrolProof(token.Secret, mine.Public, mine.Overlay.Public,
 		sealing.Public, serving.Public))
 	// The token says where to go and which certificate that address must present. It says nothing
-	// about which bus is there, and does not need to: every token names the one the mesh runs on
+	// about which bus is there, and does not need to: there is one, and this host knows which
-	// today until the rollout (novox/hq ADR 0116 step 5), and that is what an empty Transport is.
+	// (novox/hq ADR 0131 — the mesh speaks to one seat and the old transport is gone).
+	//
+	// **It used to leave this empty** and mean "whatever the mesh runs today", which was true
+	// while two buses existed and became a refusal the moment one did: an empty transport is not
+	// the bus's name, so every enrolment ended at "this token is for the \"\" bus"
+	// (novox/hq 04-ISSUES/146). Nothing caught it because nothing had enrolled since the bus
+	// changed.
 	reply, err := link.Enrol(ctx,
-		link.Approach{Address: token.Broker, Fingerprint: token.Fingerprint},
+		link.Approach{Address: token.Broker, Fingerprint: token.Fingerprint, Transport: link.OnNATS},
 		*name, token.Secret,
 		mine.Public, mine.Overlay.Public, sealing.Public, serving.Public, reported, proof, found,
 		opts.timeout)
@@ -1008,7 +1054,12 @@ func runLink(ctx context.Context, opts options) error {
 		// standing before this machine leaves the one it is on (novox/hq design 28, task 5.2).
 		adoptDeliveredMembership(identity.Path(opts.state), &mine, say)
 
-		switch next, waiting, err := upgrade.Successor(upgrade.VersionsDir(""), version); {
+		// Asked with the version this host is RUNNING, read from where it sits — not the link-time
+		// stamp, which every delivered host carries as "development build". Asked with the stamp,
+		// a delivered host never matched the newest delivered version, so it stood aside on every
+		// push for ever, and standing aside cancels the report, so the mesh never heard from it
+		// again (novox/hq 04-ISSUES/163).
+		switch next, waiting, err := upgrade.Successor(upgrade.VersionsDir(""), runningVersion()); {
 		case err != nil:
 			// Said, not fatal. A host that cannot read the delivered versions is still running this
 			// machine correctly; what it has lost is the ability to be replaced.
@@ -1246,6 +1297,21 @@ func applyDeclared(ctx context.Context, opts options, raw []byte, sched *apply.S
 	return applyAndKeep(ctx, opts, raw, nil, sched, say)
 }
 
+// announceOr is what the apply writes its detail with, given what the caller has to say things with.
+//
+// **Never nil.** This argument was nil on the serving path, and nil is silence: everything the apply
+// says — a file held, a container replaced, the found firewall retired — was visible when a person ran
+// the one-shot command and discarded in the way the host actually runs (novox/hq 04-ISSUES/143).
+//
+// Named rather than written inline at the call site so both paths reach the apply the same way, and so
+// a reader asking "where does the apply's output go" finds one answer.
+func announceOr(say link.Announce) func(string) {
+	if say == nil {
+		return func(string) {}
+	}
+	return say
+}
+
 // applying serialises applies within this process.
 //
 // **Two things apply here: the link and the reconcile loop**, and each reads the node's state,
@@ -1308,8 +1374,18 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
 
 	// Declared, not carried. A declaration from the mesh removes only what the mesh previously
 	// declared — never what this machine raised for itself from its bundle (04-ISSUES/010).
+	// **What the apply says goes to the console, which is the journal when this runs as a service.**
+	//
+	// It was nil, and nil is silence. The one-shot path has always passed a real one, so every detail
+	// the apply produces — a file held, a container replaced, the found firewall retired — was visible
+	// when a person ran it by hand and discarded in the way the host actually runs. Measured: after a
+	// machine was converged and its found firewall was not retired, what the host decided was
+	// unrecoverable, because it had said it to nobody (novox/hq 04-ISSUES/143).
+	//
+	// `say` already reaches stdout, and the launcher's unit sends that to the journal, so this needs
+	// no new mechanism — only for the argument to be passed.
 	outcome, updated, applyErr := apply.ApplyKeeping(ctx, built, declared, known, store.OriginDeclared,
-		apply.ExecRunner, nil, sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)))
+		apply.ExecRunner, announceOr(say), sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)))
 
 	// The mode the mesh said, recorded whichever way the apply went: the declaration is kept
 	// either way, and the node is held to it from the next reconcile (novox/hq ADR 0100).
@@ -1335,7 +1411,7 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
 		sched.Sync(declared, held)
 	}
 
-	report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw), Host: version}
+	report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw), Host: runningVersion()}
 	// Which of this machine's links face outside, for the filter the mesh writes around them
 	// (novox/hq ADR 0140). Reported whatever the node's mode: a converged node's filter needs it,
 	// and an adopted one becomes converged without a further round trip. A machine that cannot read

cmd/mesh-host/main_test.go

@@ -568,3 +568,28 @@ func TestAConvergedMachineSaysItsOutwardLinksUnasked(t *testing.T) {
 		t.Fatal("a refused report is offered as news about the machine")
 	}
 }
+
+// **A host running as a service says what its apply did.**
+//
+// The serving path passed nil where the apply writes its detail, and nil is silence. The one-shot path
+// has always passed a real function, so everything the apply says was visible when a person ran it by
+// hand and discarded in the way the host actually runs. Measured before this was written: a machine was
+// converged, its found firewall was not retired, and what the host decided was unrecoverable because it
+// had been said to nobody (novox/hq 04-ISSUES/143).
+//
+// This asserts only that the apply's log is never nil and that a line reaches what the caller gave.
+// **It cannot catch the fault it was written for** — a call site passing nil directly — because that is
+// wiring, and wiring is only proved by running the thing. That proof is a deployed host whose journal
+// carries the apply's detail, which is how this fix was verified.
+func TestTheApplysLogIsNeverNil(t *testing.T) {
+	if announceOr(nil) == nil {
+		t.Fatal("a host with nowhere to say things got a nil log, which the apply will call")
+	}
+	announceOr(nil)("this goes nowhere and must not panic")
+
+	var said []string
+	announceOr(func(line string) { said = append(said, line) })("  disabled ufw")
+	if len(said) != 1 || !strings.Contains(said[0], "disabled ufw") {
+		t.Fatalf("the apply's detail did not reach the caller's announce: %v", said)
+	}
+}

cmd/mesh-host/version_from_path_test.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// A component's version comes from where it sits, not from its linker (novox/hq ADR 0142). The mesh's
+// toolchain stamps no version — a build does not know what it will be called — so a delivered host
+// read as "development build" and the mesh could not tell which host a machine ran (04-ISSUES/161).
+
+func TestADeliveredHostReadsItsVersionFromItsPath(t *testing.T) {
+	// A delivered host lives at <libexec>/versions/<version>/<binary>.
+	dir := t.TempDir()
+	versioned := filepath.Join(dir, "versions", "637f65559d16")
+	if err := os.MkdirAll(versioned, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	self := filepath.Join(versioned, "nox-mesh-host")
+	if err := os.WriteFile(self, []byte("#!/bin/sh\n"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if got := versionAt(self, "development build"); got != "637f65559d16" {
+		t.Fatalf("a delivered host read its version as %q", got)
+	}
+}
+
+func TestAHostPlacedByHandKeepsItsStamp(t *testing.T) {
+	// The honest answer for one the mesh did not deliver — and every machine is in that state until
+	// a delivery reaches it.
+	if got := versionAt("/usr/bin/nox-mesh-host", "04a27ca"); got != "04a27ca" {
+		t.Fatalf("a hand-placed host read its version as %q", got)
+	}
+}
+
+func TestADirectoryThatIsNotAVersionIsNotReadAsOne(t *testing.T) {
+	// A binary sitting anywhere else must not have its parent directory's name read as a version.
+	for _, path := range []string{
+		"/opt/somewhere/nox-mesh-host",
+		"/usr/lib/nox-mesh-host/launch",
+		"/home/someone/build/nox-mesh-host",
+	} {
+		if got := versionAt(path, "the stamp"); got != "the stamp" {
+			t.Fatalf("%s read its version as %q", path, got)
+		}
+	}
+}

examples/foundation-first-node-nats.lock

@@ -127,12 +127,21 @@
     {
       "id": "bus-certificate",
       "type": "action",
-      "command": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls",
+      // **The mesh makes its own** (novox/hq 04-ISSUES/146). This ran `openssl` inside the
-                  "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927",
+      // broker's image while the broker was one that carried it; the bus that replaced it has a
-                  "-c", "test -f /tls/tls.crt || (openssl req -x509 -newkey rsa:2048 -nodes -keyout /tls/tls.key -out /tls/tls.crt -days 3650 -subj '/CN=mesh-broker' -addext 'subjectAltName=DNS:mesh-broker,IP:127.0.0.1' >/dev/null 2>&1 && chmod 644 /tls/tls.crt && chmod 600 /tls/tls.key)"],
+      // shell and no openssl, and no other image the bundle names has one either. So the program
-      "verify": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls",
+      // that needs the certificate writes it — already on this machine, since the schema step ran
-                 "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927",
+      // it, and asking nothing of the image it writes into. Self-signed on purpose: a host pins
-                 "-c", "test -s /tls/tls.crt && openssl x509 -in /tls/tls.crt -noout"]
+      // this server's exact certificate (novox/hq ADR 0004), and at this moment there is no mesh
+      // to ask an authority of.
+      // `--user 0:0` because the volume is root's and this image runs as nobody, which is right
+      // for the long-running control plane and wrong for a one-shot writing into a fresh volume.
+      "command": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
+                  "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
+                  "broker", "certificate", "--into", "/tls"],
+      "verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
+                 "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
+                 "broker", "certificate", "--check", "--into", "/tls"]
     },
     {
       "id": "bus-conf-dir",

internal/link/held_note_test.go

@@ -0,0 +1,46 @@
+package link
+
+import (
+	"strings"
+	"testing"
+)
+
+// The count that did not add up was the only symptom sixteen held resources had, and reading it meant
+// opening the node's state file by hand (novox/hq 04-ISSUES/125). The line that says what an apply did
+// says what it did not, too.
+
+func TestTheApplyLineSaysWhatItHeldAndForWhichModule(t *testing.T) {
+	got := heldNote([]Held{
+		{ID: "ca", Module: "route-proxy", Kind: "directory"},
+		{ID: "certs", Module: "route-proxy", Kind: "directory"},
+		{ID: "server", Module: "route-proxy", Kind: "container"},
+		{ID: "mail", Module: "mailu", Kind: "container"},
+	})
+	if !strings.Contains(got, "4 held") {
+		t.Fatalf("the count of what was held is not in the line: %q", got)
+	}
+	// The module is the thing an operator can act on: `take` takes a module.
+	if !strings.Contains(got, "route-proxy: 3") || !strings.Contains(got, "mailu: 1") {
+		t.Fatalf("the line does not break the holds down by module: %q", got)
+	}
+	// Ordered, so two machines holding the same things read the same and a diff of two reports is
+	// about what changed.
+	if strings.Index(got, "mailu") > strings.Index(got, "route-proxy") {
+		t.Fatalf("modules are not in a stated order: %q", got)
+	}
+	// It says why, because "held" alone reads as a failure and this is correct behaviour.
+	if !strings.Contains(got, "taken") {
+		t.Fatalf("the line does not say a hold ends when the module is taken: %q", got)
+	}
+}
+
+func TestAnApplyThatHeldNothingSaysNothingExtra(t *testing.T) {
+	// A converged machine holds nothing, which is most applies. Reporting "0 held" on every one of
+	// them is how a line stops being read.
+	if got := heldNote(nil); got != "" {
+		t.Fatalf("an apply with no holds added %q to its line", got)
+	}
+	if got := heldNote([]Held{}); got != "" {
+		t.Fatalf("an apply with no holds added %q to its line", got)
+	}
+}

internal/link/pinned.go

@@ -73,8 +73,19 @@ func PinnedConfig(pin string) (*tls.Config, error) {
 	}, nil
 }
 
-// Dial opens a TLS connection to the broker, refusing anything but the pinned certificate.
+// dialPinned completes a TLS handshake against an address, refusing anything but the pinned
-func Dial(address, pin string, timeout time.Duration) (*tls.Conn, error) {
+// certificate.
+//
+// **Not how the bus is reached, and it used to be** (novox/hq 04-ISSUES/146). Enrolment opened one
+// of these before it said anything, which was right while the broker answered TLS immediately and
+// wrong the moment the mesh moved to a bus that speaks its own protocol first. The pin itself was
+// never the problem — PinnedConfig is what the NATS client is given, and the verification runs
+// inside the handshake that client performs.
+//
+// It stays here because this is where the pin is proven: the tests beside it run a real TLS server
+// and assert that a wrong certificate is refused before a byte of application data is sent. What it
+// must not become again is something a caller uses to reach the bus.
+func dialPinned(address, pin string, timeout time.Duration) (*tls.Conn, error) {
 	config, err := PinnedConfig(pin)
 	if err != nil {
 		return nil, err
@@ -86,7 +97,7 @@ func Dial(address, pin string, timeout time.Duration) (*tls.Conn, error) {
 		if errors.Is(err, ErrWrongCertificate) {
 			return nil, err
 		}
-		return nil, fmt.Errorf("cannot reach the broker at %s: %w", address, err)
+		return nil, fmt.Errorf("cannot reach %s: %w", address, err)
 	}
 	return conn, nil
 }

internal/link/pinned_test.go

@@ -63,7 +63,7 @@ func server(t *testing.T) (address string, fingerprint string) {
 
 func TestTheRightBrokerIsAccepted(t *testing.T) {
 	address, pin := server(t)
-	conn, err := Dial(address, pin, 5*time.Second)
+	conn, err := dialPinned(address, pin, 5*time.Second)
 	if err != nil {
 		t.Fatalf("the broker its token describes was refused: %v", err)
 	}
@@ -76,7 +76,7 @@ func TestADifferentBrokerIsRefused(t *testing.T) {
 	address, _ := server(t)
 	_, other := server(t)
 
-	_, err := Dial(address, other, 5*time.Second)
+	_, err := dialPinned(address, other, 5*time.Second)
 	if err == nil {
 		t.Fatal("a broker presenting a different certificate was accepted")
 	}
@@ -130,7 +130,7 @@ func TestNothingIsSentToTheWrongBroker(t *testing.T) {
 
 	// A pin for a certificate this server does not have.
 	_, elsewhere := server(t)
-	if _, err := Dial(listener.Addr().String(), elsewhere, 5*time.Second); err == nil {
+	if _, err := dialPinned(listener.Addr().String(), elsewhere, 5*time.Second); err == nil {
 		t.Fatal("the impostor was accepted")
 	}
 	if n := <-received; n > 0 {
@@ -160,7 +160,7 @@ func TestAnUnreachableBrokerIsAnOrdinaryFailure(t *testing.T) {
 	address := listener.Addr().String()
 	listener.Close()
 
-	_, err = Dial(address, pin, 2*time.Second)
+	_, err = dialPinned(address, pin, 2*time.Second)
 	if err == nil {
 		t.Fatal("dialling a closed port succeeded")
 	}

internal/link/run.go

@@ -6,6 +6,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"sort"
+	"strings"
 	"time"
 )
 
@@ -250,9 +252,11 @@ func Run(ctx context.Context, m Membership, apply Applier, say Announce, timeout
 			case report.Refused != "":
 				say("refused a declaration: " + report.Refused)
 			case len(report.Failed) > 0:
-				say(fmt.Sprintf("applied %d and failed: %v", len(report.Applied), report.Failed))
+				say(fmt.Sprintf("applied %d and failed: %v%s",
+					len(report.Applied), report.Failed, heldNote(report.Held)))
 			default:
-				say(fmt.Sprintf("applied %d resource(s)", len(report.Applied)))
+				say(fmt.Sprintf("applied %d resource(s)%s",
+					len(report.Applied), heldNote(report.Held)))
 			}
 			publishReport(ctx, link, m, report, say, timeout)
 			// Settled after the report is published. A node that dies between applying and
@@ -392,3 +396,36 @@ func publishAlive(ctx context.Context, bus Bus, m Membership, say Announce,
 		say("could not tell the mesh this node is here: " + err.Error())
 	}
 }
+
+// heldNote is what this apply did NOT do, for the line that says what it did.
+//
+// **A count that does not add up is the only symptom a held resource had** (novox/hq 04-ISSUES/125).
+// An adopted node keeps what it found until its module is taken (ADR 0100), and that is correct — but
+// it was recorded only in the node's own state file. On the edge cut-over the mesh sent 346 resources,
+// the journal said it applied 330, and nothing anywhere said which sixteen or why. Reading it took
+// opening state.json by hand; not reading it took every public name on the machine down, because the
+// operator had four green surfaces and a discrepancy nobody could interpret.
+//
+// So the line that reports the apply carries it. Grouped by module and ordered by name, because the
+// sentence an operator needs is "route-proxy is assigned and not taken", and the module is the thing
+// they can act on — `take` is the verb, and it takes a module.
+func heldNote(held []Held) string {
+	if len(held) == 0 {
+		return ""
+	}
+	byModule := map[string]int{}
+	for _, h := range held {
+		byModule[h.Module]++
+	}
+	names := make([]string, 0, len(byModule))
+	for name := range byModule {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+	parts := make([]string, 0, len(names))
+	for _, name := range names {
+		parts = append(parts, fmt.Sprintf("%s: %d", name, byModule[name]))
+	}
+	return fmt.Sprintf(", %d held until their module is taken (%s)",
+		len(held), strings.Join(parts, ", "))
+}

packaging/launcher_is_delivered_test.go

@@ -0,0 +1,48 @@
+package packaging_test
+
+import (
+	"encoding/json"
+	"os"
+	"testing"
+)
+
+// The mesh delivers the launcher, so the manifest carries a copy of it (novox/hq 04-ISSUES/142).
+//
+// **Two copies of one script is a drift waiting to happen**, and the only reason to accept it is that
+// a file resource is written atomically — temp file, then rename — while an archive writes in place
+// with truncate. The running launcher keeps the inode it was started from and the next start picks up
+// the new one; unpacking an archive over it would truncate the file a running shell is reading.
+//
+// So: two copies, and this is the check that they are the same one.
+func TestTheManifestCarriesTheLauncherExactly(t *testing.T) {
+	onDisk, err := os.ReadFile("nox-mesh-host-launch")
+	if err != nil {
+		t.Fatal(err)
+	}
+	raw, err := os.ReadFile("../module.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var manifest struct {
+		Resources []struct {
+			ID      string `json:"id"`
+			Content string `json:"content"`
+		} `json:"resources"`
+	}
+	if err := json.Unmarshal(raw, &manifest); err != nil {
+		t.Fatal(err)
+	}
+	for _, r := range manifest.Resources {
+		if r.ID != "launcher" {
+			continue
+		}
+		if r.Content != string(onDisk) {
+			t.Fatal("the launcher the mesh would deliver is not the launcher in this repository. " +
+				"Copy packaging/nox-mesh-host-launch into module.json's `launcher` resource — the " +
+				"machines run what the manifest says, and this file is what gets reviewed")
+		}
+		return
+	}
+	t.Fatal("module.json declares no `launcher` resource, so nothing delivers the launcher and a " +
+		"delivered host version is never started")
+}