Say what a container's host entries now are

novox/hq ADR 0148: the mesh's names are no longer among them, only what
the module declared. Comment only; the digest is unchanged.

cmd/mesh-host/main.go

@@ -51,6 +51,43 @@ var builtFor = ""
 
 var version = "development build"
 
+// runningVersion is this host's version: the directory it was delivered into, or the link-time stamp
+// for one placed by hand.
+//
+// **From where it sits, not from its linker** (novox/hq ADR 0142): "It is unpacked into a directory
+// named for its version, so it can read its own version from its path. The stamp goes, and with it the
+// need for a build to know what it will be called."
+//
+// The mesh's toolchain does not stamp a version, on purpose — a build does not know what it will be
+// called — so a delivered host read as "development build" and the mesh could not tell which host any
+// machine ran (novox/hq 04-ISSUES/161, and 087 for why that matters). The path knows: a delivered host
+// lives at `<libexec>/versions/<version>/<binary>`.
+//
+// A host placed by hand keeps its stamp, which is the honest answer for one the mesh did not deliver.
+func runningVersion() string {
+	self, err := os.Executable()
+	if err != nil {
+		return version
+	}
+	return versionAt(self, version)
+}
+
+// versionAt is runningVersion's decision, with the executable's path and the link-time stamp given —
+// so a test can ask it about a path without being that binary.
+func versionAt(self, stamped string) string {
+	// .../versions/<version>/<binary> — the parent is the version, and its parent is the versions
+	// directory. Checked rather than assumed, so a binary somewhere else does not read a directory
+	// name as a version.
+	dir := filepath.Dir(self)
+	if filepath.Base(filepath.Dir(dir)) != upgrade.VersionsDirName {
+		return stamped
+	}
+	if name := filepath.Base(dir); name != "" && name != "." && name != string(filepath.Separator) {
+		return name
+	}
+	return stamped
+}
+
 const usage = `mesh-host — the node host
 
   profile     what this machine can be asked to do
@@ -254,7 +291,7 @@ func run(ctx context.Context, command string, opts options) error {
 		return runLink(ctx, opts)
 
 	case "version":
-		fmt.Println(version)
+		fmt.Println(runningVersion())
 		return nil
 
 	case "", "help", "-h", "--help":
@@ -420,10 +457,10 @@ func short(digest string) string {
 // them again — and everything the mesh declared read as no longer declared and removed. Even the
 // very declaration the mesh last sent, applied from a file, would plan to remove the foundation.
 // `apply FILE` is for a machine the mesh has not spoken to, and is refused saying so.
-func refuseStale(known store.State, kept store.Declared, keptErr error, digest string, from provenance) error {
+func refuseStale(known store.State, kept store.Declared, keptErr error, digest string, from provenance, sequence int64) error {
 	switch from {
 	case fromDeclared:
-		return nil
+		return refuseOlder(kept, keptErr, sequence)
 	case fromBundle:
 		if known.Genesis == nil || known.Genesis.Digest == digest {
 			return nil
@@ -520,7 +557,7 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
 	if err := apply.CheckMode(known, d); err != nil {
 		return err
 	}
-	if err := refuseStale(known, kept, keptErr, digest, from); err != nil {
+	if err := refuseStale(known, kept, keptErr, digest, from, d.Sequence); err != nil {
 		return err
 	}
 
@@ -594,8 +631,8 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
 	//
 	// A failure to record is reported and does not fail the apply. The apply worked; what is
 	// lost is a rollback's ability to come back here, which is worse to hide than to say.
-	if version != "" {
-		if err := upgrade.RecordKnownGood(upgrade.KnownGoodPath(opts.state), version); err != nil {
+	if v := runningVersion(); v != "" {
+		if err := upgrade.RecordKnownGood(upgrade.KnownGoodPath(opts.state), v); err != nil {
 			fmt.Fprintf(os.Stderr,
 				"mesh-host: applied, but could not record %s as known-good: %v\n"+
 					"  a rollback would have nothing to return to.\n", version, err)
@@ -816,13 +853,6 @@ func enrol(ctx context.Context, opts options) error {
 		Fingerprint: firstNonEmpty(reply.Fingerprint, token.Fingerprint),
 		Signer:      firstNonEmpty2(reply.Signer, token.Signer),
 		Password:    reply.Password,
-		// **Which bus this membership is for, said rather than left empty** (novox/hq
-		// 04-ISSUES/146). The link refuses a membership that names another bus, and an empty name
-		// is not this one's — so a node enrolled without it came up and reconnected for ever
-		// against its own record: "this membership is for \"\", and the mesh's bus is nats". The
-		// reply does not carry it because there is one bus and the host knows which (ADR 0131);
-		// what was missing was writing that down where the link reads it.
-		Transport: link.OnNATS,
 	}
 	if mine.Membership.Password == "" {
 		// The mesh did not replace the token's secret, so it is still this node's broker
@@ -1024,7 +1054,12 @@ func runLink(ctx context.Context, opts options) error {
 		// standing before this machine leaves the one it is on (novox/hq design 28, task 5.2).
 		adoptDeliveredMembership(identity.Path(opts.state), &mine, say)
 
-		switch next, waiting, err := upgrade.Successor(upgrade.VersionsDir(""), version); {
+		// Asked with the version this host is RUNNING, read from where it sits — not the link-time
+		// stamp, which every delivered host carries as "development build". Asked with the stamp,
+		// a delivered host never matched the newest delivered version, so it stood aside on every
+		// push for ever, and standing aside cancels the report, so the mesh never heard from it
+		// again (novox/hq 04-ISSUES/163).
+		switch next, waiting, err := upgrade.Successor(upgrade.VersionsDir(""), runningVersion()); {
 		case err != nil:
 			// Said, not fatal. A host that cannot read the delivered versions is still running this
 			// machine correctly; what it has lost is the ability to be replaced.
@@ -1376,7 +1411,7 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
 		sched.Sync(declared, held)
 	}
 
-	report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw), Host: version}
+	report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw), Host: runningVersion()}
 	// Which of this machine's links face outside, for the filter the mesh writes around them
 	// (novox/hq ADR 0140). Reported whatever the node's mode: a converged node's filter needs it,
 	// and an adopted one becomes converged without a further round trip. A machine that cannot read
@@ -1559,3 +1594,32 @@ func adoptDeliveredMembership(identityPath string, mine *identity.Identity, say
 	say(fmt.Sprintf("moving to the %s bus at %s — restarting to dial it", next.Transport, next.Broker))
 	os.Exit(0)
 }
+
+// refuseOlder refuses a declaration from the mesh that is older than the one this node holds.
+//
+// **By sequence, not by arrival** (novox/hq 04-ISSUES/107). What the host kept is the last thing
+// the mesh said, signed; a declaration whose sequence is lower was composed before it, whatever
+// order they arrived in — a backlog drained after the node was away, or a broker that split a burst.
+// Applying it would make the machine into something the mesh had already moved past, which is the
+// incident of issue 104 by another door.
+//
+// Only when both sides claim an order. A declaration with no sequence is one an older controller
+// sent, and one kept with no sequence is one this host received before it understood them; in either
+// case there is no order to compare, and refusing on a guess would strand the node the moment the
+// controller is older than the host. Equal is the same declaration again, which reconciling is for.
+func refuseOlder(kept store.Declared, keptErr error, sequence int64) error {
+	if sequence == 0 || keptErr != nil {
+		return nil
+	}
+	last, err := declaration.ParseTrusted(kept.Declaration)
+	if err != nil || last.Sequence == 0 {
+		return nil
+	}
+	if sequence < last.Sequence {
+		return fmt.Errorf("this declaration is older than what the mesh last said to this node: it "+
+			"is sequence %d, and the one kept here is %d. It arrived late — a backlog, or a broker "+
+			"that split a burst — and applying it would make this machine into something the mesh has "+
+			"already moved past. Refused whole; nothing was applied", sequence, last.Sequence)
+	}
+	return nil
+}

cmd/mesh-host/order_test.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// A declaration carries no order, so a host cannot tell an older one from a newer (novox/hq
+// 04-ISSUES/107). Its only identity was the digest of its bytes: "not the last" could be said,
+// "older" could not.
+
+func keptWith(t *testing.T, sequence int64) store.Declared {
+	t.Helper()
+	body, err := json.Marshal(map[string]any{
+		"declaration": 1, "resources": []any{}, "owns_nothing": true, "sequence": sequence,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return store.Declared{Declaration: body, Signature: []byte("x")}
+}
+
+func TestAnOlderDeclarationFromTheMeshIsRefused(t *testing.T) {
+	err := refuseOlder(keptWith(t, 7), nil, 5)
+	if err == nil {
+		t.Fatal("sequence 5 was accepted over a kept 7")
+	}
+	if !strings.Contains(err.Error(), "older") || !strings.Contains(err.Error(), "nothing was applied") {
+		t.Fatalf("the refusal does not say what it is: %v", err)
+	}
+}
+
+func TestANewerOrEqualDeclarationIsNot(t *testing.T) {
+	if err := refuseOlder(keptWith(t, 7), nil, 8); err != nil {
+		t.Fatalf("sequence 8 was refused over a kept 7: %v", err)
+	}
+	// Equal is the same declaration again, which reconciling is for.
+	if err := refuseOlder(keptWith(t, 7), nil, 7); err != nil {
+		t.Fatalf("the same sequence was refused: %v", err)
+	}
+}
+
+func TestNoOrderClaimedMeansNoOrderCompared(t *testing.T) {
+	// An older controller sends none; a host that received before it understood them kept none.
+	// Refusing on a guess would strand a node the moment the controller is older than the host.
+	if err := refuseOlder(keptWith(t, 7), nil, 0); err != nil {
+		t.Fatalf("a declaration claiming no order was refused: %v", err)
+	}
+	if err := refuseOlder(keptWith(t, 0), nil, 3); err != nil {
+		t.Fatalf("a declaration was refused against a kept one that claimed no order: %v", err)
+	}
+	if err := refuseOlder(store.Declared{}, store.ErrNothingDeclared, 3); err != nil {
+		t.Fatalf("a first declaration was refused: %v", err)
+	}
+}

cmd/mesh-host/version_from_path_test.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// A component's version comes from where it sits, not from its linker (novox/hq ADR 0142). The mesh's
+// toolchain stamps no version — a build does not know what it will be called — so a delivered host
+// read as "development build" and the mesh could not tell which host a machine ran (04-ISSUES/161).
+
+func TestADeliveredHostReadsItsVersionFromItsPath(t *testing.T) {
+	// A delivered host lives at <libexec>/versions/<version>/<binary>.
+	dir := t.TempDir()
+	versioned := filepath.Join(dir, "versions", "637f65559d16")
+	if err := os.MkdirAll(versioned, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	self := filepath.Join(versioned, "nox-mesh-host")
+	if err := os.WriteFile(self, []byte("#!/bin/sh\n"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if got := versionAt(self, "development build"); got != "637f65559d16" {
+		t.Fatalf("a delivered host read its version as %q", got)
+	}
+}
+
+func TestAHostPlacedByHandKeepsItsStamp(t *testing.T) {
+	// The honest answer for one the mesh did not deliver — and every machine is in that state until
+	// a delivery reaches it.
+	if got := versionAt("/usr/bin/nox-mesh-host", "04a27ca"); got != "04a27ca" {
+		t.Fatalf("a hand-placed host read its version as %q", got)
+	}
+}
+
+func TestADirectoryThatIsNotAVersionIsNotReadAsOne(t *testing.T) {
+	// A binary sitting anywhere else must not have its parent directory's name read as a version.
+	for _, path := range []string{
+		"/opt/somewhere/nox-mesh-host",
+		"/usr/lib/nox-mesh-host/launch",
+		"/home/someone/build/nox-mesh-host",
+	} {
+		if got := versionAt(path, "the stamp"); got != "the stamp" {
+			t.Fatalf("%s read its version as %q", path, got)
+		}
+	}
+}

internal/apply/apply.go

@@ -1501,13 +1501,15 @@ func containerSpecReading(r *declaration.Container, declares, reads map[string]s
 	for _, a := range r.Args {
 		b.WriteString("arg " + a + "\n")
 	}
-	// **The mesh's names are part of what a container is** (novox/hq 04-ISSUES/135). A container
-	// resolves every other machine and every public name through the entries the mesh gives it at
-	// creation, and nothing re-reads them afterwards — so a container left alone when the roster
-	// moved is one that cannot reach anything by name, for ever, while every check reports it
-	// running. That is exactly what happened when this mesh's overlay range changed: one container
-	// whose image and files never changed kept an address five days out of date and restarted
-	// 2286 times against a database it could no longer find.
+	// **A container's declared names are part of what it is** (novox/hq 04-ISSUES/135). What is
+	// here is what the module declared for itself and nothing else: the mesh's own names are no
+	// longer written into a container (ADR 0148) — they were once, every container got the whole
+	// roster at creation and nothing re-read it, so one left alone when the roster moved could not
+	// reach anything by name for as long as it ran while every check reported it running; and once
+	// the roster was in this digest so that could be caught, one name moving anywhere replaced
+	// every container in the mesh (04-ISSUES/151). A container resolves a mesh name through the
+	// machine's resolver at the moment it asks. What a module declares does not move when the
+	// roster does, so hashing it costs nothing and catches a manifest that changed.
 	//
 	// Sorted, so the digest does not move for a reordering nobody made.
 	hosts := append([]string(nil), r.Hosts...)

internal/declaration/declaration.go

@@ -1137,6 +1137,19 @@ type Declaration struct {
 	// converged node — which is every node the mesh raised before adoption existed, and so the
 	// only form an older controller ever sends (novox/hq ADR 0100).
 	Adoption *Adoption
+
+	// Sequence orders this declaration against every other the mesh has sent this node: each
+	// send is one higher than the last, assigned under the control plane's hold on the node
+	// (novox/hq 04-ISSUES/107). Zero is a declaration that carries no order — every one an older
+	// controller sent, and the bundle genesis applies — and a host makes no ordering claim about
+	// one of those.
+	//
+	// **The one property a declaration needs that its signature does not give it.** A signature
+	// says the mesh sent this; it cannot say the mesh sent it AFTER the one the host is holding.
+	// Before this, "older" was inferred from arrival within a batch and a 750ms window, and a
+	// backlog longer than the batch, or a slow broker, applied a declaration the mesh had already
+	// superseded.
+	Sequence int64
 }
 
 // Adoption is a node's mode, as the controller records it: the node is adopted, and these are
@@ -1274,6 +1287,9 @@ type envelope struct {
 	// a bug quietly strip a machine.
 	OwnsNothing bool              `json:"owns_nothing,omitempty"`
 	Resources   []json.RawMessage `json:"resources"`
+	// Sequence is optional on the wire, so a controller that does not send one is still
+	// understood: absent reads as zero, which is "no ordering claimed" rather than "first".
+	Sequence int64 `json:"sequence,omitempty"`
 }
 
 func parse(raw []byte, allowActions bool) (*Declaration, error) {
@@ -1290,7 +1306,7 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
 			env.Version, Version)}}
 	}
 
-	d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption}
+	d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption, Sequence: env.Sequence}
 	var problems []string
 
 	if len(env.Resources) == 0 && !env.OwnsNothing {

internal/link/asking_nats.go

@@ -3,7 +3,6 @@ package link
 import (
 	"context"
 	"crypto/rand"
-	"crypto/sha256"
 	"encoding/hex"
 	"errors"
 	"fmt"
@@ -64,15 +63,8 @@ func presentNats(_ context.Context, to Approach, node, secret string,
 	// subscribe its own inbox and nothing else (design 25 §6). The secret is its password, the same
 	// string the request claims, so the server proves somebody holds the token and the request
 	// proves the same thing to the controller without it having to ask the server who connected.
-	// **Its own inbox space, because that is the only one it may listen in** (novox/hq
-	// 04-ISSUES/146). A JetStream publish waits for the stream's acknowledgement on an inbox the
-	// client picks, and the client's default is `_INBOX.<random>` — which this user may not
-	// subscribe to, so the enrolment failed with a permissions violation on a subject nobody had
-	// chosen. The permission is `_INBOX.enrol.<node>.>` (design 25 §6), so the client is told to
-	// pick its inboxes there; the reply address below is in the same space for the same reason.
 	conn, err := nats.Connect(natsURL(to.Address),
 		nats.Secure(config),
-		nats.CustomInboxPrefix("_INBOX.enrol."+node),
 		nats.UserInfo("enrol."+node, secret),
 		nats.Name("mesh-host/enrol/"+node),
 		nats.Timeout(timeout),
@@ -132,19 +124,7 @@ func (a *natsAsking) Ask(ctx context.Context, request []byte, wait time.Duration
 	defer cancel()
 	// Into the stream and awaited: an enrolment the bus never accepted must fail here rather than be
 	// assumed, because the node has nothing else to go on.
-	//
-	// **Once, however many times it is sent** (novox/hq 04-ISSUES/146). The client re-publishes when
-	// an acknowledgement is slow, and the mesh enrolled the machine on each copy — minting a second
-	// credential, which replaced the first, which is the one the node had already been given. The
-	// machine then reconnected for ever as a user whose password the mesh had rotated out from under
-	// it, and the controller's log said "enrolled anchor" twice in the same second.
-	//
-	// The id is the message: the same bytes carry the same id, so the stream discards the client's
-	// own retry, and a genuine second attempt — which carries a new reply address — is a different
-	// message and is let through.
-	sum := sha256.Sum256(addressed)
-	if _, err := a.js.Publish(EnrolSubject, addressed,
-		nats.MsgId(hex.EncodeToString(sum[:])), nats.Context(publish)); err != nil {
+	if _, err := a.js.Publish(EnrolSubject, addressed, nats.Context(publish)); err != nil {
 		return nil, fmt.Errorf("cannot ask the mesh to enrol this node: %w", err)
 	}
 

internal/link/held_note_test.go

@@ -0,0 +1,46 @@
+package link
+
+import (
+	"strings"
+	"testing"
+)
+
+// The count that did not add up was the only symptom sixteen held resources had, and reading it meant
+// opening the node's state file by hand (novox/hq 04-ISSUES/125). The line that says what an apply did
+// says what it did not, too.
+
+func TestTheApplyLineSaysWhatItHeldAndForWhichModule(t *testing.T) {
+	got := heldNote([]Held{
+		{ID: "ca", Module: "route-proxy", Kind: "directory"},
+		{ID: "certs", Module: "route-proxy", Kind: "directory"},
+		{ID: "server", Module: "route-proxy", Kind: "container"},
+		{ID: "mail", Module: "mailu", Kind: "container"},
+	})
+	if !strings.Contains(got, "4 held") {
+		t.Fatalf("the count of what was held is not in the line: %q", got)
+	}
+	// The module is the thing an operator can act on: `take` takes a module.
+	if !strings.Contains(got, "route-proxy: 3") || !strings.Contains(got, "mailu: 1") {
+		t.Fatalf("the line does not break the holds down by module: %q", got)
+	}
+	// Ordered, so two machines holding the same things read the same and a diff of two reports is
+	// about what changed.
+	if strings.Index(got, "mailu") > strings.Index(got, "route-proxy") {
+		t.Fatalf("modules are not in a stated order: %q", got)
+	}
+	// It says why, because "held" alone reads as a failure and this is correct behaviour.
+	if !strings.Contains(got, "taken") {
+		t.Fatalf("the line does not say a hold ends when the module is taken: %q", got)
+	}
+}
+
+func TestAnApplyThatHeldNothingSaysNothingExtra(t *testing.T) {
+	// A converged machine holds nothing, which is most applies. Reporting "0 held" on every one of
+	// them is how a line stops being read.
+	if got := heldNote(nil); got != "" {
+		t.Fatalf("an apply with no holds added %q to its line", got)
+	}
+	if got := heldNote([]Held{}); got != "" {
+		t.Fatalf("an apply with no holds added %q to its line", got)
+	}
+}

internal/link/order_test.go

@@ -0,0 +1,50 @@
+package link
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+)
+
+// The drain picked the last to arrive. A backlog longer than the batch, or a broker that split a
+// burst, delivered a superseded declaration last (novox/hq 04-ISSUES/107).
+
+func sequenced(t *testing.T, n int64) *said {
+	t.Helper()
+	inner, err := json.Marshal(map[string]any{"declaration": 1, "resources": []any{}, "sequence": n})
+	if err != nil {
+		t.Fatal(err)
+	}
+	body, err := json.Marshal(Signed{Declaration: inner, Signature: []byte("s")})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return &said{body: body}
+}
+
+func TestTheDrainKeepsTheHighestSequenceNotTheLastToArrive(t *testing.T) {
+	waiting := make(chan Declaration, 8)
+	waiting <- sequenced(t, 9)
+	waiting <- sequenced(t, 4) // arrived last, composed earlier
+	latest, aside := newest(waiting, sequenced(t, 8), 30*time.Millisecond)
+	if got := sequenceOf(latest.Body()); got != 9 {
+		t.Fatalf("the drain kept sequence %d, and 9 was waiting", got)
+	}
+	if len(aside) != 2 {
+		t.Fatalf("%d set aside, wanted 2 (the 8 and the late 4)", len(aside))
+	}
+}
+
+func TestWithoutSequencesTheLastToArriveStillWins(t *testing.T) {
+	// The behaviour this had before, kept for a controller that sends no order.
+	apply, aside := newest(arriving("two", "three"), &said{body: []byte("one")}, 30*time.Millisecond)
+	if string(apply.Body()) != "three" || len(aside) != 2 {
+		t.Fatalf("applied %q with %d set aside", apply.Body(), len(aside))
+	}
+}
+
+func TestAnUnreadableBodyClaimsNoOrder(t *testing.T) {
+	if got := sequenceOf([]byte("not json")); got != 0 {
+		t.Fatalf("garbage claimed sequence %d", got)
+	}
+}

internal/link/run.go

@@ -6,6 +6,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"sort"
+	"strings"
 	"time"
 )
 
@@ -250,9 +252,11 @@ func Run(ctx context.Context, m Membership, apply Applier, say Announce, timeout
 			case report.Refused != "":
 				say("refused a declaration: " + report.Refused)
 			case len(report.Failed) > 0:
-				say(fmt.Sprintf("applied %d and failed: %v", len(report.Applied), report.Failed))
+				say(fmt.Sprintf("applied %d and failed: %v%s",
+					len(report.Applied), report.Failed, heldNote(report.Held)))
 			default:
-				say(fmt.Sprintf("applied %d resource(s)", len(report.Applied)))
+				say(fmt.Sprintf("applied %d resource(s)%s",
+					len(report.Applied), heldNote(report.Held)))
 			}
 			publishReport(ctx, link, m, report, say, timeout)
 			// Settled after the report is published. A node that dies between applying and
@@ -296,6 +300,16 @@ func newest(arriving <-chan Declaration, first Declaration, window time.Duration
 			if !ok {
 				return latest, superseded
 			}
+			// **By sequence when both carry one, by arrival when either does not** (novox/hq
+			// 04-ISSUES/107). Arrival is what this window had to go on, and it is wrong exactly
+			// when it matters — a backlog drained out of order. A declaration that says where it
+			// stands is believed over when it turned up; one that does not is the older
+			// controller's, and arrival is all there is.
+			if sequenceOf(next.Body()) < sequenceOf(latest.Body()) &&
+				sequenceOf(next.Body()) > 0 && sequenceOf(latest.Body()) > 0 {
+				superseded = append(superseded, next)
+				continue
+			}
 			superseded = append(superseded, latest)
 			latest = next
 		case <-time.After(window):
@@ -304,6 +318,24 @@ func newest(arriving <-chan Declaration, first Declaration, window time.Duration
 	}
 }
 
+// sequenceOf is the order a signed declaration claims, or zero when it claims none or cannot be
+// read. Read from the envelope alone; the signature is verified later, when the winner is applied,
+// and a forged message that lied about its sequence would only set aside real ones — which are
+// reported as set aside, and the next push sends the current one again.
+func sequenceOf(body []byte) int64 {
+	var signed Signed
+	if err := json.Unmarshal(body, &signed); err != nil {
+		return 0
+	}
+	var d struct {
+		Sequence int64 `json:"sequence"`
+	}
+	if err := json.Unmarshal(signed.Declaration, &d); err != nil {
+		return 0
+	}
+	return d.Sequence
+}
+
 // declaredIn is the id a signed declaration carries, for a report about one that was not applied.
 // Empty if the message is not one — a forged or garbled message is refused by handleBody when its
 // turn comes; here it is only named.
@@ -392,3 +424,36 @@ func publishAlive(ctx context.Context, bus Bus, m Membership, say Announce,
 		say("could not tell the mesh this node is here: " + err.Error())
 	}
 }
+
+// heldNote is what this apply did NOT do, for the line that says what it did.
+//
+// **A count that does not add up is the only symptom a held resource had** (novox/hq 04-ISSUES/125).
+// An adopted node keeps what it found until its module is taken (ADR 0100), and that is correct — but
+// it was recorded only in the node's own state file. On the edge cut-over the mesh sent 346 resources,
+// the journal said it applied 330, and nothing anywhere said which sixteen or why. Reading it took
+// opening state.json by hand; not reading it took every public name on the machine down, because the
+// operator had four green surfaces and a discrepancy nobody could interpret.
+//
+// So the line that reports the apply carries it. Grouped by module and ordered by name, because the
+// sentence an operator needs is "route-proxy is assigned and not taken", and the module is the thing
+// they can act on — `take` is the verb, and it takes a module.
+func heldNote(held []Held) string {
+	if len(held) == 0 {
+		return ""
+	}
+	byModule := map[string]int{}
+	for _, h := range held {
+		byModule[h.Module]++
+	}
+	names := make([]string, 0, len(byModule))
+	for name := range byModule {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+	parts := make([]string, 0, len(names))
+	for _, name := range names {
+		parts = append(parts, fmt.Sprintf("%s: %d", name, byModule[name]))
+	}
+	return fmt.Sprintf(", %d held until their module is taken (%s)",
+		len(held), strings.Join(parts, ", "))
+}

packaging/launcher_is_delivered_test.go

@@ -0,0 +1,48 @@
+package packaging_test
+
+import (
+	"encoding/json"
+	"os"
+	"testing"
+)
+
+// The mesh delivers the launcher, so the manifest carries a copy of it (novox/hq 04-ISSUES/142).
+//
+// **Two copies of one script is a drift waiting to happen**, and the only reason to accept it is that
+// a file resource is written atomically — temp file, then rename — while an archive writes in place
+// with truncate. The running launcher keeps the inode it was started from and the next start picks up
+// the new one; unpacking an archive over it would truncate the file a running shell is reading.
+//
+// So: two copies, and this is the check that they are the same one.
+func TestTheManifestCarriesTheLauncherExactly(t *testing.T) {
+	onDisk, err := os.ReadFile("nox-mesh-host-launch")
+	if err != nil {
+		t.Fatal(err)
+	}
+	raw, err := os.ReadFile("../module.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var manifest struct {
+		Resources []struct {
+			ID      string `json:"id"`
+			Content string `json:"content"`
+		} `json:"resources"`
+	}
+	if err := json.Unmarshal(raw, &manifest); err != nil {
+		t.Fatal(err)
+	}
+	for _, r := range manifest.Resources {
+		if r.ID != "launcher" {
+			continue
+		}
+		if r.Content != string(onDisk) {
+			t.Fatal("the launcher the mesh would deliver is not the launcher in this repository. " +
+				"Copy packaging/nox-mesh-host-launch into module.json's `launcher` resource — the " +
+				"machines run what the manifest says, and this file is what gets reviewed")
+		}
+		return
+	}
+	t.Fatal("module.json declares no `launcher` resource, so nothing delivers the launcher and a " +
+		"delivered host version is never started")
+}