Merge pull request 'A take is a comparison: the host's facts, former targets, and strays (hq ADR 0163)' (#63) from feat/a-take-is-a-comparison-the-hosts-facts into main

Every held thing carries what a take compares: for a found container its image and the image's date,
the networks it is on and the other containers on each, its mounts and published ports, beside the
declared image (and its date once pulled), ports and volumes, with the downgrade decided when both
dates are known; for a found file whether the declared content differs and how, as lines lost and
lines new. A resource whose target moved keeps the former target on record as an orphan, so the next
apply removes the container or file the host wrote under the old name (issue 097). Every apply reports
the strays: containers the mesh neither wrote nor holds.

cmd/mesh-host/main.go

@@ -51,6 +51,43 @@ var builtFor = ""
 
 var version = "development build"
 
+// runningVersion is this host's version: the directory it was delivered into, or the link-time stamp
+// for one placed by hand.
+//
+// **From where it sits, not from its linker** (novox/hq ADR 0142): "It is unpacked into a directory
+// named for its version, so it can read its own version from its path. The stamp goes, and with it the
+// need for a build to know what it will be called."
+//
+// The mesh's toolchain does not stamp a version, on purpose — a build does not know what it will be
+// called — so a delivered host read as "development build" and the mesh could not tell which host any
+// machine ran (novox/hq 04-ISSUES/161, and 087 for why that matters). The path knows: a delivered host
+// lives at `<libexec>/versions/<version>/<binary>`.
+//
+// A host placed by hand keeps its stamp, which is the honest answer for one the mesh did not deliver.
+func runningVersion() string {
+	self, err := os.Executable()
+	if err != nil {
+		return version
+	}
+	return versionAt(self, version)
+}
+
+// versionAt is runningVersion's decision, with the executable's path and the link-time stamp given —
+// so a test can ask it about a path without being that binary.
+func versionAt(self, stamped string) string {
+	// .../versions/<version>/<binary> — the parent is the version, and its parent is the versions
+	// directory. Checked rather than assumed, so a binary somewhere else does not read a directory
+	// name as a version.
+	dir := filepath.Dir(self)
+	if filepath.Base(filepath.Dir(dir)) != upgrade.VersionsDirName {
+		return stamped
+	}
+	if name := filepath.Base(dir); name != "" && name != "." && name != string(filepath.Separator) {
+		return name
+	}
+	return stamped
+}
+
 const usage = `mesh-host — the node host
 
   profile     what this machine can be asked to do
@@ -254,7 +291,7 @@ func run(ctx context.Context, command string, opts options) error {
 		return runLink(ctx, opts)
 
 	case "version":
-		fmt.Println(version)
+		fmt.Println(runningVersion())
 		return nil
 
 	case "", "help", "-h", "--help":
@@ -420,10 +457,10 @@ func short(digest string) string {
 // them again — and everything the mesh declared read as no longer declared and removed. Even the
 // very declaration the mesh last sent, applied from a file, would plan to remove the foundation.
 // `apply FILE` is for a machine the mesh has not spoken to, and is refused saying so.
-func refuseStale(known store.State, kept store.Declared, keptErr error, digest string, from provenance) error {
+func refuseStale(known store.State, kept store.Declared, keptErr error, digest string, from provenance, sequence int64) error {
 	switch from {
 	case fromDeclared:
-		return nil
+		return refuseOlder(kept, keptErr, sequence)
 	case fromBundle:
 		if known.Genesis == nil || known.Genesis.Digest == digest {
 			return nil
@@ -520,7 +557,7 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
 	if err := apply.CheckMode(known, d); err != nil {
 		return err
 	}
-	if err := refuseStale(known, kept, keptErr, digest, from); err != nil {
+	if err := refuseStale(known, kept, keptErr, digest, from, d.Sequence); err != nil {
 		return err
 	}
 
@@ -594,8 +631,8 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
 	//
 	// A failure to record is reported and does not fail the apply. The apply worked; what is
 	// lost is a rollback's ability to come back here, which is worse to hide than to say.
-	if version != "" {
+	if v := runningVersion(); v != "" {
-		if err := upgrade.RecordKnownGood(upgrade.KnownGoodPath(opts.state), version); err != nil {
+		if err := upgrade.RecordKnownGood(upgrade.KnownGoodPath(opts.state), v); err != nil {
 			fmt.Fprintf(os.Stderr,
 				"mesh-host: applied, but could not record %s as known-good: %v\n"+
 					"  a rollback would have nothing to return to.\n", version, err)
@@ -706,15 +743,18 @@ func enrol(ctx context.Context, opts options) error {
 	fmt.Printf("  signing key         %s\n",
 		base64.StdEncoding.EncodeToString(token.Signer)[:16]+"...")
 
-	// The check that has to happen before this machine says anything.
+	// **The pin is checked by the connection that presents this token, not by a dial of our own**
-	conn, err := link.Dial(token.Broker, token.Fingerprint, opts.timeout)
+	// (novox/hq 04-ISSUES/146). This opened a raw TLS connection to the bus first, which worked
-	if err != nil {
+	// against the broker the mesh used to run and cannot work against the one it runs now: NATS
-		return err
+	// speaks its own protocol before it upgrades to TLS, so an immediate handshake is answered
-	}
+	// with a plaintext line and the enrolment failed with "first record does not look like a TLS
-	defer conn.Close()
+	// handshake" — on every node that has tried to join since the bus changed, which is why this
-	fmt.Println("\nthe broker presented the certificate this token pins")
+	// went unnoticed: none had.
-
+	//
-	conn.Close()
+	// What ADR 0004 requires still holds, and holds better: the client that presents the token
+	// carries the same pinned configuration, reads the server's greeting, upgrades, and the
+	// verification runs inside that handshake — so the one-time secret is sent only after the
+	// certificate has been checked, and nothing of this node's reaches an impostor.
 
 	mine, err := identity.Generate(*name)
 	if err != nil {
@@ -778,20 +818,23 @@ func enrol(ctx context.Context, opts options) error {
 	// control plane cannot decide what a node should run without it, so it travels with the
 	// request instead of being asked for in a second round trip.
 	detected := profile.Detect(ctx, profile.Default(nil), opts.timeout)
-	reported := map[string]any{}
+	reported := profileAsReported(detected)
-	if raw, err := json.Marshal(detected); err == nil {
-		_ = json.Unmarshal(raw, &reported)
-	}
 
 	// Signed with the identity just generated, so the mesh can tell this machine from anyone else
 	// who knows its public key (novox/hq issue 083).
 	proof := mine.Sign(link.EnrolProof(token.Secret, mine.Public, mine.Overlay.Public,
 		sealing.Public, serving.Public))
 	// The token says where to go and which certificate that address must present. It says nothing
-	// about which bus is there, and does not need to: every token names the one the mesh runs on
+	// about which bus is there, and does not need to: there is one, and this host knows which
-	// today until the rollout (novox/hq ADR 0116 step 5), and that is what an empty Transport is.
+	// (novox/hq ADR 0131 — the mesh speaks to one seat and the old transport is gone).
+	//
+	// **It used to leave this empty** and mean "whatever the mesh runs today", which was true
+	// while two buses existed and became a refusal the moment one did: an empty transport is not
+	// the bus's name, so every enrolment ended at "this token is for the \"\" bus"
+	// (novox/hq 04-ISSUES/146). Nothing caught it because nothing had enrolled since the bus
+	// changed.
 	reply, err := link.Enrol(ctx,
-		link.Approach{Address: token.Broker, Fingerprint: token.Fingerprint},
+		link.Approach{Address: token.Broker, Fingerprint: token.Fingerprint, Transport: link.OnNATS},
 		*name, token.Secret,
 		mine.Public, mine.Overlay.Public, sealing.Public, serving.Public, reported, proof, found,
 		opts.timeout)
@@ -807,6 +850,13 @@ func enrol(ctx context.Context, opts options) error {
 		Fingerprint: firstNonEmpty(reply.Fingerprint, token.Fingerprint),
 		Signer:      firstNonEmpty2(reply.Signer, token.Signer),
 		Password:    reply.Password,
+		// **Which bus this membership is for, said rather than left empty** (novox/hq
+		// 04-ISSUES/146). The link refuses a membership that names another bus, and an empty name
+		// is not this one's — so a node enrolled without it came up and reconnected for ever
+		// against its own record: "this membership is for \"\", and the mesh's bus is nats". The
+		// reply does not carry it because there is one bus and the host knows which (ADR 0131);
+		// what was missing was writing that down where the link reads it.
+		Transport: link.OnNATS,
 	}
 	if mine.Membership.Password == "" {
 		// The mesh did not replace the token's secret, so it is still this node's broker
@@ -1008,7 +1058,12 @@ func runLink(ctx context.Context, opts options) error {
 		// standing before this machine leaves the one it is on (novox/hq design 28, task 5.2).
 		adoptDeliveredMembership(identity.Path(opts.state), &mine, say)
 
-		switch next, waiting, err := upgrade.Successor(upgrade.VersionsDir(""), version); {
+		// Asked with the version this host is RUNNING, read from where it sits — not the link-time
+		// stamp, which every delivered host carries as "development build". Asked with the stamp,
+		// a delivered host never matched the newest delivered version, so it stood aside on every
+		// push for ever, and standing aside cancels the report, so the mesh never heard from it
+		// again (novox/hq 04-ISSUES/163).
+		switch next, waiting, err := upgrade.Successor(upgrade.VersionsDir(""), runningVersion()); {
 		case err != nil:
 			// Said, not fatal. A host that cannot read the delivered versions is still running this
 			// machine correctly; what it has lost is the ability to be replaced.
@@ -1360,7 +1415,8 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
 		sched.Sync(declared, held)
 	}
 
-	report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw), Host: version}
+	report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw), Host: runningVersion(),
+		Profile: profileAsReported(profile.Detect(ctx, profile.Default(nil), opts.timeout))}
 	// Which of this machine's links face outside, for the filter the mesh writes around them
 	// (novox/hq ADR 0140). Reported whatever the node's mode: a converged node's filter needs it,
 	// and an adopted one becomes converged without a further round trip. A machine that cannot read
@@ -1374,7 +1430,15 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
 	// node never reads as converged (novox/hq ADR 0100).
 	for _, h := range updated.Held {
 		report.Held = append(report.Held, link.Held{ID: h.ID, Module: h.Module, Kind: h.Kind,
-			Target: h.Target, Since: h.Since, Changed: h.Changed, Kept: h.Kept})
+			Target: h.Target, Since: h.Since, Changed: h.Changed, Kept: h.Kept, Facts: factsAsReported(h.Facts)})
+	}
+	// And what runs here that nobody asked for (novox/hq ADR 0163).
+	if strays, err := apply.Strays(ctx, apply.ExecRunner, updated); err != nil {
+		fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not list what else runs here: %v\n", err)
+	} else {
+		for _, s := range strays {
+			report.Strays = append(report.Strays, link.Stray{Kind: s.Kind, Name: s.Name, Detail: s.Detail})
+		}
 	}
 	if declared.Adoption != nil {
 		if updated.Firewall != nil {
@@ -1543,3 +1607,54 @@ func adoptDeliveredMembership(identityPath string, mine *identity.Identity, say
 	say(fmt.Sprintf("moving to the %s bus at %s — restarting to dial it", next.Transport, next.Broker))
 	os.Exit(0)
 }
+
+// refuseOlder refuses a declaration from the mesh that is older than the one this node holds.
+//
+// **By sequence, not by arrival** (novox/hq 04-ISSUES/107). What the host kept is the last thing
+// the mesh said, signed; a declaration whose sequence is lower was composed before it, whatever
+// order they arrived in — a backlog drained after the node was away, or a broker that split a burst.
+// Applying it would make the machine into something the mesh had already moved past, which is the
+// incident of issue 104 by another door.
+//
+// Only when both sides claim an order. A declaration with no sequence is one an older controller
+// sent, and one kept with no sequence is one this host received before it understood them; in either
+// case there is no order to compare, and refusing on a guess would strand the node the moment the
+// controller is older than the host. Equal is the same declaration again, which reconciling is for.
+func refuseOlder(kept store.Declared, keptErr error, sequence int64) error {
+	if sequence == 0 || keptErr != nil {
+		return nil
+	}
+	last, err := declaration.ParseTrusted(kept.Declaration)
+	if err != nil || last.Sequence == 0 {
+		return nil
+	}
+	if sequence < last.Sequence {
+		return fmt.Errorf("this declaration is older than what the mesh last said to this node: it "+
+			"is sequence %d, and the one kept here is %d. It arrived late — a backlog, or a broker "+
+			"that split a burst — and applying it would make this machine into something the mesh has "+
+			"already moved past. Refused whole; nothing was applied", sequence, last.Sequence)
+	}
+	return nil
+}
+
+// profileAsReported is the profile as the mesh reads it — the same bytes enrolment sends, so a
+// report's profile and an enrolment's are one shape on the controller's side (novox/hq ADR 0161).
+func profileAsReported(detected profile.Profile) map[string]any {
+	reported := map[string]any{}
+	if raw, err := json.Marshal(detected); err == nil {
+		_ = json.Unmarshal(raw, &reported)
+	}
+	return reported
+}
+
+// factsAsReported is a held thing's facts as the mesh reads them: the same bytes the host keeps.
+func factsAsReported(f *store.Facts) map[string]any {
+	if f == nil {
+		return nil
+	}
+	out := map[string]any{}
+	if raw, err := json.Marshal(f); err == nil {
+		_ = json.Unmarshal(raw, &out)
+	}
+	return out
+}

cmd/mesh-host/order_test.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// A declaration carries no order, so a host cannot tell an older one from a newer (novox/hq
+// 04-ISSUES/107). Its only identity was the digest of its bytes: "not the last" could be said,
+// "older" could not.
+
+func keptWith(t *testing.T, sequence int64) store.Declared {
+	t.Helper()
+	body, err := json.Marshal(map[string]any{
+		"declaration": 1, "resources": []any{}, "owns_nothing": true, "sequence": sequence,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return store.Declared{Declaration: body, Signature: []byte("x")}
+}
+
+func TestAnOlderDeclarationFromTheMeshIsRefused(t *testing.T) {
+	err := refuseOlder(keptWith(t, 7), nil, 5)
+	if err == nil {
+		t.Fatal("sequence 5 was accepted over a kept 7")
+	}
+	if !strings.Contains(err.Error(), "older") || !strings.Contains(err.Error(), "nothing was applied") {
+		t.Fatalf("the refusal does not say what it is: %v", err)
+	}
+}
+
+func TestANewerOrEqualDeclarationIsNot(t *testing.T) {
+	if err := refuseOlder(keptWith(t, 7), nil, 8); err != nil {
+		t.Fatalf("sequence 8 was refused over a kept 7: %v", err)
+	}
+	// Equal is the same declaration again, which reconciling is for.
+	if err := refuseOlder(keptWith(t, 7), nil, 7); err != nil {
+		t.Fatalf("the same sequence was refused: %v", err)
+	}
+}
+
+func TestNoOrderClaimedMeansNoOrderCompared(t *testing.T) {
+	// An older controller sends none; a host that received before it understood them kept none.
+	// Refusing on a guess would strand a node the moment the controller is older than the host.
+	if err := refuseOlder(keptWith(t, 7), nil, 0); err != nil {
+		t.Fatalf("a declaration claiming no order was refused: %v", err)
+	}
+	if err := refuseOlder(keptWith(t, 0), nil, 3); err != nil {
+		t.Fatalf("a declaration was refused against a kept one that claimed no order: %v", err)
+	}
+	if err := refuseOlder(store.Declared{}, store.ErrNothingDeclared, 3); err != nil {
+		t.Fatalf("a first declaration was refused: %v", err)
+	}
+}

cmd/mesh-host/version_from_path_test.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// A component's version comes from where it sits, not from its linker (novox/hq ADR 0142). The mesh's
+// toolchain stamps no version — a build does not know what it will be called — so a delivered host
+// read as "development build" and the mesh could not tell which host a machine ran (04-ISSUES/161).
+
+func TestADeliveredHostReadsItsVersionFromItsPath(t *testing.T) {
+	// A delivered host lives at <libexec>/versions/<version>/<binary>.
+	dir := t.TempDir()
+	versioned := filepath.Join(dir, "versions", "637f65559d16")
+	if err := os.MkdirAll(versioned, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	self := filepath.Join(versioned, "nox-mesh-host")
+	if err := os.WriteFile(self, []byte("#!/bin/sh\n"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if got := versionAt(self, "development build"); got != "637f65559d16" {
+		t.Fatalf("a delivered host read its version as %q", got)
+	}
+}
+
+func TestAHostPlacedByHandKeepsItsStamp(t *testing.T) {
+	// The honest answer for one the mesh did not deliver — and every machine is in that state until
+	// a delivery reaches it.
+	if got := versionAt("/usr/bin/nox-mesh-host", "04a27ca"); got != "04a27ca" {
+		t.Fatalf("a hand-placed host read its version as %q", got)
+	}
+}
+
+func TestADirectoryThatIsNotAVersionIsNotReadAsOne(t *testing.T) {
+	// A binary sitting anywhere else must not have its parent directory's name read as a version.
+	for _, path := range []string{
+		"/opt/somewhere/nox-mesh-host",
+		"/usr/lib/nox-mesh-host/launch",
+		"/home/someone/build/nox-mesh-host",
+	} {
+		if got := versionAt(path, "the stamp"); got != "the stamp" {
+			t.Fatalf("%s read its version as %q", path, got)
+		}
+	}
+}

examples/foundation-first-node-nats.lock

@@ -127,12 +127,21 @@
     {
       "id": "bus-certificate",
       "type": "action",
-      "command": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls",
+      // **The mesh makes its own** (novox/hq 04-ISSUES/146). This ran `openssl` inside the
-                  "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927",
+      // broker's image while the broker was one that carried it; the bus that replaced it has a
-                  "-c", "test -f /tls/tls.crt || (openssl req -x509 -newkey rsa:2048 -nodes -keyout /tls/tls.key -out /tls/tls.crt -days 3650 -subj '/CN=mesh-broker' -addext 'subjectAltName=DNS:mesh-broker,IP:127.0.0.1' >/dev/null 2>&1 && chmod 644 /tls/tls.crt && chmod 600 /tls/tls.key)"],
+      // shell and no openssl, and no other image the bundle names has one either. So the program
-      "verify": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls",
+      // that needs the certificate writes it — already on this machine, since the schema step ran
-                 "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927",
+      // it, and asking nothing of the image it writes into. Self-signed on purpose: a host pins
-                 "-c", "test -s /tls/tls.crt && openssl x509 -in /tls/tls.crt -noout"]
+      // this server's exact certificate (novox/hq ADR 0004), and at this moment there is no mesh
+      // to ask an authority of.
+      // `--user 0:0` because the volume is root's and this image runs as nobody, which is right
+      // for the long-running control plane and wrong for a one-shot writing into a fresh volume.
+      "command": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
+                  "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
+                  "broker", "certificate", "--into", "/tls"],
+      "verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
+                 "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
+                 "broker", "certificate", "--check", "--into", "/tls"]
     },
     {
       "id": "bus-conf-dir",
@@ -152,7 +161,7 @@
       "type": "file",
       "path": "/var/lib/mesh-bus-conf/accounts.conf",
       "mode": "0600",
-      "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n  MESH {\n    jetstream: enabled\n    users = [\n      { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n        publish: { allow: [\"$JS.API.>\", \"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"_INBOX.enrol.>\", \"mesh.control.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\"] }\n        subscribe: { allow: [\"$JS.API.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.seat.mesh-build-machine.event.built\"] }\n        allow_responses: { max: 1, ttl: \"1m\" }\n      } }\n    ]\n  }\n}\n"
+      "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n  MESH {\n    jetstream: enabled\n    users = [\n      { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n        publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.API.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.control.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\"] }\n        subscribe: { allow: [\"$JS.API.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.mesh-controller.tool.>\"] }\n        allow_responses: { max: 1, ttl: \"1m\" }\n      } }\n    ]\n  }\n}\n"
     },
     {
       "id": "broker",

internal/apply/apply.go

@@ -1501,13 +1501,15 @@ func containerSpecReading(r *declaration.Container, declares, reads map[string]s
 	for _, a := range r.Args {
 		b.WriteString("arg " + a + "\n")
 	}
-	// **The mesh's names are part of what a container is** (novox/hq 04-ISSUES/135). A container
+	// **A container's declared names are part of what it is** (novox/hq 04-ISSUES/135). What is
-	// resolves every other machine and every public name through the entries the mesh gives it at
+	// here is what the module declared for itself and nothing else: the mesh's own names are no
-	// creation, and nothing re-reads them afterwards — so a container left alone when the roster
+	// longer written into a container (ADR 0148) — they were once, every container got the whole
-	// moved is one that cannot reach anything by name, for ever, while every check reports it
+	// roster at creation and nothing re-read it, so one left alone when the roster moved could not
-	// running. That is exactly what happened when this mesh's overlay range changed: one container
+	// reach anything by name for as long as it ran while every check reported it running; and once
-	// whose image and files never changed kept an address five days out of date and restarted
+	// the roster was in this digest so that could be caught, one name moving anywhere replaced
-	// 2286 times against a database it could no longer find.
+	// every container in the mesh (04-ISSUES/151). A container resolves a mesh name through the
+	// machine's resolver at the moment it asks. What a module declares does not move when the
+	// roster does, so hashing it costs nothing and catches a manifest that changed.
 	//
 	// Sorted, so the digest does not move for a reordering nobody made.
 	hosts := append([]string(nil), r.Hosts...)

internal/apply/facts_test.go

@@ -0,0 +1,97 @@
+package apply
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// A take is a comparison (novox/hq ADR 0163): while a module's container is held, the host reports
+// the found image and its age beside the declared one, the networks and who else is on them, the
+// mounts and the ports — and says when the declared image is the older.
+func TestAHeldContainerCarriesTheFactsATakeCompares(t *testing.T) {
+	dir, page, m := predecessor(t)
+	m.containers["hello-web"].image = "web:1.27"
+	m.containers["hello-web"].imageID = "sha256:found"
+	m.containers["hello-web"].networks = []string{"predecessor_default"}
+	m.containers["hello-web"].mounts = []string{"/srv/web:/data"}
+	m.containers["hello-web"].ports = []string{"80/tcp>0.0.0.0:8080"}
+	m.images = map[string]string{"sha256:found": "2026-09-17T10:00:00Z", pinned: "2026-08-20T10:00:00Z"}
+	m.members = map[string][]string{"predecessor_default": {"hello-web", "office", "db"}}
+
+	_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
+	h, ok := state.HeldAt("hello-web.server")
+	if !ok || h.Facts == nil {
+		t.Fatalf("a held container carries no facts: %+v", h)
+	}
+	f := h.Facts
+	if f.Image != "web:1.27" || f.ImageCreated != "2026-09-17T10:00:00Z" {
+		t.Errorf("the found image and its age: %+v", f)
+	}
+	if f.DeclaredImage != pinned || f.DeclaredImageCreated != "2026-08-20T10:00:00Z" || !f.Downgrade {
+		t.Errorf("the declared image, its age, and that it is a downgrade: %+v", f)
+	}
+	if got := f.Networks["predecessor_default"]; len(got) != 2 || got[0] != "db" || got[1] != "office" {
+		t.Errorf("the neighbours on the found network, without the container itself: %v", f.Networks)
+	}
+	if len(f.Mounts) != 1 || f.Mounts[0] != "/srv/web:/data" || len(f.Ports) != 1 || f.Ports[0] != "80/tcp>0.0.0.0:8080" {
+		t.Errorf("mounts and ports as found: %+v", f)
+	}
+	// And the held file carries how the declared content differs from what was found.
+	p, ok := state.HeldAt("hello-web.page")
+	if !ok || p.Facts == nil || !p.Facts.Differs {
+		t.Fatalf("a held file that differs from the declared content does not say so: %+v", p)
+	}
+	joined := strings.Join(p.Facts.Difference, "\n")
+	if !strings.Contains(joined, "- the predecessor's page") || !strings.Contains(joined, "+ the mesh's page") {
+		t.Errorf("the difference does not show what is lost and what is new: %q", joined)
+	}
+	_ = os.Remove(filepath.Join(dir, "unused"))
+}
+
+// A declared image not yet on the machine leaves its age unknown and the comparison undecided.
+func TestAnImageNotYetPulledLeavesTheDowngradeUndecided(t *testing.T) {
+	dir, page, m := predecessor(t)
+	m.containers["hello-web"].image = "web:1.27"
+	m.containers["hello-web"].imageID = "sha256:found"
+	m.images = map[string]string{"sha256:found": "2026-09-17T10:00:00Z"}
+	_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
+	h, _ := state.HeldAt("hello-web.server")
+	if h.Facts == nil || h.Facts.DeclaredImageCreated != "" || h.Facts.Downgrade {
+		t.Fatalf("an unknown declared age decided a downgrade: %+v", h.Facts)
+	}
+}
+
+func TestTheDifferenceIsWhatIsLostAndWhatIsNew(t *testing.T) {
+	differs, lines := differenceOf("a\nprivate scope: local\nb\n", "a\nb\nupstream: public\n")
+	if !differs || len(lines) != 2 || lines[0] != "- private scope: local" || lines[1] != "+ upstream: public" {
+		t.Fatalf("got %v %v", differs, lines)
+	}
+	if differs, lines := differenceOf("same\n", "same\n"); differs || lines != nil {
+		t.Fatalf("identical content differs: %v %v", differs, lines)
+	}
+}
+
+// What runs on the machine that the mesh neither wrote nor holds is reported (ADR 0163).
+func TestStraysAreWhatRunsHereThatNobodyAsked(t *testing.T) {
+	m := &machine{containers: map[string]*fakeContainer{
+		"hello-web":  {id: "ours", running: true, image: "web:1"},
+		"gitea-old":  {id: "left-behind", running: true, image: "gitea:1.22"},
+		"held-thing": {id: "found", running: true, image: "x:1"},
+	}}
+	known := store.State{
+		Resources: []store.Applied{{ID: "hello-web.server", Type: "container", Target: "hello-web"}},
+		Held:      []store.Held{{ID: "other.server", Kind: "container", Target: "held-thing"}},
+	}
+	strays, err := Strays(context.Background(), m.run, known)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(strays) != 1 || strays[0].Name != "gitea-old" || !strings.Contains(strays[0].Detail, "gitea:1.22") {
+		t.Fatalf("strays: %+v", strays)
+	}
+}

internal/apply/hold.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"syscall"
 	"time"
@@ -433,6 +434,32 @@ type foundContainer struct {
 	id      string
 	running bool
 	spec    string
+	// What a take compares (novox/hq ADR 0163): the image and its id, the networks the container
+	// is on, its mounts and its published ports — empty from a runtime (or a test's fake) that
+	// answers the short form.
+	image    string
+	imageID  string
+	networks []string
+	mounts   []string
+	ports    []string
+}
+
+// foundFormat is what inspectFound asks the runtime for, tab-separated: the three a hold has
+// always needed, then the facts a take compares.
+const foundFormat = "{{.Id}}\t{{.State.Running}}\t{{index .Config.Labels \"" + specLabel + "\"}}" +
+	"\t{{.Config.Image}}\t{{.Image}}" +
+	"\t{{range $k, $v := .NetworkSettings.Networks}}{{$k}},{{end}}" +
+	"\t{{range .Mounts}}{{.Source}}:{{.Destination}},{{end}}" +
+	"\t{{range $p, $b := .NetworkSettings.Ports}}{{$p}}{{range $b}}>{{.HostIp}}:{{.HostPort}}{{end}},{{end}}"
+
+func splitList(s string) []string {
+	var out []string
+	for _, part := range strings.Split(s, ",") {
+		if part = strings.TrimSpace(part); part != "" {
+			out = append(out, part)
+		}
+	}
+	return out
 }
 
 // inspectFound reads a container by name the way a hold needs it: its id, whether it runs, and
@@ -451,8 +478,7 @@ func inspectFound(ctx context.Context, name string, run Runner) (foundContainer,
 	if err != nil {
 		return foundContainer{}, false, fmt.Errorf("%w, so nothing can be said about %q", err, name)
 	}
-	out, err := run(ctx, cri, "container", "inspect", "--format",
+	out, err := run(ctx, cri, "container", "inspect", "--format", foundFormat, name)
-		"{{.Id}}\t{{.State.Running}}\t{{index .Config.Labels \""+specLabel+"\"}}", name)
 	if err != nil {
 		if absent(err) {
 			return foundContainer{}, false, nil
@@ -466,14 +492,100 @@ func inspectFound(ctx context.Context, name string, run Runner) (foundContainer,
 			name, err)
 	}
 	parts := strings.Split(strings.TrimSpace(out), "\t")
-	for len(parts) < 3 {
+	for len(parts) < 8 {
 		parts = append(parts, "")
 	}
 	spec := strings.TrimSpace(parts[2])
 	if spec == "<no value>" {
 		spec = ""
 	}
-	return foundContainer{id: strings.TrimSpace(parts[0]), running: parts[1] == "true", spec: spec}, true, nil
+	return foundContainer{id: strings.TrimSpace(parts[0]), running: parts[1] == "true", spec: spec,
+		image: strings.TrimSpace(parts[3]), imageID: strings.TrimSpace(parts[4]),
+		networks: splitList(parts[5]), mounts: splitList(parts[6]), ports: splitList(parts[7])}, true, nil
+}
+
+// factsOf is what a take would compare for a found container (novox/hq ADR 0163): the found
+// image and when it was made, the networks and who else is on them, mounts and ports — beside
+// what the module declares, and the declared image's date when that image is on the machine.
+// Every question the runtime cannot answer leaves its fact empty; a preview says so rather than
+// guesses.
+func factsOf(ctx context.Context, seen foundContainer, res *declaration.Container, run Runner) *Facts {
+	cri, err := containerRuntime(ctx, run)
+	if err != nil {
+		return nil
+	}
+	f := &store.Facts{Image: seen.image, Mounts: seen.mounts, Ports: seen.ports,
+		DeclaredImage: res.Image, DeclaredPorts: res.Ports, DeclaredVolumes: res.Volumes}
+	if seen.imageID != "" {
+		if out, err := run(ctx, cri, "image", "inspect", "--format", "{{.Created}}", seen.imageID); err == nil {
+			f.ImageCreated = strings.TrimSpace(out)
+		}
+	}
+	if res.Image != "" {
+		if out, err := run(ctx, cri, "image", "inspect", "--format", "{{.Created}}", res.Image); err == nil {
+			f.DeclaredImageCreated = strings.TrimSpace(out)
+		}
+	}
+	if found, err := time.Parse(time.RFC3339Nano, f.ImageCreated); err == nil {
+		if declared, err := time.Parse(time.RFC3339Nano, f.DeclaredImageCreated); err == nil {
+			f.Downgrade = declared.Before(found)
+		}
+	}
+	for _, network := range seen.networks {
+		if f.Networks == nil {
+			f.Networks = map[string][]string{}
+		}
+		var members []string
+		if out, err := run(ctx, cri, "network", "inspect", "--format",
+			"{{range .Containers}}{{.Name}},{{end}}", network); err == nil {
+			for _, m := range splitList(out) {
+				if m != res.Name {
+					members = append(members, m)
+				}
+			}
+		}
+		sort.Strings(members)
+		f.Networks[network] = members
+	}
+	return (*Facts)(f)
+}
+
+// Facts is store.Facts, named here so hold's callers read as one vocabulary.
+type Facts = store.Facts
+
+// differenceOf is how a found file differs from the declared content: the lines only the found
+// file has, marked -, then the lines only the declared content has, marked +, in their own order,
+// bounded so a report stays a report. Not a diff tool's output: the question a take answers is
+// "what would be lost and what would be new", and that is these two lists.
+func differenceOf(found, declared string) (bool, []string) {
+	if found == declared {
+		return false, nil
+	}
+	const bound = 40
+	count := func(s string) map[string]int {
+		out := map[string]int{}
+		for _, line := range strings.Split(s, "\n") {
+			out[line]++
+		}
+		return out
+	}
+	inFound, inDeclared := count(found), count(declared)
+	var out []string
+	add := func(mark, s string, other map[string]int) {
+		seen := map[string]int{}
+		for _, line := range strings.Split(s, "\n") {
+			seen[line]++
+			if seen[line] > other[line] && len(out) < bound {
+				out = append(out, mark+" "+line)
+			}
+		}
+	}
+	add("-", found, inDeclared)
+	add("+", declared, inFound)
+	if len(out) >= bound {
+		out = append(out, "… and more")
+	}
+	return true, out
 }
 
 // absent is whether a runtime said the thing is not there, rather than failing to answer. Its own
@@ -540,6 +652,12 @@ func hold(ctx context.Context, sys system.System, r declaration.Resource, module
 			} else if digestOf(string(content)) != h.Digest {
 				changed = "rewritten"
 			}
+			// What a take would replace it with, and how that differs (novox/hq ADR 0163): a
+			// file declared whole is compared whole; one written into is not replaced at all.
+			if res.Into == "" {
+				differs, lines := differenceOf(string(content), res.Content)
+				h.Facts = &Facts{Differs: differs, Difference: lines}
+			}
 		}
 	case *declaration.Directory:
 		info, err := os.Lstat(res.Path)
@@ -628,6 +746,9 @@ func hold(ctx context.Context, sys system.System, r declaration.Resource, module
 		case h.Running && !seen.running:
 			changed = "stopped"
 		}
+		if exists {
+			h.Facts = factsOf(ctx, seen, res, run)
+		}
 	default:
 		return out, h, fmt.Errorf("a %s cannot be held", r.Kind())
 	}

internal/apply/hold_test.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"testing"
 
@@ -18,6 +19,10 @@ import (
 // label when a host made it. Every command it is asked is written down.
 type machine struct {
 	containers map[string]*fakeContainer
+	// images is what `image inspect --format {{.Created}}` answers per image or id; members is
+	// what `network inspect` lists per network (ADR 0163).
+	images  map[string]string
+	members map[string][]string
 	asked   []string
 	// wgUp is what `wg show interfaces` answers: the tunnels up on the machine.
 	wgUp string
@@ -97,6 +102,9 @@ type fakeContainer struct {
 	id      string
 	running bool
 	spec    string
+	// What a take compares (ADR 0163), answered in the long inspect form when set.
+	image, imageID          string
+	networks, mounts, ports []string
 }
 
 func (m *machine) run(_ context.Context, name string, args ...string) (string, error) {
@@ -140,9 +148,38 @@ func (m *machine) run(_ context.Context, name string, args ...string) (string, e
 			running = "true"
 		}
 		if strings.HasPrefix(args[3], "{{.Id}}") {
-			return c.id + "\t" + running + "\t" + c.spec + "\n", nil
+			line := c.id + "\t" + running + "\t" + c.spec
+			if c.image != "" {
+				line += "\t" + c.image + "\t" + c.imageID + "\t" + strings.Join(c.networks, ",") + "," +
+					"\t" + strings.Join(c.mounts, ",") + "," + "\t" + strings.Join(c.ports, ",") + ","
+			}
+			return line + "\n", nil
 		}
 		return running + "\t" + c.spec + "\n", nil
+	case "image":
+		if len(args) > 1 && args[1] == "inspect" {
+			if created, ok := m.images[args[len(args)-1]]; ok {
+				return created + "\n", nil
+			}
+			return "", errors.New("no such image")
+		}
+		return "", nil
+	case "network":
+		if len(args) > 1 && args[1] == "inspect" {
+			return strings.Join(m.members[args[len(args)-1]], ",") + ",\n", nil
+		}
+		return "", nil
+	case "ps":
+		var lines []string
+		for name, c := range m.containers {
+			state := "exited"
+			if c.running {
+				state = "running"
+			}
+			lines = append(lines, name+"\t"+c.image+"\t"+state)
+		}
+		sort.Strings(lines)
+		return strings.Join(lines, "\n") + "\n", nil
 	case "rm":
 		delete(m.containers, args[len(args)-1])
 		return "", nil

internal/apply/strays.go

@@ -0,0 +1,54 @@
+package apply
+
+import (
+	"context"
+	"sort"
+	"strings"
+
+	"github.com/novox/mesh-host/internal/declaration"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// Strays is what runs on the machine that the mesh neither wrote nor holds (novox/hq ADR 0163):
+// every container the runtime has that no record names and no hold names. The question nothing
+// answered on 2026-09-23, when a renamed resource left its old container running for a day; asked
+// on every apply now, and reported, so a thing left behind is seen the day it is left.
+//
+// Containers only, today. A listener nobody declared is harder to attribute to a thing, and the
+// machine's own services are not strays; that account is issue 160's.
+func Strays(ctx context.Context, run Runner, known store.State) ([]store.Stray, error) {
+	cri, err := containerRuntime(ctx, run)
+	if err != nil {
+		return nil, nil // a machine with no runtime has no containers to stray
+	}
+	out, err := run(ctx, cri, "ps", "-a", "--format", "{{.Names}}\t{{.Image}}\t{{.State}}")
+	if err != nil {
+		return nil, err
+	}
+	ours := map[string]bool{}
+	for _, r := range known.Resources {
+		if declaration.Type(r.Type) == declaration.TypeContainer {
+			ours[r.Target] = true
+		}
+	}
+	for _, h := range known.Held {
+		if h.Kind == string(declaration.TypeContainer) {
+			ours[h.Target] = true
+		}
+	}
+	var strays []store.Stray
+	for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
+		parts := strings.Split(line, "\t")
+		name := strings.TrimSpace(parts[0])
+		if name == "" || ours[name] {
+			continue
+		}
+		detail := ""
+		if len(parts) > 2 {
+			detail = strings.TrimSpace(parts[1]) + ", " + strings.TrimSpace(parts[2])
+		}
+		strays = append(strays, store.Stray{Kind: string(declaration.TypeContainer), Name: name, Detail: detail})
+	}
+	sort.Slice(strays, func(i, j int) bool { return strays[i].Name < strays[j].Name })
+	return strays, nil
+}

internal/declaration/declaration.go

@@ -1137,6 +1137,19 @@ type Declaration struct {
 	// converged node — which is every node the mesh raised before adoption existed, and so the
 	// only form an older controller ever sends (novox/hq ADR 0100).
 	Adoption *Adoption
+
+	// Sequence orders this declaration against every other the mesh has sent this node: each
+	// send is one higher than the last, assigned under the control plane's hold on the node
+	// (novox/hq 04-ISSUES/107). Zero is a declaration that carries no order — every one an older
+	// controller sent, and the bundle genesis applies — and a host makes no ordering claim about
+	// one of those.
+	//
+	// **The one property a declaration needs that its signature does not give it.** A signature
+	// says the mesh sent this; it cannot say the mesh sent it AFTER the one the host is holding.
+	// Before this, "older" was inferred from arrival within a batch and a 750ms window, and a
+	// backlog longer than the batch, or a slow broker, applied a declaration the mesh had already
+	// superseded.
+	Sequence int64
 }
 
 // Adoption is a node's mode, as the controller records it: the node is adopted, and these are
@@ -1274,6 +1287,9 @@ type envelope struct {
 	// a bug quietly strip a machine.
 	OwnsNothing bool              `json:"owns_nothing,omitempty"`
 	Resources   []json.RawMessage `json:"resources"`
+	// Sequence is optional on the wire, so a controller that does not send one is still
+	// understood: absent reads as zero, which is "no ordering claimed" rather than "first".
+	Sequence int64 `json:"sequence,omitempty"`
 }
 
 func parse(raw []byte, allowActions bool) (*Declaration, error) {
@@ -1290,7 +1306,7 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
 			env.Version, Version)}}
 	}
 
-	d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption}
+	d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption, Sequence: env.Sequence}
 	var problems []string
 
 	if len(env.Resources) == 0 && !env.OwnsNothing {

internal/link/asking_nats.go

@@ -3,6 +3,7 @@ package link
 import (
 	"context"
 	"crypto/rand"
+	"crypto/sha256"
 	"encoding/hex"
 	"errors"
 	"fmt"
@@ -63,8 +64,15 @@ func presentNats(_ context.Context, to Approach, node, secret string,
 	// subscribe its own inbox and nothing else (design 25 §6). The secret is its password, the same
 	// string the request claims, so the server proves somebody holds the token and the request
 	// proves the same thing to the controller without it having to ask the server who connected.
+	// **Its own inbox space, because that is the only one it may listen in** (novox/hq
+	// 04-ISSUES/146). A JetStream publish waits for the stream's acknowledgement on an inbox the
+	// client picks, and the client's default is `_INBOX.<random>` — which this user may not
+	// subscribe to, so the enrolment failed with a permissions violation on a subject nobody had
+	// chosen. The permission is `_INBOX.enrol.<node>.>` (design 25 §6), so the client is told to
+	// pick its inboxes there; the reply address below is in the same space for the same reason.
 	conn, err := nats.Connect(natsURL(to.Address),
 		nats.Secure(config),
+		nats.CustomInboxPrefix("_INBOX.enrol."+node),
 		nats.UserInfo("enrol."+node, secret),
 		nats.Name("mesh-host/enrol/"+node),
 		nats.Timeout(timeout),
@@ -124,7 +132,19 @@ func (a *natsAsking) Ask(ctx context.Context, request []byte, wait time.Duration
 	defer cancel()
 	// Into the stream and awaited: an enrolment the bus never accepted must fail here rather than be
 	// assumed, because the node has nothing else to go on.
-	if _, err := a.js.Publish(EnrolSubject, addressed, nats.Context(publish)); err != nil {
+	//
+	// **Once, however many times it is sent** (novox/hq 04-ISSUES/146). The client re-publishes when
+	// an acknowledgement is slow, and the mesh enrolled the machine on each copy — minting a second
+	// credential, which replaced the first, which is the one the node had already been given. The
+	// machine then reconnected for ever as a user whose password the mesh had rotated out from under
+	// it, and the controller's log said "enrolled anchor" twice in the same second.
+	//
+	// The id is the message: the same bytes carry the same id, so the stream discards the client's
+	// own retry, and a genuine second attempt — which carries a new reply address — is a different
+	// message and is let through.
+	sum := sha256.Sum256(addressed)
+	if _, err := a.js.Publish(EnrolSubject, addressed,
+		nats.MsgId(hex.EncodeToString(sum[:])), nats.Context(publish)); err != nil {
 		return nil, fmt.Errorf("cannot ask the mesh to enrol this node: %w", err)
 	}
 

internal/link/held_note_test.go

@@ -0,0 +1,46 @@
+package link
+
+import (
+	"strings"
+	"testing"
+)
+
+// The count that did not add up was the only symptom sixteen held resources had, and reading it meant
+// opening the node's state file by hand (novox/hq 04-ISSUES/125). The line that says what an apply did
+// says what it did not, too.
+
+func TestTheApplyLineSaysWhatItHeldAndForWhichModule(t *testing.T) {
+	got := heldNote([]Held{
+		{ID: "ca", Module: "route-proxy", Kind: "directory"},
+		{ID: "certs", Module: "route-proxy", Kind: "directory"},
+		{ID: "server", Module: "route-proxy", Kind: "container"},
+		{ID: "mail", Module: "mailu", Kind: "container"},
+	})
+	if !strings.Contains(got, "4 held") {
+		t.Fatalf("the count of what was held is not in the line: %q", got)
+	}
+	// The module is the thing an operator can act on: `take` takes a module.
+	if !strings.Contains(got, "route-proxy: 3") || !strings.Contains(got, "mailu: 1") {
+		t.Fatalf("the line does not break the holds down by module: %q", got)
+	}
+	// Ordered, so two machines holding the same things read the same and a diff of two reports is
+	// about what changed.
+	if strings.Index(got, "mailu") > strings.Index(got, "route-proxy") {
+		t.Fatalf("modules are not in a stated order: %q", got)
+	}
+	// It says why, because "held" alone reads as a failure and this is correct behaviour.
+	if !strings.Contains(got, "taken") {
+		t.Fatalf("the line does not say a hold ends when the module is taken: %q", got)
+	}
+}
+
+func TestAnApplyThatHeldNothingSaysNothingExtra(t *testing.T) {
+	// A converged machine holds nothing, which is most applies. Reporting "0 held" on every one of
+	// them is how a line stops being read.
+	if got := heldNote(nil); got != "" {
+		t.Fatalf("an apply with no holds added %q to its line", got)
+	}
+	if got := heldNote([]Held{}); got != "" {
+		t.Fatalf("an apply with no holds added %q to its line", got)
+	}
+}

internal/link/messages.go

@@ -110,6 +110,15 @@ type Report struct {
 	// and the mesh's up in its place, and where the found configuration's original was kept.
 	Tunnel *CarriedTunnel `json:"tunnel,omitempty"`
 
+	// Strays is what runs on the machine that the mesh neither wrote nor holds (novox/hq ADR
+	// 0163): containers nobody declared and nobody holds, the ones a cutover leaves behind.
+	Strays []Stray `json:"strays,omitempty"`
+
+	// Profile is what this machine can do, detected again by the apply that reports (novox/hq
+	// ADR 0161) — the same shape enrolment sends — so a capability gained or lost since enrolment,
+	// a network manager switched, reaches the mesh at the next push rather than never.
+	Profile map[string]any `json:"profile,omitempty"`
+
 	// Host is the version of the host that produced this report (novox/hq ADR 0141).
 	//
 	// Without it nothing can say a machine is behind, so "every machine current with its source"
@@ -200,6 +209,16 @@ type Held struct {
 	Changed string `json:"changed,omitempty"`
 	// Kept is where a file's original was kept.
 	Kept string `json:"kept,omitempty"`
+	// Facts is the found thing beside what the module declares — what a take compares (novox/hq
+	// ADR 0163). The same shape the host keeps; the controller reads it as data.
+	Facts map[string]any `json:"facts,omitempty"`
+}
+
+// A Stray is a container the mesh neither wrote nor holds (ADR 0163).
+type Stray struct {
+	Kind   string `json:"kind"`
+	Name   string `json:"name"`
+	Detail string `json:"detail,omitempty"`
 }
 
 // Reach is one thing reachable on the machine: a listening socket, or a published container port.

internal/link/order_test.go

@@ -0,0 +1,50 @@
+package link
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+)
+
+// The drain picked the last to arrive. A backlog longer than the batch, or a broker that split a
+// burst, delivered a superseded declaration last (novox/hq 04-ISSUES/107).
+
+func sequenced(t *testing.T, n int64) *said {
+	t.Helper()
+	inner, err := json.Marshal(map[string]any{"declaration": 1, "resources": []any{}, "sequence": n})
+	if err != nil {
+		t.Fatal(err)
+	}
+	body, err := json.Marshal(Signed{Declaration: inner, Signature: []byte("s")})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return &said{body: body}
+}
+
+func TestTheDrainKeepsTheHighestSequenceNotTheLastToArrive(t *testing.T) {
+	waiting := make(chan Declaration, 8)
+	waiting <- sequenced(t, 9)
+	waiting <- sequenced(t, 4) // arrived last, composed earlier
+	latest, aside := newest(waiting, sequenced(t, 8), 30*time.Millisecond)
+	if got := sequenceOf(latest.Body()); got != 9 {
+		t.Fatalf("the drain kept sequence %d, and 9 was waiting", got)
+	}
+	if len(aside) != 2 {
+		t.Fatalf("%d set aside, wanted 2 (the 8 and the late 4)", len(aside))
+	}
+}
+
+func TestWithoutSequencesTheLastToArriveStillWins(t *testing.T) {
+	// The behaviour this had before, kept for a controller that sends no order.
+	apply, aside := newest(arriving("two", "three"), &said{body: []byte("one")}, 30*time.Millisecond)
+	if string(apply.Body()) != "three" || len(aside) != 2 {
+		t.Fatalf("applied %q with %d set aside", apply.Body(), len(aside))
+	}
+}
+
+func TestAnUnreadableBodyClaimsNoOrder(t *testing.T) {
+	if got := sequenceOf([]byte("not json")); got != 0 {
+		t.Fatalf("garbage claimed sequence %d", got)
+	}
+}

internal/link/pinned.go

@@ -73,8 +73,19 @@ func PinnedConfig(pin string) (*tls.Config, error) {
 	}, nil
 }
 
-// Dial opens a TLS connection to the broker, refusing anything but the pinned certificate.
+// dialPinned completes a TLS handshake against an address, refusing anything but the pinned
-func Dial(address, pin string, timeout time.Duration) (*tls.Conn, error) {
+// certificate.
+//
+// **Not how the bus is reached, and it used to be** (novox/hq 04-ISSUES/146). Enrolment opened one
+// of these before it said anything, which was right while the broker answered TLS immediately and
+// wrong the moment the mesh moved to a bus that speaks its own protocol first. The pin itself was
+// never the problem — PinnedConfig is what the NATS client is given, and the verification runs
+// inside the handshake that client performs.
+//
+// It stays here because this is where the pin is proven: the tests beside it run a real TLS server
+// and assert that a wrong certificate is refused before a byte of application data is sent. What it
+// must not become again is something a caller uses to reach the bus.
+func dialPinned(address, pin string, timeout time.Duration) (*tls.Conn, error) {
 	config, err := PinnedConfig(pin)
 	if err != nil {
 		return nil, err
@@ -86,7 +97,7 @@ func Dial(address, pin string, timeout time.Duration) (*tls.Conn, error) {
 		if errors.Is(err, ErrWrongCertificate) {
 			return nil, err
 		}
-		return nil, fmt.Errorf("cannot reach the broker at %s: %w", address, err)
+		return nil, fmt.Errorf("cannot reach %s: %w", address, err)
 	}
 	return conn, nil
 }

internal/link/pinned_test.go

@@ -63,7 +63,7 @@ func server(t *testing.T) (address string, fingerprint string) {
 
 func TestTheRightBrokerIsAccepted(t *testing.T) {
 	address, pin := server(t)
-	conn, err := Dial(address, pin, 5*time.Second)
+	conn, err := dialPinned(address, pin, 5*time.Second)
 	if err != nil {
 		t.Fatalf("the broker its token describes was refused: %v", err)
 	}
@@ -76,7 +76,7 @@ func TestADifferentBrokerIsRefused(t *testing.T) {
 	address, _ := server(t)
 	_, other := server(t)
 
-	_, err := Dial(address, other, 5*time.Second)
+	_, err := dialPinned(address, other, 5*time.Second)
 	if err == nil {
 		t.Fatal("a broker presenting a different certificate was accepted")
 	}
@@ -130,7 +130,7 @@ func TestNothingIsSentToTheWrongBroker(t *testing.T) {
 
 	// A pin for a certificate this server does not have.
 	_, elsewhere := server(t)
-	if _, err := Dial(listener.Addr().String(), elsewhere, 5*time.Second); err == nil {
+	if _, err := dialPinned(listener.Addr().String(), elsewhere, 5*time.Second); err == nil {
 		t.Fatal("the impostor was accepted")
 	}
 	if n := <-received; n > 0 {
@@ -160,7 +160,7 @@ func TestAnUnreachableBrokerIsAnOrdinaryFailure(t *testing.T) {
 	address := listener.Addr().String()
 	listener.Close()
 
-	_, err = Dial(address, pin, 2*time.Second)
+	_, err = dialPinned(address, pin, 2*time.Second)
 	if err == nil {
 		t.Fatal("dialling a closed port succeeded")
 	}

internal/link/run.go

@@ -6,6 +6,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"sort"
+	"strings"
 	"time"
 )
 
@@ -250,9 +252,11 @@ func Run(ctx context.Context, m Membership, apply Applier, say Announce, timeout
 			case report.Refused != "":
 				say("refused a declaration: " + report.Refused)
 			case len(report.Failed) > 0:
-				say(fmt.Sprintf("applied %d and failed: %v", len(report.Applied), report.Failed))
+				say(fmt.Sprintf("applied %d and failed: %v%s",
+					len(report.Applied), report.Failed, heldNote(report.Held)))
 			default:
-				say(fmt.Sprintf("applied %d resource(s)", len(report.Applied)))
+				say(fmt.Sprintf("applied %d resource(s)%s",
+					len(report.Applied), heldNote(report.Held)))
 			}
 			publishReport(ctx, link, m, report, say, timeout)
 			// Settled after the report is published. A node that dies between applying and
@@ -296,6 +300,16 @@ func newest(arriving <-chan Declaration, first Declaration, window time.Duration
 			if !ok {
 				return latest, superseded
 			}
+			// **By sequence when both carry one, by arrival when either does not** (novox/hq
+			// 04-ISSUES/107). Arrival is what this window had to go on, and it is wrong exactly
+			// when it matters — a backlog drained out of order. A declaration that says where it
+			// stands is believed over when it turned up; one that does not is the older
+			// controller's, and arrival is all there is.
+			if sequenceOf(next.Body()) < sequenceOf(latest.Body()) &&
+				sequenceOf(next.Body()) > 0 && sequenceOf(latest.Body()) > 0 {
+				superseded = append(superseded, next)
+				continue
+			}
 			superseded = append(superseded, latest)
 			latest = next
 		case <-time.After(window):
@@ -304,6 +318,24 @@ func newest(arriving <-chan Declaration, first Declaration, window time.Duration
 	}
 }
 
+// sequenceOf is the order a signed declaration claims, or zero when it claims none or cannot be
+// read. Read from the envelope alone; the signature is verified later, when the winner is applied,
+// and a forged message that lied about its sequence would only set aside real ones — which are
+// reported as set aside, and the next push sends the current one again.
+func sequenceOf(body []byte) int64 {
+	var signed Signed
+	if err := json.Unmarshal(body, &signed); err != nil {
+		return 0
+	}
+	var d struct {
+		Sequence int64 `json:"sequence"`
+	}
+	if err := json.Unmarshal(signed.Declaration, &d); err != nil {
+		return 0
+	}
+	return d.Sequence
+}
+
 // declaredIn is the id a signed declaration carries, for a report about one that was not applied.
 // Empty if the message is not one — a forged or garbled message is refused by handleBody when its
 // turn comes; here it is only named.
@@ -392,3 +424,36 @@ func publishAlive(ctx context.Context, bus Bus, m Membership, say Announce,
 		say("could not tell the mesh this node is here: " + err.Error())
 	}
 }
+
+// heldNote is what this apply did NOT do, for the line that says what it did.
+//
+// **A count that does not add up is the only symptom a held resource had** (novox/hq 04-ISSUES/125).
+// An adopted node keeps what it found until its module is taken (ADR 0100), and that is correct — but
+// it was recorded only in the node's own state file. On the edge cut-over the mesh sent 346 resources,
+// the journal said it applied 330, and nothing anywhere said which sixteen or why. Reading it took
+// opening state.json by hand; not reading it took every public name on the machine down, because the
+// operator had four green surfaces and a discrepancy nobody could interpret.
+//
+// So the line that reports the apply carries it. Grouped by module and ordered by name, because the
+// sentence an operator needs is "route-proxy is assigned and not taken", and the module is the thing
+// they can act on — `take` is the verb, and it takes a module.
+func heldNote(held []Held) string {
+	if len(held) == 0 {
+		return ""
+	}
+	byModule := map[string]int{}
+	for _, h := range held {
+		byModule[h.Module]++
+	}
+	names := make([]string, 0, len(byModule))
+	for name := range byModule {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+	parts := make([]string, 0, len(names))
+	for _, name := range names {
+		parts = append(parts, fmt.Sprintf("%s: %d", name, byModule[name]))
+	}
+	return fmt.Sprintf(", %d held until their module is taken (%s)",
+		len(held), strings.Join(parts, ", "))
+}

internal/profile/detectors.go

@@ -20,6 +20,14 @@ const (
 	// state: whether one IS running. Assignment needs the first.
 	CapSeat       = "seat"
 	CapPrivileged = "privileged"
+
+	// The network manager this machine runs, one capability per dialect (novox/hq ADR 0161): the
+	// uplink seat's holder declares its own, so the holder for a manager the machine does not run
+	// is refused the way any missing capability is, naming it. Active, not installed — a machine
+	// may have two of these on disk and runs one.
+	CapUplinkNetworkManager  = "uplink-networkmanager"
+	CapUplinkSystemdNetworkd = "uplink-systemd-networkd"
+	CapUplinkDhcpcd          = "uplink-dhcpcd"
 )
 
 // commandCapability is the shape most detectors take: run something, and treat a working
@@ -202,6 +210,21 @@ func Default(runner Runner) []Detector {
 			why:    "asks the kernel for interfaces — needs the module, not just the tool",
 			runner: runner,
 		},
+		commandCapability{
+			name: CapUplinkNetworkManager, command: "systemctl", args: []string{"is-active", "NetworkManager.service"},
+			why:    "asks the init whether NetworkManager is running — the dialect the uplink seat's holder must speak",
+			runner: runner,
+		},
+		commandCapability{
+			name: CapUplinkSystemdNetworkd, command: "systemctl", args: []string{"is-active", "systemd-networkd.service"},
+			why:    "asks the init whether systemd-networkd is running — the dialect the uplink seat's holder must speak",
+			runner: runner,
+		},
+		commandCapability{
+			name: CapUplinkDhcpcd, command: "systemctl", args: []string{"is-active", "dhcpcd.service"},
+			why:    "asks the init whether dhcpcd is running — the dialect the uplink seat's holder must speak",
+			runner: runner,
+		},
 	}
 }
 

internal/profile/uplink_test.go

@@ -0,0 +1,39 @@
+package profile
+
+import (
+	"context"
+	"errors"
+	"testing"
+)
+
+// The uplink seat's holder must be the dialect the machine runs (novox/hq ADR 0161): the profile
+// names the network manager found active, one capability per manager, and nothing for one that is
+// merely installed.
+func TestTheProfileNamesTheNetworkManagerThatIsRunning(t *testing.T) {
+	runner := func(_ context.Context, name string, args ...string) (string, error) {
+		if name == "systemctl" && len(args) == 2 && args[0] == "is-active" {
+			if args[1] == "NetworkManager.service" {
+				return "active\n", nil
+			}
+			return "inactive\n", errors.New("exit status 3")
+		}
+		return "", errors.New("not here")
+	}
+	var have []Detector
+	for _, d := range Default(Runner(runner)) {
+		switch d.Name() {
+		case CapUplinkNetworkManager, CapUplinkSystemdNetworkd, CapUplinkDhcpcd:
+			have = append(have, d)
+		}
+	}
+	if len(have) != 3 {
+		t.Fatalf("expected a detector per manager, found %d", len(have))
+	}
+	for _, d := range have {
+		v := d.Detect(context.Background())
+		want := d.Name() == CapUplinkNetworkManager
+		if v.Present != want {
+			t.Errorf("%s: present=%v, want %v (%s)", d.Name(), v.Present, want, v.Detail)
+		}
+	}
+}

internal/store/former_test.go

@@ -0,0 +1,29 @@
+package store
+
+import "testing"
+
+// A resource whose target moves leaves what the host wrote under the old target on record as a
+// former one, undeclared by construction, so the next apply removes it (novox/hq issue 097, ADR 0163).
+func TestARecordWhoseTargetMovedKeepsTheFormerTargetToRemove(t *testing.T) {
+	s := State{}
+	s.Record(Applied{ID: "gitea.server", Type: "container", Target: "mesh-gitea", Origin: OriginDeclared})
+	s.Record(Applied{ID: "gitea.server", Type: "container", Target: "gitea", Origin: OriginDeclared})
+	if len(s.Resources) != 2 {
+		t.Fatalf("a moved target produced %d record(s): %+v", len(s.Resources), s.Resources)
+	}
+	orphans := s.Orphans(map[string]bool{"gitea.server": true}, OriginDeclared)
+	if len(orphans) != 1 || orphans[0].Target != "mesh-gitea" || !IsFormer(orphans[0].ID) {
+		t.Fatalf("the former target is not an orphan to remove: %+v", orphans)
+	}
+	s.Forget(orphans[0].ID)
+	if len(s.Resources) != 1 || s.Resources[0].Target != "gitea" {
+		t.Fatalf("forgetting the former target touched the current one: %+v", s.Resources)
+	}
+	// The same target again is not a move; a carried record is not the host's to remove.
+	s.Record(Applied{ID: "gitea.server", Type: "container", Target: "gitea", Origin: OriginDeclared})
+	s.Record(Applied{ID: "bundle", Type: "file", Target: "/a"})
+	s.Record(Applied{ID: "bundle", Type: "file", Target: "/b"})
+	if len(s.Resources) != 2 {
+		t.Fatalf("an unmoved or carried record grew the list: %+v", s.Resources)
+	}
+}

internal/store/store.go

@@ -18,6 +18,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
 	"time"
 )
 
@@ -274,6 +275,41 @@ type Held struct {
 	// reverted: that is how a predecessor still writing is caught.
 	Changed   string    `json:"changed,omitempty"`
 	ChangedAt time.Time `json:"changed_at,omitempty"`
+	// Facts is what a take would compare: the found thing beside what the module declares
+	// (novox/hq ADR 0163). Read fresh on every apply while held, so the controller's preview
+	// speaks of the machine as it is.
+	Facts *Facts `json:"facts,omitempty"`
+}
+
+// Facts is a held thing beside what its module declares — what a take compares (ADR 0163).
+type Facts struct {
+	// A found container: the image it runs and when that image was made; the networks it is on
+	// and the other containers on each; what it mounts; what it publishes.
+	Image        string              `json:"image,omitempty"`
+	ImageCreated string              `json:"image_created,omitempty"`
+	Networks     map[string][]string `json:"networks,omitempty"`
+	Mounts       []string            `json:"mounts,omitempty"`
+	Ports        []string            `json:"ports,omitempty"`
+	// What the module declares for it, and the declared image's creation date when the image
+	// is on the machine already.
+	DeclaredImage        string   `json:"declared_image,omitempty"`
+	DeclaredImageCreated string   `json:"declared_image_created,omitempty"`
+	DeclaredPorts        []string `json:"declared_ports,omitempty"`
+	DeclaredVolumes      []string `json:"declared_volumes,omitempty"`
+	// Downgrade is true when both creation dates are known and the declared image is the older.
+	Downgrade bool `json:"downgrade,omitempty"`
+	// A found file: whether the declared content differs from what was found, and how, as lines
+	// only in the found file (-) and lines only in the declared one (+), bounded.
+	Differs    bool     `json:"differs,omitempty"`
+	Difference []string `json:"difference,omitempty"`
+}
+
+// A Stray is something running on the machine that the mesh neither wrote nor holds
+// (novox/hq ADR 0163): the answer to "what is here that nobody asked for".
+type Stray struct {
+	Kind   string `json:"kind"`
+	Name   string `json:"name"`
+	Detail string `json:"detail,omitempty"`
 }
 
 // Recorded reports whether this host has a record, of any origin, of putting something of this
@@ -462,6 +498,20 @@ func Save(path string, s State) error {
 func (s *State) Record(a Applied) {
 	for i, existing := range s.Resources {
 		if existing.ID == a.ID {
+			// A resource whose target moved leaves what the host wrote under the old target
+			// behind — a container under the old name, a file at the old path. Rewriting the
+			// record would erase the only trace of it (novox/hq issue 097, ADR 0163), so the old
+			// target stays on record as a former one, undeclared by construction, until the next
+			// apply removes it the way it removes anything the host wrote and no longer declares.
+			// What was found is held, never recorded here, and so never removed by this.
+			if originOf(existing) == OriginDeclared && existing.Target != "" && a.Target != "" &&
+				existing.Target != a.Target && existing.Type == a.Type {
+				former := existing
+				former.ID = FormerID(existing.ID, existing.Target)
+				s.Resources[i] = a
+				s.Resources = append(s.Resources, former)
+				return
+			}
 			s.Resources[i] = a
 			return
 		}
@@ -469,6 +519,13 @@ func (s *State) Record(a Applied) {
 	s.Resources = append(s.Resources, a)
 }
 
+// FormerID names the record of a resource's former target: the resource's id and the target it
+// had, so the record is distinct from the current one and is never what a declaration names.
+func FormerID(id, target string) string { return id + "@former:" + target }
+
+// IsFormer says whether a record names a former target.
+func IsFormer(id string) bool { return strings.Contains(id, "@former:") }
+
 // Forget drops a resource from what the node owns.
 func (s *State) Forget(id string) {
 	kept := s.Resources[:0]

packaging/launcher_is_delivered_test.go

@@ -0,0 +1,48 @@
+package packaging_test
+
+import (
+	"encoding/json"
+	"os"
+	"testing"
+)
+
+// The mesh delivers the launcher, so the manifest carries a copy of it (novox/hq 04-ISSUES/142).
+//
+// **Two copies of one script is a drift waiting to happen**, and the only reason to accept it is that
+// a file resource is written atomically — temp file, then rename — while an archive writes in place
+// with truncate. The running launcher keeps the inode it was started from and the next start picks up
+// the new one; unpacking an archive over it would truncate the file a running shell is reading.
+//
+// So: two copies, and this is the check that they are the same one.
+func TestTheManifestCarriesTheLauncherExactly(t *testing.T) {
+	onDisk, err := os.ReadFile("nox-mesh-host-launch")
+	if err != nil {
+		t.Fatal(err)
+	}
+	raw, err := os.ReadFile("../module.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var manifest struct {
+		Resources []struct {
+			ID      string `json:"id"`
+			Content string `json:"content"`
+		} `json:"resources"`
+	}
+	if err := json.Unmarshal(raw, &manifest); err != nil {
+		t.Fatal(err)
+	}
+	for _, r := range manifest.Resources {
+		if r.ID != "launcher" {
+			continue
+		}
+		if r.Content != string(onDisk) {
+			t.Fatal("the launcher the mesh would deliver is not the launcher in this repository. " +
+				"Copy packaging/nox-mesh-host-launch into module.json's `launcher` resource — the " +
+				"machines run what the manifest says, and this file is what gets reviewed")
+		}
+		return
+	}
+	t.Fatal("module.json declares no `launcher` resource, so nothing delivers the launcher and a " +
+		"delivered host version is never started")
+}