A taken container keeps a found network, a left-out module is kept, and genesis raises the forge as its module declares (hq ADR 0163)

A container may name networks it also joins once created, for the per-machine
setting that keeps a found network while a neighbour still resolves it there:
joined after the run, part of the spec, refused when it cannot be joined.

A declaration may say which modules the mesh left out because a stored setting
cannot compose with its definition. Absence used to read as removal; a left-out
module's records are kept and said, and its holds are not released.

Genesis raises the bootstrap forge under the gitea module's container name, with
its image digest and its data directory mounted at /data, so the module holds it
by the found rule instead of raising a second forge beside it (issue 090). The
network is the one difference left for a take to say. Before this the forge had
no volume: its repositories were the container's, lost with it.

cmd/mesh-host/main.go

@@ -33,6 +33,7 @@ import (
 	"github.com/novox/mesh-host/internal/identity"
 	"github.com/novox/mesh-host/internal/inventory"
 	"github.com/novox/mesh-host/internal/link"
+	"github.com/novox/mesh-host/internal/outward"
 	"github.com/novox/mesh-host/internal/profile"
 	"github.com/novox/mesh-host/internal/reachable"
 	"github.com/novox/mesh-host/internal/store"
@@ -50,6 +51,43 @@ var builtFor = ""
 
 var version = "development build"
 
+// runningVersion is this host's version: the directory it was delivered into, or the link-time stamp
+// for one placed by hand.
+//
+// **From where it sits, not from its linker** (novox/hq ADR 0142): "It is unpacked into a directory
+// named for its version, so it can read its own version from its path. The stamp goes, and with it the
+// need for a build to know what it will be called."
+//
+// The mesh's toolchain does not stamp a version, on purpose — a build does not know what it will be
+// called — so a delivered host read as "development build" and the mesh could not tell which host any
+// machine ran (novox/hq 04-ISSUES/161, and 087 for why that matters). The path knows: a delivered host
+// lives at `<libexec>/versions/<version>/<binary>`.
+//
+// A host placed by hand keeps its stamp, which is the honest answer for one the mesh did not deliver.
+func runningVersion() string {
+	self, err := os.Executable()
+	if err != nil {
+		return version
+	}
+	return versionAt(self, version)
+}
+
+// versionAt is runningVersion's decision, with the executable's path and the link-time stamp given —
+// so a test can ask it about a path without being that binary.
+func versionAt(self, stamped string) string {
+	// .../versions/<version>/<binary> — the parent is the version, and its parent is the versions
+	// directory. Checked rather than assumed, so a binary somewhere else does not read a directory
+	// name as a version.
+	dir := filepath.Dir(self)
+	if filepath.Base(filepath.Dir(dir)) != upgrade.VersionsDirName {
+		return stamped
+	}
+	if name := filepath.Base(dir); name != "" && name != "." && name != string(filepath.Separator) {
+		return name
+	}
+	return stamped
+}
+
 const usage = `mesh-host — the node host
 
   profile     what this machine can be asked to do
@@ -253,7 +291,7 @@ func run(ctx context.Context, command string, opts options) error {
 		return runLink(ctx, opts)
 
 	case "version":
-		fmt.Println(version)
+		fmt.Println(runningVersion())
 		return nil
 
 	case "", "help", "-h", "--help":
@@ -419,10 +457,10 @@ func short(digest string) string {
 // them again — and everything the mesh declared read as no longer declared and removed. Even the
 // very declaration the mesh last sent, applied from a file, would plan to remove the foundation.
 // `apply FILE` is for a machine the mesh has not spoken to, and is refused saying so.
-func refuseStale(known store.State, kept store.Declared, keptErr error, digest string, from provenance) error {
+func refuseStale(known store.State, kept store.Declared, keptErr error, digest string, from provenance, sequence int64) error {
 	switch from {
 	case fromDeclared:
-		return nil
+		return refuseOlder(kept, keptErr, sequence)
 	case fromBundle:
 		if known.Genesis == nil || known.Genesis.Digest == digest {
 			return nil
@@ -519,7 +557,7 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
 	if err := apply.CheckMode(known, d); err != nil {
 		return err
 	}
-	if err := refuseStale(known, kept, keptErr, digest, from); err != nil {
+	if err := refuseStale(known, kept, keptErr, digest, from, d.Sequence); err != nil {
 		return err
 	}
 
@@ -593,13 +631,27 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
 	//
 	// A failure to record is reported and does not fail the apply. The apply worked; what is
 	// lost is a rollback's ability to come back here, which is worse to hide than to say.
-	if version != "" {
-		if err := upgrade.RecordKnownGood(upgrade.KnownGoodPath(opts.state), version); err != nil {
+	if v := runningVersion(); v != "" {
+		if err := upgrade.RecordKnownGood(upgrade.KnownGoodPath(opts.state), v); err != nil {
 			fmt.Fprintf(os.Stderr,
 				"mesh-host: applied, but could not record %s as known-good: %v\n"+
 					"  a rollback would have nothing to return to.\n", version, err)
 		}
 	}
+	// And retire what is older than this version's predecessor, on the same evidence known-good is
+	// written on (novox/hq ADR 0141). The predecessor stays, because it is exactly what a rollback
+	// starts; everything before it has no reader. Never this version, whatever the answer.
+	//
+	// A failure is said and does not fail the apply, for the reason above: what is lost is disk, and
+	// hiding it would make a machine quietly fill up.
+	if version != "" {
+		if retired, err := upgrade.Retire(upgrade.VersionsDir(""), version); err != nil {
+			fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not retire an older host: %v\n", err)
+		} else if len(retired) > 0 {
+			fmt.Fprintf(os.Stderr, "mesh-host: retired the host version(s) %s\n",
+				strings.Join(retired, ", "))
+		}
+	}
 	// And tell the launcher this start worked. Without it the counter only climbs, and a node
 	// that has been up for months rolls itself back on its third ordinary restart.
 	if err := upgrade.ClearAttempts(upgrade.AttemptsPath(opts.state)); err != nil {
@@ -691,15 +743,18 @@ func enrol(ctx context.Context, opts options) error {
 	fmt.Printf("  signing key         %s\n",
 		base64.StdEncoding.EncodeToString(token.Signer)[:16]+"...")
 
-	// The check that has to happen before this machine says anything.
-	conn, err := link.Dial(token.Broker, token.Fingerprint, opts.timeout)
-	if err != nil {
-		return err
-	}
-	defer conn.Close()
-	fmt.Println("\nthe broker presented the certificate this token pins")
-
-	conn.Close()
+	// **The pin is checked by the connection that presents this token, not by a dial of our own**
+	// (novox/hq 04-ISSUES/146). This opened a raw TLS connection to the bus first, which worked
+	// against the broker the mesh used to run and cannot work against the one it runs now: NATS
+	// speaks its own protocol before it upgrades to TLS, so an immediate handshake is answered
+	// with a plaintext line and the enrolment failed with "first record does not look like a TLS
+	// handshake" — on every node that has tried to join since the bus changed, which is why this
+	// went unnoticed: none had.
+	//
+	// What ADR 0004 requires still holds, and holds better: the client that presents the token
+	// carries the same pinned configuration, reads the server's greeting, upgrades, and the
+	// verification runs inside that handshake — so the one-time secret is sent only after the
+	// certificate has been checked, and nothing of this node's reaches an impostor.
 
 	mine, err := identity.Generate(*name)
 	if err != nil {
@@ -763,20 +818,23 @@ func enrol(ctx context.Context, opts options) error {
 	// control plane cannot decide what a node should run without it, so it travels with the
 	// request instead of being asked for in a second round trip.
 	detected := profile.Detect(ctx, profile.Default(nil), opts.timeout)
-	reported := map[string]any{}
-	if raw, err := json.Marshal(detected); err == nil {
-		_ = json.Unmarshal(raw, &reported)
-	}
+	reported := profileAsReported(detected)
 
 	// Signed with the identity just generated, so the mesh can tell this machine from anyone else
 	// who knows its public key (novox/hq issue 083).
 	proof := mine.Sign(link.EnrolProof(token.Secret, mine.Public, mine.Overlay.Public,
 		sealing.Public, serving.Public))
 	// The token says where to go and which certificate that address must present. It says nothing
-	// about which bus is there, and does not need to: every token names the one the mesh runs on
-	// today until the rollout (novox/hq ADR 0116 step 5), and that is what an empty Transport is.
+	// about which bus is there, and does not need to: there is one, and this host knows which
+	// (novox/hq ADR 0131 — the mesh speaks to one seat and the old transport is gone).
+	//
+	// **It used to leave this empty** and mean "whatever the mesh runs today", which was true
+	// while two buses existed and became a refusal the moment one did: an empty transport is not
+	// the bus's name, so every enrolment ended at "this token is for the \"\" bus"
+	// (novox/hq 04-ISSUES/146). Nothing caught it because nothing had enrolled since the bus
+	// changed.
 	reply, err := link.Enrol(ctx,
-		link.Approach{Address: token.Broker, Fingerprint: token.Fingerprint},
+		link.Approach{Address: token.Broker, Fingerprint: token.Fingerprint, Transport: link.OnNATS},
 		*name, token.Secret,
 		mine.Public, mine.Overlay.Public, sealing.Public, serving.Public, reported, proof, found,
 		opts.timeout)
@@ -792,6 +850,13 @@ func enrol(ctx context.Context, opts options) error {
 		Fingerprint: firstNonEmpty(reply.Fingerprint, token.Fingerprint),
 		Signer:      firstNonEmpty2(reply.Signer, token.Signer),
 		Password:    reply.Password,
+		// **Which bus this membership is for, said rather than left empty** (novox/hq
+		// 04-ISSUES/146). The link refuses a membership that names another bus, and an empty name
+		// is not this one's — so a node enrolled without it came up and reconnected for ever
+		// against its own record: "this membership is for \"\", and the mesh's bus is nats". The
+		// reply does not carry it because there is one bus and the host knows which (ADR 0131);
+		// what was missing was writing that down where the link reads it.
+		Transport: link.OnNATS,
 	}
 	if mine.Membership.Password == "" {
 		// The mesh did not replace the token's secret, so it is still this node's broker
@@ -978,12 +1043,36 @@ func runLink(ctx context.Context, opts options) error {
 	sched := apply.NewScheduler(apply.SystemClock(), apply.ExecRunner, say)
 	go sched.Run(ctx)
 
+	// **Standing aside for a successor happens between reconciles and nowhere else** (novox/hq ADR
+	// 0141). A host that stood aside mid-apply is the half-configured machine this host exists to
+	// prevent, so the question is asked after an apply has finished and the answer is a clean exit —
+	// which the launcher already reads as "run whatever is on disk now".
+	aside, standAside := context.WithCancel(ctx)
+	defer standAside()
+	stoodAside := false
+
 	applier := func(ctx context.Context, raw, signature []byte) link.Report {
 		report := applyAndKeep(ctx, opts, raw, &store.Declared{Declaration: raw, Signature: signature}, sched, say)
 		// **A declaration may carry this machine's membership for another bus.** It arrives as a
 		// sealed file like any secret, and is read after the rest has applied so the bus it names is
 		// standing before this machine leaves the one it is on (novox/hq design 28, task 5.2).
 		adoptDeliveredMembership(identity.Path(opts.state), &mine, say)
+
+		// Asked with the version this host is RUNNING, read from where it sits — not the link-time
+		// stamp, which every delivered host carries as "development build". Asked with the stamp,
+		// a delivered host never matched the newest delivered version, so it stood aside on every
+		// push for ever, and standing aside cancels the report, so the mesh never heard from it
+		// again (novox/hq 04-ISSUES/163).
+		switch next, waiting, err := upgrade.Successor(upgrade.VersionsDir(""), runningVersion()); {
+		case err != nil:
+			// Said, not fatal. A host that cannot read the delivered versions is still running this
+			// machine correctly; what it has lost is the ability to be replaced.
+			say(fmt.Sprintf("cannot tell whether a newer host is delivered: %v", err))
+		case waiting:
+			say(fmt.Sprintf("host %s is delivered; standing aside so the launcher runs it", next.Version))
+			stoodAside = true
+			standAside()
+		}
 		return report
 	}
 
@@ -1014,7 +1103,7 @@ func runLink(ctx context.Context, opts options) error {
 		}}
 	})
 
-	return link.HoldRoused(ctx, link.Membership{
+	held := link.HoldRoused(aside, link.Membership{
 		Node:        mine.Node,
 		Broker:      mine.Membership.Broker,
 		Fingerprint: mine.Membership.Fingerprint,
@@ -1022,6 +1111,13 @@ func runLink(ctx context.Context, opts options) error {
 		Transport:   mine.Membership.Transport,
 		Signer:      mine.Membership.Signer,
 	}, applier, say, opts.timeout, rousedBySignal(ctx), outbox)
+	// **Cleanly**, or the launcher counts standing aside as a crash and rolls the new host back
+	// before it has run once. The context this returns on was cancelled deliberately, so its error
+	// is not a fault to report.
+	if stoodAside {
+		return nil
+	}
+	return held
 }
 
 // adoptionWatch remembers what the node last said about what it holds and its firewall, so a
@@ -1036,7 +1132,12 @@ type adoptionWatch struct {
 // is what the controller previews a flip from, so a port that opens or closes between deliveries
 // must reach it too (novox/hq ADR 0100).
 func adoptionFingerprint(r link.Report) string {
-	parts := []string{"firewall=" + r.Firewall}
+	// **Which links face outside is part of it, though it is not about adoption** (novox/hq ADR
+	// 0140). The mesh composes no filter for a machine that has not said, so a machine whose links
+	// changed — or which has only just learnt to say — has to say so without being asked. Left out,
+	// it could only speak when a declaration arrived, and a declaration cannot be composed until it
+	// has spoken: a machine waiting for a push that is waiting for the machine.
+	parts := []string{"firewall=" + r.Firewall, "outward=" + strings.Join(r.Outward, ",")}
 	for _, h := range r.Held {
 		parts = append(parts, "held "+h.ID+"="+h.Changed)
 	}
@@ -1044,7 +1145,7 @@ func adoptionFingerprint(r link.Report) string {
 		parts = append(parts, fmt.Sprintf("reach %s %s:%d %s %v %d", reach.Protocol, reach.Address,
 			reach.Port, reach.By, reach.Published, reach.ContainerPort))
 	}
-	sort.Strings(parts[1:])
+	sort.Strings(parts[2:])
 	return strings.Join(parts, "\n")
 }
 
@@ -1152,7 +1253,17 @@ func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, s
 		// A reconcile is otherwise silent. On an adopted node it speaks when what it holds or
 		// its firewall changed, because that is how a predecessor still writing is caught
 		// (novox/hq ADR 0100); publish decides whether anything did.
-		if publish != nil && report.Refused == "" && (len(report.Held) > 0 || report.Firewall != "") {
+		//
+		// **And on any node, when it can say which links face outside** (novox/hq ADR 0140). A
+		// converged node holds nothing and found no firewall, so this gate closed on it and the
+		// node could speak only in reply to a declaration — while the mesh composes no declaration
+		// for a node that has not said which links face outside. A machine waiting for a push that
+		// was waiting for the machine, and measured: three converged machines sat silent while the
+		// control plane refused to send them a filter.
+		//
+		// Offered, not published: whether it is news is still the watch's to decide, so an
+		// unchanged answer costs one comparison every reconcile and nothing on the bus.
+		if publish != nil && worthSaying(report) {
 			publish(report)
 		}
 		switch {
@@ -1164,6 +1275,23 @@ func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, s
 	}
 }
 
+// worthSaying is whether a reconcile's report carries anything the mesh needs to hear unasked.
+//
+// **Named rather than written into the loop**, so the rule can be tested. It was a condition inline
+// and it was wrong in a way nothing could catch: it asked only what an adopted node reports, so a
+// converged node — which holds nothing and found no firewall — could speak only in reply to a
+// declaration, while the mesh composes no declaration for a node that has not said which links face
+// outside (novox/hq ADR 0140). Three machines sat silent waiting for a push that was waiting for them.
+//
+// A refused report says nothing about the machine, so it is not news; the refusal is said on the
+// console where a person reads it.
+func worthSaying(report link.Report) bool {
+	if report.Refused != "" {
+		return false
+	}
+	return len(report.Held) > 0 || report.Firewall != "" || len(report.Outward) > 0
+}
+
 // applyDeclared applies a declaration that has already been proved to come from the mesh.
 //
 // Signature checking happens before this is called, in the link. By the time anything here runs,
@@ -1173,6 +1301,21 @@ func applyDeclared(ctx context.Context, opts options, raw []byte, sched *apply.S
 	return applyAndKeep(ctx, opts, raw, nil, sched, say)
 }
 
+// announceOr is what the apply writes its detail with, given what the caller has to say things with.
+//
+// **Never nil.** This argument was nil on the serving path, and nil is silence: everything the apply
+// says — a file held, a container replaced, the found firewall retired — was visible when a person ran
+// the one-shot command and discarded in the way the host actually runs (novox/hq 04-ISSUES/143).
+//
+// Named rather than written inline at the call site so both paths reach the apply the same way, and so
+// a reader asking "where does the apply's output go" finds one answer.
+func announceOr(say link.Announce) func(string) {
+	if say == nil {
+		return func(string) {}
+	}
+	return say
+}
+
 // applying serialises applies within this process.
 //
 // **Two things apply here: the link and the reconcile loop**, and each reads the node's state,
@@ -1235,8 +1378,18 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
 
 	// Declared, not carried. A declaration from the mesh removes only what the mesh previously
 	// declared — never what this machine raised for itself from its bundle (04-ISSUES/010).
+	// **What the apply says goes to the console, which is the journal when this runs as a service.**
+	//
+	// It was nil, and nil is silence. The one-shot path has always passed a real one, so every detail
+	// the apply produces — a file held, a container replaced, the found firewall retired — was visible
+	// when a person ran it by hand and discarded in the way the host actually runs. Measured: after a
+	// machine was converged and its found firewall was not retired, what the host decided was
+	// unrecoverable, because it had said it to nobody (novox/hq 04-ISSUES/143).
+	//
+	// `say` already reaches stdout, and the launcher's unit sends that to the journal, so this needs
+	// no new mechanism — only for the argument to be passed.
 	outcome, updated, applyErr := apply.ApplyKeeping(ctx, built, declared, known, store.OriginDeclared,
-		apply.ExecRunner, nil, sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)))
+		apply.ExecRunner, announceOr(say), sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)))
 
 	// The mode the mesh said, recorded whichever way the apply went: the declaration is kept
 	// either way, and the node is held to it from the next reconcile (novox/hq ADR 0100).
@@ -1262,12 +1415,30 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
 		sched.Sync(declared, held)
 	}
 
-	report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw)}
+	report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw), Host: runningVersion(),
+		Profile: profileAsReported(profile.Detect(ctx, profile.Default(nil), opts.timeout))}
+	// Which of this machine's links face outside, for the filter the mesh writes around them
+	// (novox/hq ADR 0140). Reported whatever the node's mode: a converged node's filter needs it,
+	// and an adopted one becomes converged without a further round trip. A machine that cannot read
+	// its own routing table says nothing rather than guessing, and is sent no filter.
+	if links, err := outward.Links(""); err != nil {
+		fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not read which links face outside: %v\n", err)
+	} else {
+		report.Outward = links
+	}
 	// What this node found and holds, its firewall, and what is reachable on it — so an adopted
 	// node never reads as converged (novox/hq ADR 0100).
 	for _, h := range updated.Held {
 		report.Held = append(report.Held, link.Held{ID: h.ID, Module: h.Module, Kind: h.Kind,
-			Target: h.Target, Since: h.Since, Changed: h.Changed, Kept: h.Kept})
+			Target: h.Target, Since: h.Since, Changed: h.Changed, Kept: h.Kept, Facts: factsAsReported(h.Facts)})
+	}
+	// And what runs here that nobody asked for (novox/hq ADR 0163).
+	if strays, err := apply.Strays(ctx, apply.ExecRunner, updated); err != nil {
+		fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not list what else runs here: %v\n", err)
+	} else {
+		for _, s := range strays {
+			report.Strays = append(report.Strays, link.Stray{Kind: s.Kind, Name: s.Name, Detail: s.Detail})
+		}
 	}
 	if declared.Adoption != nil {
 		if updated.Firewall != nil {
@@ -1436,3 +1607,54 @@ func adoptDeliveredMembership(identityPath string, mine *identity.Identity, say
 	say(fmt.Sprintf("moving to the %s bus at %s — restarting to dial it", next.Transport, next.Broker))
 	os.Exit(0)
 }
+
+// refuseOlder refuses a declaration from the mesh that is older than the one this node holds.
+//
+// **By sequence, not by arrival** (novox/hq 04-ISSUES/107). What the host kept is the last thing
+// the mesh said, signed; a declaration whose sequence is lower was composed before it, whatever
+// order they arrived in — a backlog drained after the node was away, or a broker that split a burst.
+// Applying it would make the machine into something the mesh had already moved past, which is the
+// incident of issue 104 by another door.
+//
+// Only when both sides claim an order. A declaration with no sequence is one an older controller
+// sent, and one kept with no sequence is one this host received before it understood them; in either
+// case there is no order to compare, and refusing on a guess would strand the node the moment the
+// controller is older than the host. Equal is the same declaration again, which reconciling is for.
+func refuseOlder(kept store.Declared, keptErr error, sequence int64) error {
+	if sequence == 0 || keptErr != nil {
+		return nil
+	}
+	last, err := declaration.ParseTrusted(kept.Declaration)
+	if err != nil || last.Sequence == 0 {
+		return nil
+	}
+	if sequence < last.Sequence {
+		return fmt.Errorf("this declaration is older than what the mesh last said to this node: it "+
+			"is sequence %d, and the one kept here is %d. It arrived late — a backlog, or a broker "+
+			"that split a burst — and applying it would make this machine into something the mesh has "+
+			"already moved past. Refused whole; nothing was applied", sequence, last.Sequence)
+	}
+	return nil
+}
+
+// profileAsReported is the profile as the mesh reads it — the same bytes enrolment sends, so a
+// report's profile and an enrolment's are one shape on the controller's side (novox/hq ADR 0161).
+func profileAsReported(detected profile.Profile) map[string]any {
+	reported := map[string]any{}
+	if raw, err := json.Marshal(detected); err == nil {
+		_ = json.Unmarshal(raw, &reported)
+	}
+	return reported
+}
+
+// factsAsReported is a held thing's facts as the mesh reads them: the same bytes the host keeps.
+func factsAsReported(f *store.Facts) map[string]any {
+	if f == nil {
+		return nil
+	}
+	out := map[string]any{}
+	if raw, err := json.Marshal(f); err == nil {
+		_ = json.Unmarshal(raw, &out)
+	}
+	return out
+}

cmd/mesh-host/main_test.go

@@ -501,3 +501,95 @@ func TestReconcileAfterAControllerDeclarationDoesNotReapplyTheBundle(t *testing.
 		t.Errorf("with nothing said, reconcile did not reach for the carried bundle: %v", err)
 	}
 }
+
+// **A machine says which links face outside without being asked.**
+//
+// The mesh composes no filter for a machine that has not said (novox/hq ADR 0140), and a machine only
+// speaks unasked when this fingerprint changes. Left out of it, a machine that has just learnt to say
+// could speak only when a declaration arrived — and a declaration cannot be composed until it has
+// spoken. A machine waiting for a push that is waiting for the machine.
+func TestANewOutwardLinkIsSaidUnasked(t *testing.T) {
+	w := &adoptionWatch{}
+	first := link.Report{Firewall: "none"}
+	if !w.differs(first) {
+		t.Fatal("the first report should differ from nothing")
+	}
+	w.said(first)
+
+	// Only the links changed, and nothing about adoption.
+	learnt := link.Report{Firewall: "none", Outward: []string{"eth0"}}
+	if !w.differs(learnt) {
+		t.Fatal("a machine that has just learnt which links face outside would never say so, " +
+			"and could then never be sent a filter")
+	}
+	w.said(learnt)
+	if w.differs(link.Report{Firewall: "none", Outward: []string{"eth0"}}) {
+		t.Fatal("the same links are reported as a change, so the machine would speak on every reconcile")
+	}
+
+	// And a link that changes — a laptop moving from a cable to a radio — is said too, because the
+	// filter is written around the old one until it is.
+	if !w.differs(link.Report{Firewall: "none", Outward: []string{"wlan0"}}) {
+		t.Fatal("a changed outward link is not said, so the filter stays written around the old one")
+	}
+}
+
+// **A converged machine can say which links face outside, unasked.**
+//
+// A reconcile is otherwise silent, and the condition deciding when it speaks asked only what an
+// adopted node reports — what it holds, and the firewall it found. A converged node has neither, so
+// it could speak only in reply to a declaration, and the mesh composes no declaration for a node
+// that has not said which links face outside (novox/hq ADR 0140). Measured: three converged machines
+// sat silent while the control plane refused to send them a filter.
+func TestAConvergedMachineSaysItsOutwardLinksUnasked(t *testing.T) {
+	// A converged node's reconcile: nothing held, no found firewall, and the links it can see.
+	converged := link.Report{Outward: []string{"eth0"}}
+	if !worthSaying(converged) {
+		t.Fatal("a converged machine cannot say which links face outside, so it can never be " +
+			"sent a filter — a machine waiting for a push that is waiting for the machine")
+	}
+
+	// An adopted node's reasons still hold, because that is how a predecessor still writing is caught.
+	if !worthSaying(link.Report{Firewall: "ufw"}) {
+		t.Fatal("an adopted machine no longer says which firewall it found")
+	}
+	if !worthSaying(link.Report{Held: []link.Held{{ID: "a-file"}}}) {
+		t.Fatal("an adopted machine no longer says what it holds")
+	}
+
+	// And a reconcile with nothing to say stays silent, or every machine speaks every five minutes
+	// about nothing.
+	if worthSaying(link.Report{}) {
+		t.Fatal("a reconcile with nothing to say speaks anyway")
+	}
+
+	// A refused report says nothing about the machine; the refusal is for the console.
+	if worthSaying(link.Report{Outward: []string{"eth0"}, Refused: "not for this node"}) {
+		t.Fatal("a refused report is offered as news about the machine")
+	}
+}
+
+// **A host running as a service says what its apply did.**
+//
+// The serving path passed nil where the apply writes its detail, and nil is silence. The one-shot path
+// has always passed a real function, so everything the apply says was visible when a person ran it by
+// hand and discarded in the way the host actually runs. Measured before this was written: a machine was
+// converged, its found firewall was not retired, and what the host decided was unrecoverable because it
+// had been said to nobody (novox/hq 04-ISSUES/143).
+//
+// This asserts only that the apply's log is never nil and that a line reaches what the caller gave.
+// **It cannot catch the fault it was written for** — a call site passing nil directly — because that is
+// wiring, and wiring is only proved by running the thing. That proof is a deployed host whose journal
+// carries the apply's detail, which is how this fix was verified.
+func TestTheApplysLogIsNeverNil(t *testing.T) {
+	if announceOr(nil) == nil {
+		t.Fatal("a host with nowhere to say things got a nil log, which the apply will call")
+	}
+	announceOr(nil)("this goes nowhere and must not panic")
+
+	var said []string
+	announceOr(func(line string) { said = append(said, line) })("  disabled ufw")
+	if len(said) != 1 || !strings.Contains(said[0], "disabled ufw") {
+		t.Fatalf("the apply's detail did not reach the caller's announce: %v", said)
+	}
+}

cmd/mesh-host/order_test.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// A declaration carries no order, so a host cannot tell an older one from a newer (novox/hq
+// 04-ISSUES/107). Its only identity was the digest of its bytes: "not the last" could be said,
+// "older" could not.
+
+func keptWith(t *testing.T, sequence int64) store.Declared {
+	t.Helper()
+	body, err := json.Marshal(map[string]any{
+		"declaration": 1, "resources": []any{}, "owns_nothing": true, "sequence": sequence,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return store.Declared{Declaration: body, Signature: []byte("x")}
+}
+
+func TestAnOlderDeclarationFromTheMeshIsRefused(t *testing.T) {
+	err := refuseOlder(keptWith(t, 7), nil, 5)
+	if err == nil {
+		t.Fatal("sequence 5 was accepted over a kept 7")
+	}
+	if !strings.Contains(err.Error(), "older") || !strings.Contains(err.Error(), "nothing was applied") {
+		t.Fatalf("the refusal does not say what it is: %v", err)
+	}
+}
+
+func TestANewerOrEqualDeclarationIsNot(t *testing.T) {
+	if err := refuseOlder(keptWith(t, 7), nil, 8); err != nil {
+		t.Fatalf("sequence 8 was refused over a kept 7: %v", err)
+	}
+	// Equal is the same declaration again, which reconciling is for.
+	if err := refuseOlder(keptWith(t, 7), nil, 7); err != nil {
+		t.Fatalf("the same sequence was refused: %v", err)
+	}
+}
+
+func TestNoOrderClaimedMeansNoOrderCompared(t *testing.T) {
+	// An older controller sends none; a host that received before it understood them kept none.
+	// Refusing on a guess would strand a node the moment the controller is older than the host.
+	if err := refuseOlder(keptWith(t, 7), nil, 0); err != nil {
+		t.Fatalf("a declaration claiming no order was refused: %v", err)
+	}
+	if err := refuseOlder(keptWith(t, 0), nil, 3); err != nil {
+		t.Fatalf("a declaration was refused against a kept one that claimed no order: %v", err)
+	}
+	if err := refuseOlder(store.Declared{}, store.ErrNothingDeclared, 3); err != nil {
+		t.Fatalf("a first declaration was refused: %v", err)
+	}
+}

cmd/mesh-host/version_from_path_test.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// A component's version comes from where it sits, not from its linker (novox/hq ADR 0142). The mesh's
+// toolchain stamps no version — a build does not know what it will be called — so a delivered host
+// read as "development build" and the mesh could not tell which host a machine ran (04-ISSUES/161).
+
+func TestADeliveredHostReadsItsVersionFromItsPath(t *testing.T) {
+	// A delivered host lives at <libexec>/versions/<version>/<binary>.
+	dir := t.TempDir()
+	versioned := filepath.Join(dir, "versions", "637f65559d16")
+	if err := os.MkdirAll(versioned, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	self := filepath.Join(versioned, "nox-mesh-host")
+	if err := os.WriteFile(self, []byte("#!/bin/sh\n"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if got := versionAt(self, "development build"); got != "637f65559d16" {
+		t.Fatalf("a delivered host read its version as %q", got)
+	}
+}
+
+func TestAHostPlacedByHandKeepsItsStamp(t *testing.T) {
+	// The honest answer for one the mesh did not deliver — and every machine is in that state until
+	// a delivery reaches it.
+	if got := versionAt("/usr/bin/nox-mesh-host", "04a27ca"); got != "04a27ca" {
+		t.Fatalf("a hand-placed host read its version as %q", got)
+	}
+}
+
+func TestADirectoryThatIsNotAVersionIsNotReadAsOne(t *testing.T) {
+	// A binary sitting anywhere else must not have its parent directory's name read as a version.
+	for _, path := range []string{
+		"/opt/somewhere/nox-mesh-host",
+		"/usr/lib/nox-mesh-host/launch",
+		"/home/someone/build/nox-mesh-host",
+	} {
+		if got := versionAt(path, "the stamp"); got != "the stamp" {
+			t.Fatalf("%s read its version as %q", path, got)
+		}
+	}
+}

examples/foundation-first-node-nats.lock

@@ -127,12 +127,21 @@
     {
       "id": "bus-certificate",
       "type": "action",
-      "command": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls",
-                  "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927",
-                  "-c", "test -f /tls/tls.crt || (openssl req -x509 -newkey rsa:2048 -nodes -keyout /tls/tls.key -out /tls/tls.crt -days 3650 -subj '/CN=mesh-broker' -addext 'subjectAltName=DNS:mesh-broker,IP:127.0.0.1' >/dev/null 2>&1 && chmod 644 /tls/tls.crt && chmod 600 /tls/tls.key)"],
-      "verify": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls",
-                 "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927",
-                 "-c", "test -s /tls/tls.crt && openssl x509 -in /tls/tls.crt -noout"]
+      // **The mesh makes its own** (novox/hq 04-ISSUES/146). This ran `openssl` inside the
+      // broker's image while the broker was one that carried it; the bus that replaced it has a
+      // shell and no openssl, and no other image the bundle names has one either. So the program
+      // that needs the certificate writes it — already on this machine, since the schema step ran
+      // it, and asking nothing of the image it writes into. Self-signed on purpose: a host pins
+      // this server's exact certificate (novox/hq ADR 0004), and at this moment there is no mesh
+      // to ask an authority of.
+      // `--user 0:0` because the volume is root's and this image runs as nobody, which is right
+      // for the long-running control plane and wrong for a one-shot writing into a fresh volume.
+      "command": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
+                  "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
+                  "broker", "certificate", "--into", "/tls"],
+      "verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
+                 "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
+                 "broker", "certificate", "--check", "--into", "/tls"]
     },
     {
       "id": "bus-conf-dir",
@@ -152,7 +161,7 @@
       "type": "file",
       "path": "/var/lib/mesh-bus-conf/accounts.conf",
       "mode": "0600",
-      "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n  MESH {\n    jetstream: enabled\n    users = [\n      { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n        publish: { allow: [\"$JS.API.>\", \"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"_INBOX.enrol.>\", \"mesh.control.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\"] }\n        subscribe: { allow: [\"$JS.API.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\"] }\n        allow_responses: { max: 1, ttl: \"1m\" }\n      } }\n    ]\n  }\n}\n"
+      "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n  MESH {\n    jetstream: enabled\n    users = [\n      { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n        publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.API.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.control.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\"] }\n        subscribe: { allow: [\"$JS.API.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.mesh-controller.tool.>\"] }\n        allow_responses: { max: 1, ttl: \"1m\" }\n      } }\n    ]\n  }\n}\n"
     },
     {
       "id": "broker",

internal/apply/apply.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -230,6 +231,18 @@ func ApplyKeeping(
 			protecting = append(protecting, orphan)
 			continue
 		}
+		// **A module the mesh left out is not a module the mesh removed** (novox/hq ADR 0163, rule
+		// 6): its resources are absent because a setting stored for it cannot compose, and the
+		// mesh said so by name. What the host wrote for it stays as it is, recorded, until the
+		// module is declared again or unassigned.
+		if module, left := d.LeftOutModuleOf(orphan.ID); left {
+			report.Outcomes = append(report.Outcomes, Outcome{
+				ID: orphan.ID, Type: orphan.Type, Target: orphan.Target,
+				Action: "unchanged", Detail: "kept: " + module + " was left out of this declaration by the mesh, not removed",
+			})
+			log(fmt.Sprintf("  kept %s (%s): %s was left out of this declaration by the mesh, not removed", orphan.ID, orphan.Target, module))
+			continue
+		}
 		orphans = append(orphans, orphan)
 	}
 	ordered := d.Resources
@@ -270,6 +283,11 @@ func ApplyKeeping(
 			if declared[h.ID] {
 				continue
 			}
+			if slices.Contains(d.LeftOut, h.Module) {
+				// Left out, not unassigned (ADR 0163, rule 6): still held for the module, as the
+				// mesh asked.
+				continue
+			}
 			known.Release(h.ID)
 			report.Outcomes = append(report.Outcomes, Outcome{ID: h.ID, Type: h.Kind, Target: h.Target,
 				Action: "forgotten", Detail: "no longer declared; left as found"})
@@ -320,6 +338,9 @@ func ApplyKeeping(
 	// is a different thing — one is "this machine could not do it", the other is "this was never
 	// a declaration", and they are fixed in different places.
 	var failures []*Error
+	// Modules whose own step did not complete. What follows *within such a module* is not attempted;
+	// the rest of the machine is (novox/hq ADR 0136).
+	gated := map[string]bool{}
 	for i, resource := range ordered {
 		if !orphansRemoved && i == guardFirst {
 			// **Only a guard that is up may let the filter go.** Removing the derived filter's
@@ -337,6 +358,17 @@ func ApplyKeeping(
 				return report, known, err
 			}
 		}
+		// **A module whose step did not complete is skipped from there on** (novox/hq ADR 0136).
+		// Reported rather than passed over in silence: "not attempted" and "nothing to do" are
+		// different answers, and only one of them is somebody's to fix.
+		if module, ours := moduleOf(resource.Identity()); ours && gated[module] {
+			report.Outcomes = append(report.Outcomes, Outcome{
+				ID: resource.Identity(), Type: string(resource.Kind()), Target: resource.Target(),
+				Action: "skipped", Detail: "a step this module declares did not complete",
+			})
+			continue
+		}
+
 		// **On an adopted node, what is found is kept until its module is taken** (novox/hq ADR
 		// 0100, ADR 0103). Before anything is applied: whatever of a module not yet taken is
 		// present with no record of this host making it — or would reach what is — is held as it
@@ -465,9 +497,26 @@ func ApplyKeeping(
 				gates = true
 			}
 			if gates {
+				failed.Gated = true
+				// **A module's step gates that module, not the machine** (novox/hq ADR 0136).
+				//
+				// Stopping the whole apply is what this loop's own comment above calls holding a
+				// machine hostage, and it was already rejected for every other shape (04-ISSUES/011).
+				// A step exists to make something true before the next thing in *its module* needs it
+				// — a store seeded before the broker starts, a schema prepared before the version
+				// that needs it runs — so that is exactly how far the gate reaches. Everything else
+				// on the machine is independent state and is attempted.
+				//
+				// An action still gates the machine: the bootstrap is a row of them, each making the
+				// next possible, and they belong to no module.
+				if module, ours := moduleOf(resource.Identity()); ours {
+					gated[module] = true
+					log(fmt.Sprintf("  gated %s.*: a step it declares did not complete, so the rest "+
+						"of it was not attempted", module))
+					continue
+				}
 				failed.Done = report
 				failed.Others = len(failures) - 1
-				failed.Gated = true
 				return report, known, failed
 			}
 			continue
@@ -1470,6 +1519,22 @@ func containerSpecReading(r *declaration.Container, declares, reads map[string]s
 	for _, a := range r.Args {
 		b.WriteString("arg " + a + "\n")
 	}
+	// **A container's declared names are part of what it is** (novox/hq 04-ISSUES/135). What is
+	// here is what the module declared for itself and nothing else: the mesh's own names are no
+	// longer written into a container (ADR 0148) — they were once, every container got the whole
+	// roster at creation and nothing re-read it, so one left alone when the roster moved could not
+	// reach anything by name for as long as it ran while every check reported it running; and once
+	// the roster was in this digest so that could be caught, one name moving anywhere replaced
+	// every container in the mesh (04-ISSUES/151). A container resolves a mesh name through the
+	// machine's resolver at the moment it asks. What a module declares does not move when the
+	// roster does, so hashing it costs nothing and catches a manifest that changed.
+	//
+	// Sorted, so the digest does not move for a reordering nobody made.
+	hosts := append([]string(nil), r.Hosts...)
+	sort.Strings(hosts)
+	for _, h := range hosts {
+		b.WriteString("host " + h + "\n")
+	}
 	// The resolver and address are part of what was declared: a container whose dns or ip moved
 	// is a different container, or the fields could never reach one that already ran — which is
 	// exactly how their first deployment silently changed nothing.
@@ -1479,6 +1544,11 @@ func containerSpecReading(r *declaration.Container, declares, reads map[string]s
 	if r.IP != "" {
 		b.WriteString("ip " + r.IP + "\n")
 	}
+	// The networks it also joins are part of what it is (ADR 0163, rule 4): kept or let go, the
+	// container is recreated, and a neighbour's reach changes with it.
+	for _, n := range r.Networks {
+		b.WriteString("also-on " + n + "\n")
+	}
 	// The cadence is part of what was declared, so a changed schedule is a changed spec — the marker
 	// moves and the install is reported "updated" and re-established. Added only when present, so no
 	// ordinary container's or run-once step's digest moves for a field it does not set.
@@ -1719,6 +1789,15 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner,
 	if after.Spec != want {
 		return out, fmt.Errorf("container %s is not the one that was declared after creating it", r.Name)
 	}
+	// The found networks a per-machine setting keeps for it (novox/hq ADR 0163, rule 4), joined
+	// once it runs: a runtime starts a container on one network, and the others are connected.
+	// Refused, not skipped, when one cannot be joined — a neighbour that was promised to keep
+	// reaching this container by name would silently not.
+	for _, n := range r.Networks {
+		if _, err := run(ctx, cri, "network", "connect", n, r.Name); err != nil {
+			return out, fmt.Errorf("container %s could not join the kept network %s: %w", r.Name, n, err)
+		}
+	}
 
 	out.Action = "created"
 	if existed {
@@ -2247,3 +2326,25 @@ func meshMadeUnits(known store.State) map[string]bool {
 	}
 	return made
 }
+
+// moduleOf is the module a declared resource belongs to.
+//
+// The mesh composes a module's resource ids as `<module>.<its own id>`, and **a module's name may
+// contain a dot** — `novox.be` is one on this mesh — while a resource's own id never does. So the
+// owner is everything before the *last* dot; reading to the first one would make `novox.be.server`
+// belong to a module called "novox", and a gate would then skip whatever else happened to start that
+// way.
+//
+// False for what the mesh declares in its own right: the foundation's resources carry no dot at all,
+// and the adoption's are named for the mesh rather than for a module. Both belong to no module, and
+// their gate is therefore the machine's.
+func moduleOf(identity string) (string, bool) {
+	if strings.HasPrefix(identity, declaration.AdoptionPrefix) {
+		return "", false
+	}
+	at := strings.LastIndex(identity, ".")
+	if at <= 0 {
+		return "", false
+	}
+	return identity[:at], true
+}

internal/apply/facts_test.go

@@ -0,0 +1,97 @@
+package apply
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// A take is a comparison (novox/hq ADR 0163): while a module's container is held, the host reports
+// the found image and its age beside the declared one, the networks and who else is on them, the
+// mounts and the ports — and says when the declared image is the older.
+func TestAHeldContainerCarriesTheFactsATakeCompares(t *testing.T) {
+	dir, page, m := predecessor(t)
+	m.containers["hello-web"].image = "web:1.27"
+	m.containers["hello-web"].imageID = "sha256:found"
+	m.containers["hello-web"].networks = []string{"predecessor_default"}
+	m.containers["hello-web"].mounts = []string{"/srv/web:/data"}
+	m.containers["hello-web"].ports = []string{"80/tcp>0.0.0.0:8080"}
+	m.images = map[string]string{"sha256:found": "2026-09-17T10:00:00Z", pinned: "2026-08-20T10:00:00Z"}
+	m.members = map[string][]string{"predecessor_default": {"hello-web", "office", "db"}}
+
+	_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
+	h, ok := state.HeldAt("hello-web.server")
+	if !ok || h.Facts == nil {
+		t.Fatalf("a held container carries no facts: %+v", h)
+	}
+	f := h.Facts
+	if f.Image != "web:1.27" || f.ImageCreated != "2026-09-17T10:00:00Z" {
+		t.Errorf("the found image and its age: %+v", f)
+	}
+	if f.DeclaredImage != pinned || f.DeclaredImageCreated != "2026-08-20T10:00:00Z" || !f.Downgrade {
+		t.Errorf("the declared image, its age, and that it is a downgrade: %+v", f)
+	}
+	if got := f.Networks["predecessor_default"]; len(got) != 2 || got[0] != "db" || got[1] != "office" {
+		t.Errorf("the neighbours on the found network, without the container itself: %v", f.Networks)
+	}
+	if len(f.Mounts) != 1 || f.Mounts[0] != "/srv/web:/data" || len(f.Ports) != 1 || f.Ports[0] != "80/tcp>0.0.0.0:8080" {
+		t.Errorf("mounts and ports as found: %+v", f)
+	}
+	// And the held file carries how the declared content differs from what was found.
+	p, ok := state.HeldAt("hello-web.page")
+	if !ok || p.Facts == nil || !p.Facts.Differs {
+		t.Fatalf("a held file that differs from the declared content does not say so: %+v", p)
+	}
+	joined := strings.Join(p.Facts.Difference, "\n")
+	if !strings.Contains(joined, "- the predecessor's page") || !strings.Contains(joined, "+ the mesh's page") {
+		t.Errorf("the difference does not show what is lost and what is new: %q", joined)
+	}
+	_ = os.Remove(filepath.Join(dir, "unused"))
+}
+
+// A declared image not yet on the machine leaves its age unknown and the comparison undecided.
+func TestAnImageNotYetPulledLeavesTheDowngradeUndecided(t *testing.T) {
+	dir, page, m := predecessor(t)
+	m.containers["hello-web"].image = "web:1.27"
+	m.containers["hello-web"].imageID = "sha256:found"
+	m.images = map[string]string{"sha256:found": "2026-09-17T10:00:00Z"}
+	_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
+	h, _ := state.HeldAt("hello-web.server")
+	if h.Facts == nil || h.Facts.DeclaredImageCreated != "" || h.Facts.Downgrade {
+		t.Fatalf("an unknown declared age decided a downgrade: %+v", h.Facts)
+	}
+}
+
+func TestTheDifferenceIsWhatIsLostAndWhatIsNew(t *testing.T) {
+	differs, lines := differenceOf("a\nprivate scope: local\nb\n", "a\nb\nupstream: public\n")
+	if !differs || len(lines) != 2 || lines[0] != "- private scope: local" || lines[1] != "+ upstream: public" {
+		t.Fatalf("got %v %v", differs, lines)
+	}
+	if differs, lines := differenceOf("same\n", "same\n"); differs || lines != nil {
+		t.Fatalf("identical content differs: %v %v", differs, lines)
+	}
+}
+
+// What runs on the machine that the mesh neither wrote nor holds is reported (ADR 0163).
+func TestStraysAreWhatRunsHereThatNobodyAsked(t *testing.T) {
+	m := &machine{containers: map[string]*fakeContainer{
+		"hello-web":  {id: "ours", running: true, image: "web:1"},
+		"gitea-old":  {id: "left-behind", running: true, image: "gitea:1.22"},
+		"held-thing": {id: "found", running: true, image: "x:1"},
+	}}
+	known := store.State{
+		Resources: []store.Applied{{ID: "hello-web.server", Type: "container", Target: "hello-web"}},
+		Held:      []store.Held{{ID: "other.server", Kind: "container", Target: "held-thing"}},
+	}
+	strays, err := Strays(context.Background(), m.run, known)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(strays) != 1 || strays[0].Name != "gitea-old" || !strings.Contains(strays[0].Detail, "gitea:1.22") {
+		t.Fatalf("strays: %+v", strays)
+	}
+}

internal/apply/hold.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"syscall"
 	"time"
@@ -433,6 +434,32 @@ type foundContainer struct {
 	id      string
 	running bool
 	spec    string
+	// What a take compares (novox/hq ADR 0163): the image and its id, the networks the container
+	// is on, its mounts and its published ports — empty from a runtime (or a test's fake) that
+	// answers the short form.
+	image    string
+	imageID  string
+	networks []string
+	mounts   []string
+	ports    []string
+}
+
+// foundFormat is what inspectFound asks the runtime for, tab-separated: the three a hold has
+// always needed, then the facts a take compares.
+const foundFormat = "{{.Id}}\t{{.State.Running}}\t{{index .Config.Labels \"" + specLabel + "\"}}" +
+	"\t{{.Config.Image}}\t{{.Image}}" +
+	"\t{{range $k, $v := .NetworkSettings.Networks}}{{$k}},{{end}}" +
+	"\t{{range .Mounts}}{{.Source}}:{{.Destination}},{{end}}" +
+	"\t{{range $p, $b := .NetworkSettings.Ports}}{{$p}}{{range $b}}>{{.HostIp}}:{{.HostPort}}{{end}},{{end}}"
+
+func splitList(s string) []string {
+	var out []string
+	for _, part := range strings.Split(s, ",") {
+		if part = strings.TrimSpace(part); part != "" {
+			out = append(out, part)
+		}
+	}
+	return out
 }
 
 // inspectFound reads a container by name the way a hold needs it: its id, whether it runs, and
@@ -451,8 +478,7 @@ func inspectFound(ctx context.Context, name string, run Runner) (foundContainer,
 	if err != nil {
 		return foundContainer{}, false, fmt.Errorf("%w, so nothing can be said about %q", err, name)
 	}
-	out, err := run(ctx, cri, "container", "inspect", "--format",
-		"{{.Id}}\t{{.State.Running}}\t{{index .Config.Labels \""+specLabel+"\"}}", name)
+	out, err := run(ctx, cri, "container", "inspect", "--format", foundFormat, name)
 	if err != nil {
 		if absent(err) {
 			return foundContainer{}, false, nil
@@ -466,14 +492,100 @@ func inspectFound(ctx context.Context, name string, run Runner) (foundContainer,
 			name, err)
 	}
 	parts := strings.Split(strings.TrimSpace(out), "\t")
-	for len(parts) < 3 {
+	for len(parts) < 8 {
 		parts = append(parts, "")
 	}
 	spec := strings.TrimSpace(parts[2])
 	if spec == "<no value>" {
 		spec = ""
 	}
-	return foundContainer{id: strings.TrimSpace(parts[0]), running: parts[1] == "true", spec: spec}, true, nil
+	return foundContainer{id: strings.TrimSpace(parts[0]), running: parts[1] == "true", spec: spec,
+		image: strings.TrimSpace(parts[3]), imageID: strings.TrimSpace(parts[4]),
+		networks: splitList(parts[5]), mounts: splitList(parts[6]), ports: splitList(parts[7])}, true, nil
+}
+
+// factsOf is what a take would compare for a found container (novox/hq ADR 0163): the found
+// image and when it was made, the networks and who else is on them, mounts and ports — beside
+// what the module declares, and the declared image's date when that image is on the machine.
+// Every question the runtime cannot answer leaves its fact empty; a preview says so rather than
+// guesses.
+func factsOf(ctx context.Context, seen foundContainer, res *declaration.Container, run Runner) *Facts {
+	cri, err := containerRuntime(ctx, run)
+	if err != nil {
+		return nil
+	}
+	f := &store.Facts{Image: seen.image, Mounts: seen.mounts, Ports: seen.ports,
+		DeclaredImage: res.Image, DeclaredPorts: res.Ports, DeclaredVolumes: res.Volumes}
+	if seen.imageID != "" {
+		if out, err := run(ctx, cri, "image", "inspect", "--format", "{{.Created}}", seen.imageID); err == nil {
+			f.ImageCreated = strings.TrimSpace(out)
+		}
+	}
+	if res.Image != "" {
+		if out, err := run(ctx, cri, "image", "inspect", "--format", "{{.Created}}", res.Image); err == nil {
+			f.DeclaredImageCreated = strings.TrimSpace(out)
+		}
+	}
+	if found, err := time.Parse(time.RFC3339Nano, f.ImageCreated); err == nil {
+		if declared, err := time.Parse(time.RFC3339Nano, f.DeclaredImageCreated); err == nil {
+			f.Downgrade = declared.Before(found)
+		}
+	}
+	for _, network := range seen.networks {
+		if f.Networks == nil {
+			f.Networks = map[string][]string{}
+		}
+		var members []string
+		if out, err := run(ctx, cri, "network", "inspect", "--format",
+			"{{range .Containers}}{{.Name}},{{end}}", network); err == nil {
+			for _, m := range splitList(out) {
+				if m != res.Name {
+					members = append(members, m)
+				}
+			}
+		}
+		sort.Strings(members)
+		f.Networks[network] = members
+	}
+	return (*Facts)(f)
+}
+
+// Facts is store.Facts, named here so hold's callers read as one vocabulary.
+type Facts = store.Facts
+
+// differenceOf is how a found file differs from the declared content: the lines only the found
+// file has, marked -, then the lines only the declared content has, marked +, in their own order,
+// bounded so a report stays a report. Not a diff tool's output: the question a take answers is
+// "what would be lost and what would be new", and that is these two lists.
+func differenceOf(found, declared string) (bool, []string) {
+	if found == declared {
+		return false, nil
+	}
+	const bound = 40
+	count := func(s string) map[string]int {
+		out := map[string]int{}
+		for _, line := range strings.Split(s, "\n") {
+			out[line]++
+		}
+		return out
+	}
+	inFound, inDeclared := count(found), count(declared)
+	var out []string
+	add := func(mark, s string, other map[string]int) {
+		seen := map[string]int{}
+		for _, line := range strings.Split(s, "\n") {
+			seen[line]++
+			if seen[line] > other[line] && len(out) < bound {
+				out = append(out, mark+" "+line)
+			}
+		}
+	}
+	add("-", found, inDeclared)
+	add("+", declared, inFound)
+	if len(out) >= bound {
+		out = append(out, "… and more")
+	}
+	return true, out
 }
 
 // absent is whether a runtime said the thing is not there, rather than failing to answer. Its own
@@ -540,6 +652,12 @@ func hold(ctx context.Context, sys system.System, r declaration.Resource, module
 			} else if digestOf(string(content)) != h.Digest {
 				changed = "rewritten"
 			}
+			// What a take would replace it with, and how that differs (novox/hq ADR 0163): a
+			// file declared whole is compared whole; one written into is not replaced at all.
+			if res.Into == "" {
+				differs, lines := differenceOf(string(content), res.Content)
+				h.Facts = &Facts{Differs: differs, Difference: lines}
+			}
 		}
 	case *declaration.Directory:
 		info, err := os.Lstat(res.Path)
@@ -628,6 +746,9 @@ func hold(ctx context.Context, sys system.System, r declaration.Resource, module
 		case h.Running && !seen.running:
 			changed = "stopped"
 		}
+		if exists {
+			h.Facts = factsOf(ctx, seen, res, run)
+		}
 	default:
 		return out, h, fmt.Errorf("a %s cannot be held", r.Kind())
 	}

internal/apply/hold_test.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"testing"
 
@@ -18,7 +19,11 @@ import (
 // label when a host made it. Every command it is asked is written down.
 type machine struct {
 	containers map[string]*fakeContainer
-	asked      []string
+	// images is what `image inspect --format {{.Created}}` answers per image or id; members is
+	// what `network inspect` lists per network (ADR 0163).
+	images  map[string]string
+	members map[string][]string
+	asked   []string
 	// wgUp is what `wg show interfaces` answers: the tunnels up on the machine.
 	wgUp string
 	// handshakes is what `wg show <interface> latest-handshakes` answers, and handshakesFail the
@@ -97,6 +102,9 @@ type fakeContainer struct {
 	id      string
 	running bool
 	spec    string
+	// What a take compares (ADR 0163), answered in the long inspect form when set.
+	image, imageID          string
+	networks, mounts, ports []string
 }
 
 func (m *machine) run(_ context.Context, name string, args ...string) (string, error) {
@@ -140,9 +148,38 @@ func (m *machine) run(_ context.Context, name string, args ...string) (string, e
 			running = "true"
 		}
 		if strings.HasPrefix(args[3], "{{.Id}}") {
-			return c.id + "\t" + running + "\t" + c.spec + "\n", nil
+			line := c.id + "\t" + running + "\t" + c.spec
+			if c.image != "" {
+				line += "\t" + c.image + "\t" + c.imageID + "\t" + strings.Join(c.networks, ",") + "," +
+					"\t" + strings.Join(c.mounts, ",") + "," + "\t" + strings.Join(c.ports, ",") + ","
+			}
+			return line + "\n", nil
 		}
 		return running + "\t" + c.spec + "\n", nil
+	case "image":
+		if len(args) > 1 && args[1] == "inspect" {
+			if created, ok := m.images[args[len(args)-1]]; ok {
+				return created + "\n", nil
+			}
+			return "", errors.New("no such image")
+		}
+		return "", nil
+	case "network":
+		if len(args) > 1 && args[1] == "inspect" {
+			return strings.Join(m.members[args[len(args)-1]], ",") + ",\n", nil
+		}
+		return "", nil
+	case "ps":
+		var lines []string
+		for name, c := range m.containers {
+			state := "exited"
+			if c.running {
+				state = "running"
+			}
+			lines = append(lines, name+"\t"+c.image+"\t"+state)
+		}
+		sort.Strings(lines)
+		return strings.Join(lines, "\n") + "\n", nil
 	case "rm":
 		delete(m.containers, args[len(args)-1])
 		return "", nil

internal/apply/left_out_test.go

@@ -0,0 +1,136 @@
+package apply
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/declaration"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// A taken container keeps a found network by a per-machine setting (novox/hq ADR 0163, rule 4):
+// joined once it runs, part of its spec, and refused when it cannot be joined.
+func TestAContainerJoinsTheNetworksItKeeps(t *testing.T) {
+	var ran []string
+	connectFails := false
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		if name != "docker" {
+			return "", errors.New("not installed")
+		}
+		ran = append(ran, strings.Join(args, " "))
+		switch args[0] {
+		case "info":
+			return "29.0.0\n", nil
+		case "container":
+			if len(ran) > 2 {
+				return "true\t" + specOfLast, nil
+			}
+			return "false\t\n", errors.New("no such container")
+		case "run":
+			return "deadbeef\n", nil
+		case "network":
+			if connectFails {
+				return "", errors.New("network predecessor_default not found")
+			}
+		}
+		return "", nil
+	}
+	d := parseTrusted(t, `{"declaration":1,"resources":[
+	  {"id":"app","type":"container","name":"app","image":"`+pinned+`",
+	   "networks":["predecessor_default"]}
+	]}`)
+	specOfLast = containerSpec(d.Resources[0].(*declaration.Container), inputs{})
+	alone := *d.Resources[0].(*declaration.Container)
+	alone.Networks = nil
+	if specOfLast == containerSpec(&alone, inputs{}) {
+		t.Fatal("the kept network is not part of the container's spec: kept or let go, the container would be left alone")
+	}
+	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	joined := false
+	for i, line := range ran {
+		if line == "network connect predecessor_default app" {
+			joined = true
+			if ran[i-1] != "container inspect --format {{.State.Running}}\t{{index .Config.Labels \""+specLabel+"\"}} app" &&
+				!strings.HasPrefix(ran[i-1], "container inspect") {
+				t.Errorf("joined before the container was read back as running: %v", ran)
+			}
+		}
+	}
+	if !joined || report.Outcomes[0].Action != "created" {
+		t.Fatalf("the container did not join the kept network: %v\n%+v", ran, report.Outcomes)
+	}
+
+	connectFails, ran = true, nil
+	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil); err == nil ||
+		!strings.Contains(err.Error(), "could not join the kept network predecessor_default") {
+		t.Fatalf("a network that cannot be joined was passed over: %v", err)
+	}
+}
+
+var specOfLast string
+
+// A module the mesh left out of a declaration is not a module the mesh removed (novox/hq ADR 0163,
+// rule 6): what the host wrote for it stays, recorded and said; what it holds for it stays held.
+// A module simply absent is removed as it always was.
+func TestALeftOutModuleIsNeitherRemovedNorForgotten(t *testing.T) {
+	var removed []string
+	gone := map[string]bool{}
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		if name != "docker" {
+			return "", nil
+		}
+		switch args[0] {
+		case "info":
+			return "29.0.0\n", nil
+		case "rm":
+			removed = append(removed, args[len(args)-1])
+			gone[args[len(args)-1]] = true
+		case "container":
+			if gone[args[len(args)-1]] {
+				return "", errors.New("no such container")
+			}
+			return "true\tspec", nil
+		}
+		return "", nil
+	}
+	known := store.State{
+		Resources: []store.Applied{
+			{ID: "web.server", Type: "container", Target: "web", Origin: store.OriginDeclared},
+			{ID: "old.server", Type: "container", Target: "old", Origin: store.OriginDeclared},
+		},
+		Held: []store.Held{{ID: "web.page", Module: "web", Kind: "file", Target: "/srv/web/index.html"}},
+	}
+	d := parse(t, `{"declaration":1,"left_out":["web"],"resources":[
+	  {"id":"notes.conf","type":"file","path":"`+t.TempDir()+`/notes.conf","content":"x"}
+	]}`)
+	report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(removed) != 1 || removed[0] != "old" {
+		t.Fatalf("removed %v; only the module that is absent goes", removed)
+	}
+	if _, kept := state.At("container", "web"); !kept {
+		t.Fatal("the left-out module's record was forgotten")
+	}
+	if _, held := state.HeldAt("web.page"); !held {
+		t.Fatal("the left-out module's hold was released")
+	}
+	said := false
+	for _, o := range report.Outcomes {
+		if o.ID == "web.server" && o.Action == "unchanged" && strings.Contains(o.Detail, "web was left out of this declaration by the mesh") {
+			said = true
+		}
+		if o.ID == "web.server" && o.Action != "unchanged" {
+			t.Errorf("the left-out module's container was %s", o.Action)
+		}
+	}
+	if !said {
+		t.Fatalf("keeping the left-out module's container was not said: %+v", report.Outcomes)
+	}
+}

internal/apply/mesh_names_test.go

@@ -0,0 +1,54 @@
+package apply
+
+import (
+	"testing"
+
+	"github.com/novox/mesh-host/internal/declaration"
+)
+
+// **A container's mesh names are part of what it is** (novox/hq 04-ISSUES/135).
+//
+// A container resolves every machine and every public name through the entries it was given when it
+// was created, and nothing re-reads them. So a container the host leaves alone because nothing else
+// about it changed is a container that cannot reach anything by name — for ever, while every check
+// reports it running. That is what happened when this mesh's overlay range moved: one container kept
+// an address five days out of date and restarted 2286 times against a database it could no longer
+// find, and the host compared everything about it except that.
+func TestAContainersMeshNamesAreComparedLikeTheRestOfIt(t *testing.T) {
+	was := &declaration.Container{
+		Name: "umami", Image: "ghcr.io/example/umami@sha256:" + zeros(64),
+		Hosts: []string{"novox.internal:10.42.0.1", "umami.novox.be:10.42.0.1"},
+	}
+	moved := &declaration.Container{
+		Name: was.Name, Image: was.Image,
+		Hosts: []string{"novox.internal:10.10.0.1", "umami.novox.be:10.10.0.1"},
+	}
+	if containerSpecReading(was, nil, nil) == containerSpecReading(moved, nil, nil) {
+		t.Fatal("a container whose mesh names moved compares equal, so it is never recreated")
+	}
+
+	// And the order they arrive in is not a change: the digest must not move for a reordering
+	// nobody made.
+	reordered := &declaration.Container{
+		Name: moved.Name, Image: moved.Image,
+		Hosts: []string{moved.Hosts[1], moved.Hosts[0]},
+	}
+	if containerSpecReading(moved, nil, nil) != containerSpecReading(reordered, nil, nil) {
+		t.Fatal("the same names in another order read as a different container")
+	}
+
+	// A container the mesh gives no names is unaffected, so nothing is recreated for a field it
+	// does not set.
+	plain := &declaration.Container{Name: "plex", Image: was.Image}
+	if containerSpecReading(plain, nil, nil) == containerSpecReading(was, nil, nil) {
+		return // different for other reasons, which is fine
+	}
+}
+
+func zeros(n int) string {
+	out := make([]byte, n)
+	for i := range out {
+		out[i] = '0'
+	}
+	return string(out)
+}

internal/apply/runonce_test.go

@@ -316,3 +316,85 @@ func TestAContainerNamingARunOnceStepIsRecreatedWhenItRan(t *testing.T) {
 		t.Errorf("the recreation did not name the step as its reason: %+v", server)
 	}
 }
+
+func TestAFailedStepGatesItsModuleAndNotTheMachine(t *testing.T) {
+	// **The blast radius of a step is its module** (novox/hq ADR 0136). A step exists to make
+	// something true before the next thing in its own module needs it — a store seeded before the
+	// broker starts, a schema prepared before the version that needs it runs. Stopping the whole
+	// apply is what this host's own loop calls holding a machine hostage, and it was already
+	// rejected for every other shape (04-ISSUES/011): a module whose database is briefly
+	// unreachable must not stop every module declared after it.
+	var startedNames []string
+	run := func(ctx context.Context, name string, args ...string) (string, error) {
+		switch args[0] {
+		case "info":
+			return "27.0\n", nil
+		case "container":
+			return "false\t\n", errors.New("no such container")
+		case "run":
+			startedNames = append(startedNames, nameOf(args))
+			if nameOf(args) == "catalogue-prepare" {
+				return "", errors.New("exit status 1") // the schema could not be reached
+			}
+			return "deadbeef\n", nil
+		case "rm":
+			return "", nil
+		}
+		return "", nil
+	}
+	d := parseTrusted(t, `{"declaration":1,"resources":[
+	  {"id":"mesh-catalog.runtime-prepare","type":"container","name":"catalogue-prepare","image":"`+pinned+`","run-once":true},
+	  {"id":"mesh-catalog.runtime","type":"container","name":"catalogue","image":"`+pinned+`"},
+	  {"id":"gitea.server","type":"container","name":"forge","image":"`+pinned+`"}
+	]}`)
+
+	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
+	if err == nil {
+		t.Fatal("a failed step was not reported as a failure")
+	}
+	started := map[string]bool{}
+	for _, n := range startedNames {
+		started[n] = true
+	}
+	if started["catalogue"] {
+		t.Error("the module's own workload ran although its step did not complete")
+	}
+	if !started["forge"] {
+		t.Error("another module was not attempted, so one module's step held the machine hostage")
+	}
+	// And the machine's own account says which was not attempted, rather than leaving it to be
+	// inferred from silence.
+	var skipped string
+	for _, o := range report.Outcomes {
+		if o.Action == "skipped" {
+			skipped = o.ID
+		}
+	}
+	if skipped != "mesh-catalog.runtime" {
+		t.Errorf("the report does not say what was not attempted: %q", skipped)
+	}
+}
+
+func TestAResourcesOwnerIsReadToTheLastDot(t *testing.T) {
+	// **A module's name may contain a dot.** `novox.be` is one on this mesh, so reading a resource's
+	// owner to the first dot would make its resources belong to something called "novox" — and a gate
+	// would skip whatever else happened to start that way. A resource's own id never contains one,
+	// which is what makes the last dot the boundary.
+	for identity, want := range map[string]string{
+		"novox.be.server":              "novox.be",
+		"mesh-catalog.runtime-prepare": "mesh-catalog",
+		"gitea.admin-bootstrap":        "gitea",
+	} {
+		got, ours := moduleOf(identity)
+		if !ours || got != want {
+			t.Errorf("%q belongs to %q (%v), want %q", identity, got, ours, want)
+		}
+	}
+	// What the mesh declares in its own right belongs to no module: the foundation's resources carry
+	// no dot, and the adoption's are the mesh's.
+	for _, identity := range []string{"container-runtime", "store-ready", "adoption.guard", ".server"} {
+		if _, ours := moduleOf(identity); ours {
+			t.Errorf("%q was read as a module's", identity)
+		}
+	}
+}

internal/apply/strays.go

@@ -0,0 +1,54 @@
+package apply
+
+import (
+	"context"
+	"sort"
+	"strings"
+
+	"github.com/novox/mesh-host/internal/declaration"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// Strays is what runs on the machine that the mesh neither wrote nor holds (novox/hq ADR 0163):
+// every container the runtime has that no record names and no hold names. The question nothing
+// answered on 2026-09-23, when a renamed resource left its old container running for a day; asked
+// on every apply now, and reported, so a thing left behind is seen the day it is left.
+//
+// Containers only, today. A listener nobody declared is harder to attribute to a thing, and the
+// machine's own services are not strays; that account is issue 160's.
+func Strays(ctx context.Context, run Runner, known store.State) ([]store.Stray, error) {
+	cri, err := containerRuntime(ctx, run)
+	if err != nil {
+		return nil, nil // a machine with no runtime has no containers to stray
+	}
+	out, err := run(ctx, cri, "ps", "-a", "--format", "{{.Names}}\t{{.Image}}\t{{.State}}")
+	if err != nil {
+		return nil, err
+	}
+	ours := map[string]bool{}
+	for _, r := range known.Resources {
+		if declaration.Type(r.Type) == declaration.TypeContainer {
+			ours[r.Target] = true
+		}
+	}
+	for _, h := range known.Held {
+		if h.Kind == string(declaration.TypeContainer) {
+			ours[h.Target] = true
+		}
+	}
+	var strays []store.Stray
+	for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
+		parts := strings.Split(line, "\t")
+		name := strings.TrimSpace(parts[0])
+		if name == "" || ours[name] {
+			continue
+		}
+		detail := ""
+		if len(parts) > 2 {
+			detail = strings.TrimSpace(parts[1]) + ", " + strings.TrimSpace(parts[2])
+		}
+		strays = append(strays, store.Stray{Kind: string(declaration.TypeContainer), Name: name, Detail: detail})
+	}
+	sort.Slice(strays, func(i, j int) bool { return strays[i].Name < strays[j].Name })
+	return strays, nil
+}

internal/bootstrap/genesis_forge_test.go

@@ -0,0 +1,79 @@
+package bootstrap
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+// What genesis raises, it raises as the module that succeeds it declares — name, data directory
+// and image — so the module adopts it by the found rule that already exists (novox/hq ADR 0163,
+// rule 7; issue 090). The network is the one difference left: the bootstrap forge runs on the
+// machine's network to reach the store on its loopback, and a take says so.
+func TestGenesisRaisesTheForgeAsTheModuleDeclaresIt(t *testing.T) {
+	var ran [][]string
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		if name == "docker" && args[0] == "container" {
+			return "", nil // not raised yet
+		}
+		ran = append(ran, append([]string{name}, args...))
+		return "", nil
+	}
+	if err := raiseGiteaServer(context.Background(), run, time.Second, "pw", DefaultPorts(), quietly); err != nil {
+		t.Fatal(err)
+	}
+	var raised []string
+	for _, r := range ran {
+		if r[0] == "docker" && r[1] == "run" {
+			raised = r
+		}
+	}
+	line := strings.Join(raised, " ")
+	for _, want := range []string{"--name gitea ", "--volume " + giteaDataDir + ":/data", " " + giteaImage} {
+		if !strings.Contains(line+" ", want) {
+			t.Errorf("the forge is not raised with %q: %s", want, line)
+		}
+	}
+	if giteaBootstrap != ForgeModule {
+		t.Errorf("the bootstrap forge is %q and the module names its container %q", giteaBootstrap, ForgeModule)
+	}
+
+	// Against the module's own manifest, where the catalogue is checked out beside this repository.
+	var manifest []byte
+	for _, candidate := range []string{"../../../mesh-catalog/modules/gitea/module.json", "../../../../../mesh-catalog/modules/gitea/module.json"} {
+		if raw, err := os.ReadFile(filepath.Clean(candidate)); err == nil {
+			manifest = raw
+			break
+		}
+	}
+	if manifest == nil {
+		t.Skip("the catalogue is not beside this checkout; the module's pin is not compared")
+	}
+	var m struct {
+		Resources []struct {
+			ID, Type, Name, Image string
+			Volumes               []string
+		} `json:"resources"`
+	}
+	if err := json.Unmarshal(manifest, &m); err != nil {
+		t.Fatal(err)
+	}
+	for _, r := range m.Resources {
+		if r.Type != "container" || r.ID != "server" {
+			continue
+		}
+		if r.Name != giteaBootstrap {
+			t.Errorf("the module names its container %q; genesis raises %q", r.Name, giteaBootstrap)
+		}
+		if r.Image != giteaImage {
+			t.Errorf("the module pins %s; genesis raises %s — the two must move together", r.Image, giteaImage)
+		}
+		if len(r.Volumes) != 1 || !strings.HasSuffix(r.Volumes[0], ":/data") {
+			t.Errorf("the module mounts %v; genesis mounts %s:/data", r.Volumes, giteaDataDir)
+		}
+	}
+}

internal/bootstrap/inuse_test.go

@@ -91,7 +91,7 @@ func TestARerunOfGenesisIsNotAMachineInUse(t *testing.T) {
 	if err := store.Save(o.State, store.State{Resources: []store.Applied{{ID: "store", Type: "container", Target: "mesh-store"}}}); err != nil {
 		t.Fatal(err)
 	}
-	m := inUseRunner{ps: "mesh-gitea-server\t\n", ss: servingSockets}
+	m := inUseRunner{ps: giteaBootstrap + "\t\n", ss: servingSockets}
 	if err := RefuseAMachineInUse(context.Background(), o, m.run, quietly); err != nil {
 		t.Errorf("what an earlier genesis raised was counted as a machine in use: %v", err)
 	}

internal/bootstrap/phase_packages.go

@@ -25,11 +25,24 @@ const (
 	// foundationStore is the foundation's postgres container — the mesh's own memory, raised from the
 	// bundle. gitea's bootstrap database lives here too, so a mesh runs one postgres (issue 051).
 	foundationStore = "mesh-store"
-	// giteaBootstrap is the gitea server raised directly at genesis, before gitea is a module.
-	giteaBootstrap = "mesh-gitea-server"
-	// giteaImage is the same upstream image the gitea module runs, pinned identically so the module
-	// adopts the running server rather than replacing it.
-	giteaImage = "gitea/gitea@sha256:dfc61e347c8b582df918f4556401bf2cecdfbdb56c5282ae9488dd76fca3e41c"
+	// giteaBootstrap is the gitea server raised directly at genesis, before gitea is a module —
+	// under the name the gitea MODULE declares for its container, so the module finds it and holds
+	// it rather than raising a second forge beside it (novox/hq ADR 0163, rule 7; issue 090).
+	giteaBootstrap = "gitea"
+	// giteaImage is the image the gitea module declares for that container, pinned to the same
+	// digest, so taking the module over is not a downgrade and not an upgrade. **Moves with the
+	// module's pin**: the two are compared by a take, and a difference is said there — but a
+	// genesis that raised an older image than the module declares would be taken over as an
+	// upgrade on first push, which a forge holding the mesh's packages must not have done to it
+	// unannounced. Checked in TestGenesisRaisesTheForgeAsTheModuleDeclaresIt against the module's
+	// manifest where the catalogue is beside this checkout.
+	giteaImage = "gitea/gitea@sha256:87a67ee09d3ae0d1df5fda5dcda3e2a1f9236a45b0a59025d6e00e46adc43bef"
+	// giteaDataDir is where the module's `data` directory resolves on a machine with the default
+	// layout (<data root>/<module>/<id>, novox/hq ADR 0112): mounted at /data as the module mounts
+	// it, so the repositories, attachments and indexes the bootstrap forge accumulates are the
+	// module's the day it is taken — before this, the forge had no volume and its data was the
+	// container's, lost with it.
+	giteaDataDir = "/var/lib/gitea/data"
 	// packagesOrg is the npm owner: every module consumes `@novox/*` from this gitea org.
 	packagesOrg = "novox"
 	// packagesTeam is the org team whose members may read and write the org's packages.
@@ -262,9 +275,13 @@ func raiseGiteaServer(ctx context.Context, run Runner, timeout time.Duration, db
 		"run", "-d", "--name", giteaBootstrap,
 		// Host network, like the control plane: it reaches the foundation store on the machine's
 		// loopback (where the store publishes 5432) and answers on the machine's own 3000, which is
-		// where mesh-bootstrap and the builder's build containers look for it.
+		// where mesh-bootstrap and the builder's build containers look for it. The module runs
+		// bridged and publishes its ports; that is the one difference a take still has to say
+		// (ADR 0163, rule 7) — the data, the name and the image are the module's already.
 		"--network", "host",
 		"--restart", "unless-stopped",
+		// The module's data directory, so what the forge accumulates is the module's when taken.
+		"--volume", giteaDataDir + ":/data",
 	}, env...)
 	args = append(args, giteaImage)
 

internal/declaration/declaration.go

@@ -940,6 +940,13 @@ type Container struct {
 	// its siblings can name before any of them can resolve anything.
 	Dns []string `json:"dns,omitempty"`
 
+	// Networks are networks this container also joins once created, by name — a found network a
+	// per-machine setting keeps for a taken container (novox/hq ADR 0163, rule 4), so a
+	// neighbour that resolves it there keeps resolving it until the neighbour is taken too.
+	// Joined after creation, because a runtime starts a container on one network; part of the
+	// container's spec, so a network kept or let go recreates it.
+	Networks []string `json:"networks,omitempty"`
+
 	// IP is this container's address on its network, passed to the runtime unchanged.
 	//
 	// Only meaningful on a user-defined network, and refused by the runtime elsewhere. Exists for
@@ -1032,6 +1039,16 @@ func (c *Container) validate(where string, _ bool) []string {
 				"static address anywhere but a user-defined one")
 		}
 	}
+	for _, n := range c.Networks {
+		problems = append(problems, (&Network{Name: n}).validate(where+": networks", false)...)
+		if n == c.Network {
+			problems = append(problems, where+": networks names "+n+", which is already the container's network")
+		}
+	}
+	if len(c.Networks) > 0 && (c.RunOnce || c.Schedule != "") {
+		problems = append(problems, where+": networks is for a container that keeps running; a step "+
+			"runs and exits, and joins nothing afterwards")
+	}
 	return append(problems, checkImage(where, c.Image)...)
 }
 
@@ -1137,6 +1154,41 @@ type Declaration struct {
 	// converged node — which is every node the mesh raised before adoption existed, and so the
 	// only form an older controller ever sends (novox/hq ADR 0100).
 	Adoption *Adoption
+
+	// Sequence orders this declaration against every other the mesh has sent this node: each
+	// send is one higher than the last, assigned under the control plane's hold on the node
+	// (novox/hq 04-ISSUES/107). Zero is a declaration that carries no order — every one an older
+	// controller sent, and the bundle genesis applies — and a host makes no ordering claim about
+	// one of those.
+	//
+	// **The one property a declaration needs that its signature does not give it.** A signature
+	// says the mesh sent this; it cannot say the mesh sent it AFTER the one the host is holding.
+	// Before this, "older" was inferred from arrival within a batch and a 750ms window, and a
+	// backlog longer than the batch, or a slow broker, applied a declaration the mesh had already
+	// superseded.
+	Sequence int64
+
+	// LeftOut names the modules of this machine's set the mesh left out of this declaration,
+	// because a setting stored for one cannot compose with its definition (novox/hq ADR 0163,
+	// rule 6). A machine is told everything or nothing about what it IS told; this is what it is
+	// not told, said. The host keeps what it holds for a left-out module and touches none of
+	// what it wrote for it — its resources are absent from the declaration, and absence would
+	// otherwise read as removal.
+	LeftOut []string
+}
+
+// LeftOutModuleOf says which left-out module a recorded resource belongs to, if any: its id is the
+// module's name, a dot, and the module's own id for it. A module's name may contain a dot, so the
+// longest left-out name that prefixes the id wins; a false match keeps a thing an apply would
+// otherwise remove, which is the conservative mistake.
+func (d *Declaration) LeftOutModuleOf(id string) (string, bool) {
+	best := ""
+	for _, m := range d.LeftOut {
+		if strings.HasPrefix(id, m+".") && len(m) > len(best) {
+			best = m
+		}
+	}
+	return best, best != ""
 }
 
 // Adoption is a node's mode, as the controller records it: the node is adopted, and these are
@@ -1274,6 +1326,11 @@ type envelope struct {
 	// a bug quietly strip a machine.
 	OwnsNothing bool              `json:"owns_nothing,omitempty"`
 	Resources   []json.RawMessage `json:"resources"`
+	// Sequence is optional on the wire, so a controller that does not send one is still
+	// understood: absent reads as zero, which is "no ordering claimed" rather than "first".
+	Sequence int64 `json:"sequence,omitempty"`
+	// LeftOut is optional on the wire too, and absent when nothing was left out (ADR 0163).
+	LeftOut []string `json:"left_out,omitempty"`
 }
 
 func parse(raw []byte, allowActions bool) (*Declaration, error) {
@@ -1290,8 +1347,20 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
 			env.Version, Version)}}
 	}
 
-	d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption}
+	d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption, Sequence: env.Sequence,
+		LeftOut: env.LeftOut}
 	var problems []string
+	if len(env.LeftOut) > 0 && allowActions {
+		// The bundle is carried with the binary and leaves nothing out: which module a setting
+		// stopped composing for is the mesh's record (ADR 0163).
+		problems = append(problems, "a carried bundle says modules were left out, and only the "+
+			"mesh can say that")
+	}
+	for _, m := range env.LeftOut {
+		if strings.TrimSpace(m) == "" {
+			problems = append(problems, "left_out names a module with no name")
+		}
+	}
 
 	if len(env.Resources) == 0 && !env.OwnsNothing {
 		problems = append(problems, "no resources. An empty declaration is a mistake, not a "+

internal/declaration/declaration_test.go

@@ -464,3 +464,43 @@ func TestAnExplicitlyEmptyDeclarationIsAccepted(t *testing.T) {
 		t.Fatalf("an unmarked empty declaration must still be refused; got %v", err)
 	}
 }
+
+// A container's kept networks are names, not its own network, and not for a step (novox/hq ADR
+// 0163, rule 4); and the mesh may say which modules it left out, which a carried bundle may not.
+func TestKeptNetworksAndLeftOutModulesAreReadStrictly(t *testing.T) {
+	pinnedImage := "postgres@sha256:" + strings.Repeat("a", 64)
+	d, err := Parse([]byte(`{"declaration":1,"left_out":["web"],"resources":[
+	  {"id":"app","type":"container","name":"app","image":"` + pinnedImage + `","networks":["predecessor_default"]}
+	]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := d.Resources[0].(*Container).Networks; len(got) != 1 || got[0] != "predecessor_default" {
+		t.Fatalf("the kept network was not read: %v", got)
+	}
+	if m, left := d.LeftOutModuleOf("web.server"); !left || m != "web" {
+		t.Fatalf("web.server is not web's: %q %v", m, left)
+	}
+	if _, left := d.LeftOutModuleOf("webapp.server"); left {
+		t.Fatal("webapp.server was taken for web's")
+	}
+	for name, raw := range map[string]string{
+		"a bad network name": `{"declaration":1,"resources":[
+		  {"id":"app","type":"container","name":"app","image":"` + pinnedImage + `","networks":["a/b"]}]}`,
+		"its own network": `{"declaration":1,"resources":[
+		  {"id":"app","type":"container","name":"app","image":"` + pinnedImage + `","network":"own","networks":["own"]}]}`,
+		"a step": `{"declaration":1,"resources":[
+		  {"id":"app","type":"container","name":"app","image":"` + pinnedImage + `","run-once":true,"networks":["x"]}]}`,
+		"a nameless module": `{"declaration":1,"left_out":[""],"resources":[
+		  {"id":"app","type":"container","name":"app","image":"` + pinnedImage + `"}]}`,
+	} {
+		if _, err := Parse([]byte(raw)); err == nil {
+			t.Errorf("%s was accepted", name)
+		}
+	}
+	if _, err := ParseTrusted([]byte(`{"declaration":1,"left_out":["web"],"resources":[
+	  {"id":"app","type":"container","name":"app","image":"` + pinnedImage + `"}]}`)); err == nil ||
+		!strings.Contains(err.Error(), "only the mesh can say that") {
+		t.Fatalf("a carried bundle leaving modules out was accepted: %v", err)
+	}
+}

internal/link/asking_nats.go

@@ -3,6 +3,7 @@ package link
 import (
 	"context"
 	"crypto/rand"
+	"crypto/sha256"
 	"encoding/hex"
 	"errors"
 	"fmt"
@@ -63,8 +64,15 @@ func presentNats(_ context.Context, to Approach, node, secret string,
 	// subscribe its own inbox and nothing else (design 25 §6). The secret is its password, the same
 	// string the request claims, so the server proves somebody holds the token and the request
 	// proves the same thing to the controller without it having to ask the server who connected.
+	// **Its own inbox space, because that is the only one it may listen in** (novox/hq
+	// 04-ISSUES/146). A JetStream publish waits for the stream's acknowledgement on an inbox the
+	// client picks, and the client's default is `_INBOX.<random>` — which this user may not
+	// subscribe to, so the enrolment failed with a permissions violation on a subject nobody had
+	// chosen. The permission is `_INBOX.enrol.<node>.>` (design 25 §6), so the client is told to
+	// pick its inboxes there; the reply address below is in the same space for the same reason.
 	conn, err := nats.Connect(natsURL(to.Address),
 		nats.Secure(config),
+		nats.CustomInboxPrefix("_INBOX.enrol."+node),
 		nats.UserInfo("enrol."+node, secret),
 		nats.Name("mesh-host/enrol/"+node),
 		nats.Timeout(timeout),
@@ -124,7 +132,19 @@ func (a *natsAsking) Ask(ctx context.Context, request []byte, wait time.Duration
 	defer cancel()
 	// Into the stream and awaited: an enrolment the bus never accepted must fail here rather than be
 	// assumed, because the node has nothing else to go on.
-	if _, err := a.js.Publish(EnrolSubject, addressed, nats.Context(publish)); err != nil {
+	//
+	// **Once, however many times it is sent** (novox/hq 04-ISSUES/146). The client re-publishes when
+	// an acknowledgement is slow, and the mesh enrolled the machine on each copy — minting a second
+	// credential, which replaced the first, which is the one the node had already been given. The
+	// machine then reconnected for ever as a user whose password the mesh had rotated out from under
+	// it, and the controller's log said "enrolled anchor" twice in the same second.
+	//
+	// The id is the message: the same bytes carry the same id, so the stream discards the client's
+	// own retry, and a genuine second attempt — which carries a new reply address — is a different
+	// message and is let through.
+	sum := sha256.Sum256(addressed)
+	if _, err := a.js.Publish(EnrolSubject, addressed,
+		nats.MsgId(hex.EncodeToString(sum[:])), nats.Context(publish)); err != nil {
 		return nil, fmt.Errorf("cannot ask the mesh to enrol this node: %w", err)
 	}
 

internal/link/held_note_test.go

@@ -0,0 +1,46 @@
+package link
+
+import (
+	"strings"
+	"testing"
+)
+
+// The count that did not add up was the only symptom sixteen held resources had, and reading it meant
+// opening the node's state file by hand (novox/hq 04-ISSUES/125). The line that says what an apply did
+// says what it did not, too.
+
+func TestTheApplyLineSaysWhatItHeldAndForWhichModule(t *testing.T) {
+	got := heldNote([]Held{
+		{ID: "ca", Module: "route-proxy", Kind: "directory"},
+		{ID: "certs", Module: "route-proxy", Kind: "directory"},
+		{ID: "server", Module: "route-proxy", Kind: "container"},
+		{ID: "mail", Module: "mailu", Kind: "container"},
+	})
+	if !strings.Contains(got, "4 held") {
+		t.Fatalf("the count of what was held is not in the line: %q", got)
+	}
+	// The module is the thing an operator can act on: `take` takes a module.
+	if !strings.Contains(got, "route-proxy: 3") || !strings.Contains(got, "mailu: 1") {
+		t.Fatalf("the line does not break the holds down by module: %q", got)
+	}
+	// Ordered, so two machines holding the same things read the same and a diff of two reports is
+	// about what changed.
+	if strings.Index(got, "mailu") > strings.Index(got, "route-proxy") {
+		t.Fatalf("modules are not in a stated order: %q", got)
+	}
+	// It says why, because "held" alone reads as a failure and this is correct behaviour.
+	if !strings.Contains(got, "taken") {
+		t.Fatalf("the line does not say a hold ends when the module is taken: %q", got)
+	}
+}
+
+func TestAnApplyThatHeldNothingSaysNothingExtra(t *testing.T) {
+	// A converged machine holds nothing, which is most applies. Reporting "0 held" on every one of
+	// them is how a line stops being read.
+	if got := heldNote(nil); got != "" {
+		t.Fatalf("an apply with no holds added %q to its line", got)
+	}
+	if got := heldNote([]Held{}); got != "" {
+		t.Fatalf("an apply with no holds added %q to its line", got)
+	}
+}

internal/link/messages.go

@@ -110,6 +110,36 @@ type Report struct {
 	// and the mesh's up in its place, and where the found configuration's original was kept.
 	Tunnel *CarriedTunnel `json:"tunnel,omitempty"`
 
+	// Strays is what runs on the machine that the mesh neither wrote nor holds (novox/hq ADR
+	// 0163): containers nobody declared and nobody holds, the ones a cutover leaves behind.
+	Strays []Stray `json:"strays,omitempty"`
+
+	// Profile is what this machine can do, detected again by the apply that reports (novox/hq
+	// ADR 0161) — the same shape enrolment sends — so a capability gained or lost since enrolment,
+	// a network manager switched, reaches the mesh at the next push rather than never.
+	Profile map[string]any `json:"profile,omitempty"`
+
+	// Host is the version of the host that produced this report (novox/hq ADR 0141).
+	//
+	// Without it nothing can say a machine is behind, so "every machine current with its source"
+	// could not include the host — the one component the mesh did not deliver. It is a fact the
+	// machine states about itself, like the firewall it found and the links that face outside.
+	Host string `json:"host,omitempty"`
+
+	// Outward is the links on this machine that face outside it — the ones carrying a default
+	// route (novox/hq ADR 0140). Every node reports it, adopted or converged, because a converged
+	// node's filter is written around it.
+	//
+	// **It replaces a list of addresses.** The filter used to block everything passing through the
+	// machine and then allow the machine's own containers back by naming the ranges they sit on.
+	// A range describes one machine and goes stale in silence; the link carrying the default route
+	// is read afresh on every report and does not change when a module is added or removed.
+	//
+	// Empty means this machine has no route off itself. The mesh then composes no filter for it and
+	// leaves the one it has, rather than writing a rule around a link with no name — a rule set
+	// that does not load is a machine filtering nothing while its unit reports success.
+	Outward []string `json:"outward,omitempty"`
+
 	// Rekey is this node taking a found tunnel's key as its overlay key after enrolment (novox/hq
 	// ADR 0105). Not an account of the machine: a report carrying one says nothing else.
 	Rekey *Rekey `json:"rekey,omitempty"`
@@ -179,6 +209,16 @@ type Held struct {
 	Changed string `json:"changed,omitempty"`
 	// Kept is where a file's original was kept.
 	Kept string `json:"kept,omitempty"`
+	// Facts is the found thing beside what the module declares — what a take compares (novox/hq
+	// ADR 0163). The same shape the host keeps; the controller reads it as data.
+	Facts map[string]any `json:"facts,omitempty"`
+}
+
+// A Stray is a container the mesh neither wrote nor holds (ADR 0163).
+type Stray struct {
+	Kind   string `json:"kind"`
+	Name   string `json:"name"`
+	Detail string `json:"detail,omitempty"`
 }
 
 // Reach is one thing reachable on the machine: a listening socket, or a published container port.

internal/link/order_test.go

@@ -0,0 +1,50 @@
+package link
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+)
+
+// The drain picked the last to arrive. A backlog longer than the batch, or a broker that split a
+// burst, delivered a superseded declaration last (novox/hq 04-ISSUES/107).
+
+func sequenced(t *testing.T, n int64) *said {
+	t.Helper()
+	inner, err := json.Marshal(map[string]any{"declaration": 1, "resources": []any{}, "sequence": n})
+	if err != nil {
+		t.Fatal(err)
+	}
+	body, err := json.Marshal(Signed{Declaration: inner, Signature: []byte("s")})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return &said{body: body}
+}
+
+func TestTheDrainKeepsTheHighestSequenceNotTheLastToArrive(t *testing.T) {
+	waiting := make(chan Declaration, 8)
+	waiting <- sequenced(t, 9)
+	waiting <- sequenced(t, 4) // arrived last, composed earlier
+	latest, aside := newest(waiting, sequenced(t, 8), 30*time.Millisecond)
+	if got := sequenceOf(latest.Body()); got != 9 {
+		t.Fatalf("the drain kept sequence %d, and 9 was waiting", got)
+	}
+	if len(aside) != 2 {
+		t.Fatalf("%d set aside, wanted 2 (the 8 and the late 4)", len(aside))
+	}
+}
+
+func TestWithoutSequencesTheLastToArriveStillWins(t *testing.T) {
+	// The behaviour this had before, kept for a controller that sends no order.
+	apply, aside := newest(arriving("two", "three"), &said{body: []byte("one")}, 30*time.Millisecond)
+	if string(apply.Body()) != "three" || len(aside) != 2 {
+		t.Fatalf("applied %q with %d set aside", apply.Body(), len(aside))
+	}
+}
+
+func TestAnUnreadableBodyClaimsNoOrder(t *testing.T) {
+	if got := sequenceOf([]byte("not json")); got != 0 {
+		t.Fatalf("garbage claimed sequence %d", got)
+	}
+}

internal/link/pinned.go

@@ -73,8 +73,19 @@ func PinnedConfig(pin string) (*tls.Config, error) {
 	}, nil
 }
 
-// Dial opens a TLS connection to the broker, refusing anything but the pinned certificate.
-func Dial(address, pin string, timeout time.Duration) (*tls.Conn, error) {
+// dialPinned completes a TLS handshake against an address, refusing anything but the pinned
+// certificate.
+//
+// **Not how the bus is reached, and it used to be** (novox/hq 04-ISSUES/146). Enrolment opened one
+// of these before it said anything, which was right while the broker answered TLS immediately and
+// wrong the moment the mesh moved to a bus that speaks its own protocol first. The pin itself was
+// never the problem — PinnedConfig is what the NATS client is given, and the verification runs
+// inside the handshake that client performs.
+//
+// It stays here because this is where the pin is proven: the tests beside it run a real TLS server
+// and assert that a wrong certificate is refused before a byte of application data is sent. What it
+// must not become again is something a caller uses to reach the bus.
+func dialPinned(address, pin string, timeout time.Duration) (*tls.Conn, error) {
 	config, err := PinnedConfig(pin)
 	if err != nil {
 		return nil, err
@@ -86,7 +97,7 @@ func Dial(address, pin string, timeout time.Duration) (*tls.Conn, error) {
 		if errors.Is(err, ErrWrongCertificate) {
 			return nil, err
 		}
-		return nil, fmt.Errorf("cannot reach the broker at %s: %w", address, err)
+		return nil, fmt.Errorf("cannot reach %s: %w", address, err)
 	}
 	return conn, nil
 }

internal/link/pinned_test.go

@@ -63,7 +63,7 @@ func server(t *testing.T) (address string, fingerprint string) {
 
 func TestTheRightBrokerIsAccepted(t *testing.T) {
 	address, pin := server(t)
-	conn, err := Dial(address, pin, 5*time.Second)
+	conn, err := dialPinned(address, pin, 5*time.Second)
 	if err != nil {
 		t.Fatalf("the broker its token describes was refused: %v", err)
 	}
@@ -76,7 +76,7 @@ func TestADifferentBrokerIsRefused(t *testing.T) {
 	address, _ := server(t)
 	_, other := server(t)
 
-	_, err := Dial(address, other, 5*time.Second)
+	_, err := dialPinned(address, other, 5*time.Second)
 	if err == nil {
 		t.Fatal("a broker presenting a different certificate was accepted")
 	}
@@ -130,7 +130,7 @@ func TestNothingIsSentToTheWrongBroker(t *testing.T) {
 
 	// A pin for a certificate this server does not have.
 	_, elsewhere := server(t)
-	if _, err := Dial(listener.Addr().String(), elsewhere, 5*time.Second); err == nil {
+	if _, err := dialPinned(listener.Addr().String(), elsewhere, 5*time.Second); err == nil {
 		t.Fatal("the impostor was accepted")
 	}
 	if n := <-received; n > 0 {
@@ -160,7 +160,7 @@ func TestAnUnreachableBrokerIsAnOrdinaryFailure(t *testing.T) {
 	address := listener.Addr().String()
 	listener.Close()
 
-	_, err = Dial(address, pin, 2*time.Second)
+	_, err = dialPinned(address, pin, 2*time.Second)
 	if err == nil {
 		t.Fatal("dialling a closed port succeeded")
 	}

internal/link/run.go

@@ -6,6 +6,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"sort"
+	"strings"
 	"time"
 )
 
@@ -250,9 +252,11 @@ func Run(ctx context.Context, m Membership, apply Applier, say Announce, timeout
 			case report.Refused != "":
 				say("refused a declaration: " + report.Refused)
 			case len(report.Failed) > 0:
-				say(fmt.Sprintf("applied %d and failed: %v", len(report.Applied), report.Failed))
+				say(fmt.Sprintf("applied %d and failed: %v%s",
+					len(report.Applied), report.Failed, heldNote(report.Held)))
 			default:
-				say(fmt.Sprintf("applied %d resource(s)", len(report.Applied)))
+				say(fmt.Sprintf("applied %d resource(s)%s",
+					len(report.Applied), heldNote(report.Held)))
 			}
 			publishReport(ctx, link, m, report, say, timeout)
 			// Settled after the report is published. A node that dies between applying and
@@ -296,6 +300,16 @@ func newest(arriving <-chan Declaration, first Declaration, window time.Duration
 			if !ok {
 				return latest, superseded
 			}
+			// **By sequence when both carry one, by arrival when either does not** (novox/hq
+			// 04-ISSUES/107). Arrival is what this window had to go on, and it is wrong exactly
+			// when it matters — a backlog drained out of order. A declaration that says where it
+			// stands is believed over when it turned up; one that does not is the older
+			// controller's, and arrival is all there is.
+			if sequenceOf(next.Body()) < sequenceOf(latest.Body()) &&
+				sequenceOf(next.Body()) > 0 && sequenceOf(latest.Body()) > 0 {
+				superseded = append(superseded, next)
+				continue
+			}
 			superseded = append(superseded, latest)
 			latest = next
 		case <-time.After(window):
@@ -304,6 +318,24 @@ func newest(arriving <-chan Declaration, first Declaration, window time.Duration
 	}
 }
 
+// sequenceOf is the order a signed declaration claims, or zero when it claims none or cannot be
+// read. Read from the envelope alone; the signature is verified later, when the winner is applied,
+// and a forged message that lied about its sequence would only set aside real ones — which are
+// reported as set aside, and the next push sends the current one again.
+func sequenceOf(body []byte) int64 {
+	var signed Signed
+	if err := json.Unmarshal(body, &signed); err != nil {
+		return 0
+	}
+	var d struct {
+		Sequence int64 `json:"sequence"`
+	}
+	if err := json.Unmarshal(signed.Declaration, &d); err != nil {
+		return 0
+	}
+	return d.Sequence
+}
+
 // declaredIn is the id a signed declaration carries, for a report about one that was not applied.
 // Empty if the message is not one — a forged or garbled message is refused by handleBody when its
 // turn comes; here it is only named.
@@ -392,3 +424,36 @@ func publishAlive(ctx context.Context, bus Bus, m Membership, say Announce,
 		say("could not tell the mesh this node is here: " + err.Error())
 	}
 }
+
+// heldNote is what this apply did NOT do, for the line that says what it did.
+//
+// **A count that does not add up is the only symptom a held resource had** (novox/hq 04-ISSUES/125).
+// An adopted node keeps what it found until its module is taken (ADR 0100), and that is correct — but
+// it was recorded only in the node's own state file. On the edge cut-over the mesh sent 346 resources,
+// the journal said it applied 330, and nothing anywhere said which sixteen or why. Reading it took
+// opening state.json by hand; not reading it took every public name on the machine down, because the
+// operator had four green surfaces and a discrepancy nobody could interpret.
+//
+// So the line that reports the apply carries it. Grouped by module and ordered by name, because the
+// sentence an operator needs is "route-proxy is assigned and not taken", and the module is the thing
+// they can act on — `take` is the verb, and it takes a module.
+func heldNote(held []Held) string {
+	if len(held) == 0 {
+		return ""
+	}
+	byModule := map[string]int{}
+	for _, h := range held {
+		byModule[h.Module]++
+	}
+	names := make([]string, 0, len(byModule))
+	for name := range byModule {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+	parts := make([]string, 0, len(names))
+	for _, name := range names {
+		parts = append(parts, fmt.Sprintf("%s: %d", name, byModule[name]))
+	}
+	return fmt.Sprintf(", %d held until their module is taken (%s)",
+		len(held), strings.Join(parts, ", "))
+}

internal/outward/links.go

@@ -0,0 +1,147 @@
+// Package outward reads which of this machine's links face outside it (novox/hq ADR 0140).
+//
+// The filter the mesh derives constrains traffic arriving from outside the machine and says nothing
+// about traffic that did not. To write that rule the mesh has to know which links "outside" arrives
+// on, and that is a thing only the machine can say — so it says it, once per report, the way it
+// already reports the kind of firewall it found and the tunnel it carried.
+//
+// **It replaces a list of addresses.** The filter used to allow the machine's own containers back
+// through by naming the address ranges they sit on: two ranges fixed in the control plane's source
+// and the rest typed by an operator. A range describes one machine and goes stale silently
+// (novox/hq 04-ISSUES/137 and /141). A link that carries the default route is a fact the machine
+// reads afresh every time, and it does not change when a module is added or removed.
+//
+// It reads the kernel's routing tables directly rather than asking a program. A module naming a
+// program the machine does not have is how the mesh already reported success while doing nothing
+// (novox/hq 04-ISSUES/136), and every machine has /proc.
+package outward
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// ProcNet is where the kernel publishes its routing tables. A parameter so a test can hold a
+// routing table without one.
+const ProcNet = "/proc/net"
+
+// Links are the interfaces carrying a default route, for both address families, sorted and without
+// repeats.
+//
+// A machine may have more than one: a laptop with a cable and a radio has two, and both face
+// outside. A machine with none — no route off itself — returns nothing, and the mesh refuses to
+// compose a filter for it rather than writing a rule around a link with no name, which would be a
+// rule set that does not load and a machine filtering nothing while its unit reports success.
+func Links(procNet string) ([]string, error) {
+	if procNet == "" {
+		procNet = ProcNet
+	}
+	seen := map[string]bool{}
+
+	four, err := defaultsV4(filepath.Join(procNet, "route"))
+	if err != nil {
+		return nil, err
+	}
+	six, err := defaultsV6(filepath.Join(procNet, "ipv6_route"))
+	if err != nil {
+		return nil, err
+	}
+	for _, name := range append(four, six...) {
+		if name != "" && name != "lo" {
+			seen[name] = true
+		}
+	}
+
+	out := make([]string, 0, len(seen))
+	for name := range seen {
+		out = append(out, name)
+	}
+	sort.Strings(out)
+	return out, nil
+}
+
+// defaultsV4 reads /proc/net/route, whose columns are
+//
+//	Iface Destination Gateway Flags RefCnt Use Metric Mask ...
+//
+// with addresses in hexadecimal. A default route is destination zero with mask zero — the mask
+// matters, because a route to the zero address with a real mask is not a default route.
+func defaultsV4(path string) ([]string, error) {
+	lines, err := rows(path)
+	if err != nil {
+		return nil, err
+	}
+	var out []string
+	for _, fields := range lines {
+		if len(fields) < 8 {
+			continue
+		}
+		if isZeroHex(fields[1]) && isZeroHex(fields[7]) {
+			out = append(out, fields[0])
+		}
+	}
+	return out, nil
+}
+
+// defaultsV6 reads /proc/net/ipv6_route, whose columns are
+//
+//	dest destprefix src srcprefix nexthop metric refcnt use flags iface
+//
+// A default route is the zero destination with a zero prefix length.
+func defaultsV6(path string) ([]string, error) {
+	lines, err := rows(path)
+	if err != nil {
+		return nil, err
+	}
+	var out []string
+	for _, fields := range lines {
+		if len(fields) < 10 {
+			continue
+		}
+		if isZeroHex(fields[0]) && isZeroHex(fields[1]) {
+			out = append(out, fields[9])
+		}
+	}
+	return out, nil
+}
+
+// rows reads a routing table into fields per line, skipping a header and blank lines. A table that
+// is not there is not an error: a machine without the second address family has no file for it,
+// and that is not a machine that cannot be filtered.
+func rows(path string) ([][]string, error) {
+	file, err := os.Open(path)
+	if os.IsNotExist(err) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("cannot read the routing table at %s: %w", path, err)
+	}
+	defer file.Close()
+
+	var out [][]string
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" || strings.HasPrefix(line, "Iface") {
+			continue
+		}
+		out = append(out, strings.Fields(line))
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("cannot read the routing table at %s: %w", path, err)
+	}
+	return out, nil
+}
+
+// isZeroHex is whether a hexadecimal field is all zeroes, whatever its width — the v4 table writes
+// eight digits and the v6 table thirty-two, and a prefix length is two.
+func isZeroHex(field string) bool {
+	if field == "" {
+		return false
+	}
+	return strings.Trim(strings.ToLower(field), "0") == ""
+}

internal/outward/links_test.go

@@ -0,0 +1,120 @@
+package outward
+
+import (
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+)
+
+// A routing table as the kernel writes it: a default route, a route to the zero address that is not
+// one, and a route on the loopback. Only the default route's link faces outside.
+const routeV4 = `Iface	Destination	Gateway 	Flags	RefCnt	Use	Metric	Mask		MTU	Window	IRTT
+enp9s0	00000000	01FEA8C0	0003	0	0	100	00000000	0	0	0
+docker0	000011AC	00000000	0001	0	0	0	0000FFFF	0	0	0
+enp9s0	00000000	00000000	0001	0	0	100	00FFFFFF	0	0	0
+lo	00000000	00000000	0003	0	0	0	00000000	0	0	0
+`
+
+const routeV6 = `00000000000000000000000000000000 00 00000000000000000000000000000000 00 fe800000000000000000000000000001 00000400 00000001 00000000 00000003     wlan0
+fd0000000000000000000000000000000 40 00000000000000000000000000000000 00 00000000000000000000000000000000 00000100 00000000 00000000 00000001    enp9s0
+`
+
+func write(t *testing.T, dir, name, body string) {
+	t.Helper()
+	if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestLinksAreTheOnesCarryingADefaultRoute(t *testing.T) {
+	dir := t.TempDir()
+	write(t, dir, "route", routeV4)
+	write(t, dir, "ipv6_route", routeV6)
+
+	got, err := Links(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// The cable from the v4 table and the radio from the v6 one. Not docker0, whose route is not a
+	// default; not the loopback, which faces nothing; and not the v6 route with a real prefix.
+	want := []string{"enp9s0", "wlan0"}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("outward links are %v, want %v", got, want)
+	}
+}
+
+// A route to the zero address with a real mask is not a default route. Trusting the destination
+// alone would name every link with such a route as facing outside, and a filter that treats an
+// internal bridge as outward constrains this machine's own guests — the fault ADR 0140 removes.
+func TestAZeroDestinationWithAMaskIsNotADefaultRoute(t *testing.T) {
+	dir := t.TempDir()
+	write(t, dir, "route", `Iface	Destination	Gateway 	Flags	RefCnt	Use	Metric	Mask		MTU	Window	IRTT
+br-abc	00000000	00000000	0001	0	0	0	00FFFFFF	0	0	0
+`)
+	got, err := Links(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 0 {
+		t.Fatalf("outward links are %v, want none", got)
+	}
+}
+
+// A machine with no route off itself says so, rather than guessing. The mesh refuses to compose a
+// filter for it; a rule written around a link with no name does not load, and a rule set that does
+// not load is a machine filtering nothing while its unit reports success.
+func TestNoDefaultRouteIsNoLinks(t *testing.T) {
+	dir := t.TempDir()
+	write(t, dir, "route", "Iface\tDestination\tGateway \tFlags\tRefCnt\tUse\tMetric\tMask\t\tMTU\tWindow\tIRTT\n")
+	got, err := Links(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 0 {
+		t.Fatalf("outward links are %v, want none", got)
+	}
+}
+
+// A machine without the second address family has no file for it. That is not a machine that cannot
+// be filtered, so a missing table is read as no routes rather than as a failure.
+func TestAMissingTableIsNotAFailure(t *testing.T) {
+	dir := t.TempDir()
+	write(t, dir, "route", routeV4)
+	got, err := Links(dir)
+	if err != nil {
+		t.Fatalf("a missing v6 table should not fail: %v", err)
+	}
+	if !reflect.DeepEqual(got, []string{"enp9s0"}) {
+		t.Fatalf("outward links are %v, want [enp9s0]", got)
+	}
+}
+
+// The same link carrying a default route in both families is reported once.
+func TestALinkIsReportedOnce(t *testing.T) {
+	dir := t.TempDir()
+	write(t, dir, "route", routeV4)
+	write(t, dir, "ipv6_route",
+		"00000000000000000000000000000000 00 00000000000000000000000000000000 00 "+
+			"fe800000000000000000000000000001 00000400 00000001 00000000 00000003     enp9s0\n")
+	got, err := Links(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(got, []string{"enp9s0"}) {
+		t.Fatalf("outward links are %v, want [enp9s0]", got)
+	}
+}
+
+// Against this machine's own routing table, so the parse is held to what the kernel actually writes
+// and not only to a fixture written to agree with it.
+func TestAgainstThisMachinesOwnTable(t *testing.T) {
+	got, err := Links("")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) == 0 {
+		t.Skip("this machine has no default route")
+	}
+	t.Logf("this machine's outward links: %v", got)
+}

internal/profile/detectors.go

@@ -20,6 +20,14 @@ const (
 	// state: whether one IS running. Assignment needs the first.
 	CapSeat       = "seat"
 	CapPrivileged = "privileged"
+
+	// The network manager this machine runs, one capability per dialect (novox/hq ADR 0161): the
+	// uplink seat's holder declares its own, so the holder for a manager the machine does not run
+	// is refused the way any missing capability is, naming it. Active, not installed — a machine
+	// may have two of these on disk and runs one.
+	CapUplinkNetworkManager  = "uplink-networkmanager"
+	CapUplinkSystemdNetworkd = "uplink-systemd-networkd"
+	CapUplinkDhcpcd          = "uplink-dhcpcd"
 )
 
 // commandCapability is the shape most detectors take: run something, and treat a working
@@ -202,6 +210,21 @@ func Default(runner Runner) []Detector {
 			why:    "asks the kernel for interfaces — needs the module, not just the tool",
 			runner: runner,
 		},
+		commandCapability{
+			name: CapUplinkNetworkManager, command: "systemctl", args: []string{"is-active", "NetworkManager.service"},
+			why:    "asks the init whether NetworkManager is running — the dialect the uplink seat's holder must speak",
+			runner: runner,
+		},
+		commandCapability{
+			name: CapUplinkSystemdNetworkd, command: "systemctl", args: []string{"is-active", "systemd-networkd.service"},
+			why:    "asks the init whether systemd-networkd is running — the dialect the uplink seat's holder must speak",
+			runner: runner,
+		},
+		commandCapability{
+			name: CapUplinkDhcpcd, command: "systemctl", args: []string{"is-active", "dhcpcd.service"},
+			why:    "asks the init whether dhcpcd is running — the dialect the uplink seat's holder must speak",
+			runner: runner,
+		},
 	}
 }
 

internal/profile/uplink_test.go

@@ -0,0 +1,39 @@
+package profile
+
+import (
+	"context"
+	"errors"
+	"testing"
+)
+
+// The uplink seat's holder must be the dialect the machine runs (novox/hq ADR 0161): the profile
+// names the network manager found active, one capability per manager, and nothing for one that is
+// merely installed.
+func TestTheProfileNamesTheNetworkManagerThatIsRunning(t *testing.T) {
+	runner := func(_ context.Context, name string, args ...string) (string, error) {
+		if name == "systemctl" && len(args) == 2 && args[0] == "is-active" {
+			if args[1] == "NetworkManager.service" {
+				return "active\n", nil
+			}
+			return "inactive\n", errors.New("exit status 3")
+		}
+		return "", errors.New("not here")
+	}
+	var have []Detector
+	for _, d := range Default(Runner(runner)) {
+		switch d.Name() {
+		case CapUplinkNetworkManager, CapUplinkSystemdNetworkd, CapUplinkDhcpcd:
+			have = append(have, d)
+		}
+	}
+	if len(have) != 3 {
+		t.Fatalf("expected a detector per manager, found %d", len(have))
+	}
+	for _, d := range have {
+		v := d.Detect(context.Background())
+		want := d.Name() == CapUplinkNetworkManager
+		if v.Present != want {
+			t.Errorf("%s: present=%v, want %v (%s)", d.Name(), v.Present, want, v.Detail)
+		}
+	}
+}

internal/store/former_test.go

@@ -0,0 +1,29 @@
+package store
+
+import "testing"
+
+// A resource whose target moves leaves what the host wrote under the old target on record as a
+// former one, undeclared by construction, so the next apply removes it (novox/hq issue 097, ADR 0163).
+func TestARecordWhoseTargetMovedKeepsTheFormerTargetToRemove(t *testing.T) {
+	s := State{}
+	s.Record(Applied{ID: "gitea.server", Type: "container", Target: "mesh-gitea", Origin: OriginDeclared})
+	s.Record(Applied{ID: "gitea.server", Type: "container", Target: "gitea", Origin: OriginDeclared})
+	if len(s.Resources) != 2 {
+		t.Fatalf("a moved target produced %d record(s): %+v", len(s.Resources), s.Resources)
+	}
+	orphans := s.Orphans(map[string]bool{"gitea.server": true}, OriginDeclared)
+	if len(orphans) != 1 || orphans[0].Target != "mesh-gitea" || !IsFormer(orphans[0].ID) {
+		t.Fatalf("the former target is not an orphan to remove: %+v", orphans)
+	}
+	s.Forget(orphans[0].ID)
+	if len(s.Resources) != 1 || s.Resources[0].Target != "gitea" {
+		t.Fatalf("forgetting the former target touched the current one: %+v", s.Resources)
+	}
+	// The same target again is not a move; a carried record is not the host's to remove.
+	s.Record(Applied{ID: "gitea.server", Type: "container", Target: "gitea", Origin: OriginDeclared})
+	s.Record(Applied{ID: "bundle", Type: "file", Target: "/a"})
+	s.Record(Applied{ID: "bundle", Type: "file", Target: "/b"})
+	if len(s.Resources) != 2 {
+		t.Fatalf("an unmoved or carried record grew the list: %+v", s.Resources)
+	}
+}

internal/store/store.go

@@ -18,6 +18,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
 	"time"
 )
 
@@ -274,6 +275,41 @@ type Held struct {
 	// reverted: that is how a predecessor still writing is caught.
 	Changed   string    `json:"changed,omitempty"`
 	ChangedAt time.Time `json:"changed_at,omitempty"`
+	// Facts is what a take would compare: the found thing beside what the module declares
+	// (novox/hq ADR 0163). Read fresh on every apply while held, so the controller's preview
+	// speaks of the machine as it is.
+	Facts *Facts `json:"facts,omitempty"`
+}
+
+// Facts is a held thing beside what its module declares — what a take compares (ADR 0163).
+type Facts struct {
+	// A found container: the image it runs and when that image was made; the networks it is on
+	// and the other containers on each; what it mounts; what it publishes.
+	Image        string              `json:"image,omitempty"`
+	ImageCreated string              `json:"image_created,omitempty"`
+	Networks     map[string][]string `json:"networks,omitempty"`
+	Mounts       []string            `json:"mounts,omitempty"`
+	Ports        []string            `json:"ports,omitempty"`
+	// What the module declares for it, and the declared image's creation date when the image
+	// is on the machine already.
+	DeclaredImage        string   `json:"declared_image,omitempty"`
+	DeclaredImageCreated string   `json:"declared_image_created,omitempty"`
+	DeclaredPorts        []string `json:"declared_ports,omitempty"`
+	DeclaredVolumes      []string `json:"declared_volumes,omitempty"`
+	// Downgrade is true when both creation dates are known and the declared image is the older.
+	Downgrade bool `json:"downgrade,omitempty"`
+	// A found file: whether the declared content differs from what was found, and how, as lines
+	// only in the found file (-) and lines only in the declared one (+), bounded.
+	Differs    bool     `json:"differs,omitempty"`
+	Difference []string `json:"difference,omitempty"`
+}
+
+// A Stray is something running on the machine that the mesh neither wrote nor holds
+// (novox/hq ADR 0163): the answer to "what is here that nobody asked for".
+type Stray struct {
+	Kind   string `json:"kind"`
+	Name   string `json:"name"`
+	Detail string `json:"detail,omitempty"`
 }
 
 // Recorded reports whether this host has a record, of any origin, of putting something of this
@@ -462,6 +498,20 @@ func Save(path string, s State) error {
 func (s *State) Record(a Applied) {
 	for i, existing := range s.Resources {
 		if existing.ID == a.ID {
+			// A resource whose target moved leaves what the host wrote under the old target
+			// behind — a container under the old name, a file at the old path. Rewriting the
+			// record would erase the only trace of it (novox/hq issue 097, ADR 0163), so the old
+			// target stays on record as a former one, undeclared by construction, until the next
+			// apply removes it the way it removes anything the host wrote and no longer declares.
+			// What was found is held, never recorded here, and so never removed by this.
+			if originOf(existing) == OriginDeclared && existing.Target != "" && a.Target != "" &&
+				existing.Target != a.Target && existing.Type == a.Type {
+				former := existing
+				former.ID = FormerID(existing.ID, existing.Target)
+				s.Resources[i] = a
+				s.Resources = append(s.Resources, former)
+				return
+			}
 			s.Resources[i] = a
 			return
 		}
@@ -469,6 +519,13 @@ func (s *State) Record(a Applied) {
 	s.Resources = append(s.Resources, a)
 }
 
+// FormerID names the record of a resource's former target: the resource's id and the target it
+// had, so the record is distinct from the current one and is never what a declaration names.
+func FormerID(id, target string) string { return id + "@former:" + target }
+
+// IsFormer says whether a record names a former target.
+func IsFormer(id string) bool { return strings.Contains(id, "@former:") }
+
 // Forget drops a resource from what the node owns.
 func (s *State) Forget(id string) {
 	kept := s.Resources[:0]

internal/upgrade/upgrade.go

@@ -16,7 +16,9 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
+	"time"
 )
 
 // Files the launcher reads and this binary writes. Next to the store, because they are node
@@ -157,3 +159,168 @@ func ReadKnownGood(path string) (string, error) {
 	}
 	return strings.TrimSpace(string(raw)), nil
 }
+
+// Where delivered versions live, and what the binary inside one is called.
+//
+// **A directory named for its version, never a link and never a write over what is running**
+// (novox/hq ADR 0141). Two facts follow from that one choice: the kernel refuses to truncate a
+// running executable, so the path a delivery writes must not be the path being executed; and a
+// rollback needs the previous version still present, which a single path cannot offer.
+//
+// The mesh creates no links (novox/hq ADR 0012), so nothing points at "current". The version is in
+// the path, which is why nothing has to be told what is running.
+const (
+	// DefaultLibexec is where the host's own files live. Fixed rather than derived from where the
+	// running executable sits: the first host to understand any of this was copied to a machine by
+	// hand, and one that looked for its successor beside itself would never find a delivered version
+	// — which is every machine in this mesh on the day this ships.
+	DefaultLibexec  = "/usr/lib/nox-mesh-host"
+	VersionsDirName = "versions"
+	BinaryName      = "nox-mesh-host"
+	// PinnedName is the version a rollback chose, which the launcher runs instead of the newest.
+	// Without it the launcher would start the newest again and the rollback would flap.
+	PinnedName = "rollback-pinned"
+)
+
+// VersionsDir is where delivered versions live, given where the host's libexec is. An empty libexec
+// means the default, and the environment overrides it so a test needs no root.
+func VersionsDir(libexec string) string {
+	if libexec == "" {
+		libexec = os.Getenv("MESH_HOST_LIBEXEC")
+	}
+	if libexec == "" {
+		libexec = DefaultLibexec
+	}
+	return filepath.Join(libexec, VersionsDirName)
+}
+
+// PinnedPath is where a rollback records the version it chose.
+func PinnedPath(statePath string) string {
+	return filepath.Join(filepath.Dir(statePath), PinnedName)
+}
+
+// Delivered is one version present on the machine.
+type Delivered struct {
+	// Version is the directory's name, which is the version.
+	Version string
+	// Binary is the executable inside it.
+	Binary string
+	// At is when it arrived, which is how "newest" is decided.
+	At time.Time
+}
+
+// Versions are the versions delivered to this machine, newest first.
+//
+// **Newest by when it arrived, not by its name.** A version string comes from what the source was
+// tagged or described as, and those do not sort: "1.10" before "1.9", a commit hash before either.
+// Ordering by name would run an older host and call it an upgrade. When it arrived is a fact the
+// filesystem keeps and the delivery sets.
+//
+// A directory with no executable in it is not a version. A delivery that was interrupted leaves one,
+// and running the newest would then mean running nothing.
+func Versions(dir string) ([]Delivered, error) {
+	entries, err := os.ReadDir(dir)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("cannot read the delivered versions at %s: %w", dir, err)
+	}
+
+	var out []Delivered
+	for _, entry := range entries {
+		if !entry.IsDir() {
+			continue
+		}
+		binary := filepath.Join(dir, entry.Name(), BinaryName)
+		info, err := os.Stat(binary)
+		if err != nil || info.IsDir() {
+			continue
+		}
+		at := info.ModTime()
+		if d, err := entry.Info(); err == nil && d.ModTime().After(at) {
+			at = d.ModTime()
+		}
+		out = append(out, Delivered{Version: entry.Name(), Binary: binary, At: at})
+	}
+
+	// Newest first, and by name when two arrived in the same instant so the answer is never
+	// arbitrary — a test that passes half the time is worse than one that fails.
+	sort.Slice(out, func(a, b int) bool {
+		if out[a].At.Equal(out[b].At) {
+			return out[a].Version > out[b].Version
+		}
+		return out[a].At.After(out[b].At)
+	})
+	return out, nil
+}
+
+// Successor is the version this machine should be running instead of the given one, if any.
+//
+// Empty when the running version is the newest, which is the ordinary answer. The host asks this
+// between reconciles and nowhere else: standing aside mid-apply is the half-configured machine the
+// host exists to prevent (novox/hq ADR 0141).
+func Successor(dir, running string) (Delivered, bool, error) {
+	delivered, err := Versions(dir)
+	if err != nil {
+		return Delivered{}, false, err
+	}
+	if len(delivered) == 0 {
+		return Delivered{}, false, nil
+	}
+	newest := delivered[0]
+	// A machine whose running version is not among the delivered ones is the machine every mesh has
+	// one of: the host was put there by hand before any of this existed. Treating that as "stand
+	// aside" is correct — what was delivered is what the mesh asked for.
+	if newest.Version == running {
+		return Delivered{}, false, nil
+	}
+	return newest, true, nil
+}
+
+// Retire removes delivered versions older than the running one's predecessor.
+//
+// The running version and the one before it are kept, and nothing else: the predecessor is exactly
+// what a rollback starts, and every version before that is weight with no reader. Called after a
+// reconcile completes, which is the same evidence known-good is written on — retiring on any weaker
+// signal would delete the thing a failing host is about to need.
+//
+// Never the running version, whatever it is asked. A host that deleted its own image would survive
+// until it stopped and then be unstartable, and the launcher's rollback reads a version, not a
+// process.
+func Retire(dir, running string) ([]string, error) {
+	delivered, err := Versions(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	keep := map[string]bool{running: true}
+	for i, d := range delivered {
+		if d.Version != running {
+			continue
+		}
+		// Its predecessor is the next one down the list, which is the next oldest.
+		if i+1 < len(delivered) {
+			keep[delivered[i+1].Version] = true
+		}
+		break
+	}
+	// A running version that was never delivered has no predecessor among these, so the newest
+	// delivered one is what a rollback would reach for. Keep it.
+	if len(keep) == 1 && len(delivered) > 0 {
+		keep[delivered[0].Version] = true
+	}
+
+	var removed []string
+	for _, d := range delivered {
+		if keep[d.Version] {
+			continue
+		}
+		if err := os.RemoveAll(filepath.Join(dir, d.Version)); err != nil {
+			return removed, fmt.Errorf("cannot retire the host version %s: %w", d.Version, err)
+		}
+		removed = append(removed, d.Version)
+	}
+	sort.Strings(removed)
+	return removed, nil
+}

internal/upgrade/versions_test.go

@@ -0,0 +1,180 @@
+package upgrade
+
+import (
+	"os"
+	"path/filepath"
+	"reflect"
+	"sort"
+	"testing"
+	"time"
+)
+
+// deliver writes a version as a delivery would: a directory named for it with the binary inside.
+// at fixes when it arrived, because "newest" is when it arrived and a test must not race the clock.
+func deliver(t *testing.T, dir, version string, at time.Time) string {
+	t.Helper()
+	into := filepath.Join(dir, version)
+	if err := os.MkdirAll(into, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	binary := filepath.Join(into, BinaryName)
+	if err := os.WriteFile(binary, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chtimes(binary, at, at); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chtimes(into, at, at); err != nil {
+		t.Fatal(err)
+	}
+	return binary
+}
+
+// **Newest is when it arrived, not how its name sorts.**
+//
+// A version string is whatever the source was tagged or described as, and those do not sort: "1.10"
+// orders before "1.9", and a commit hash orders before either. Ordering by name would start an older
+// host and call that an upgrade.
+func TestNewestIsWhenItArrivedAndNotHowItSorts(t *testing.T) {
+	dir := t.TempDir()
+	base := time.Now().Add(-time.Hour)
+	deliver(t, dir, "1.10", base)                 // sorts LAST by name, arrived first
+	deliver(t, dir, "1.9", base.Add(time.Minute)) // sorts first by name, arrived last
+
+	got, err := Versions(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 2 || got[0].Version != "1.9" {
+		t.Fatalf("newest is %+v, want the one that arrived last (1.9)", got)
+	}
+}
+
+// A delivery that was interrupted leaves a directory with no executable in it. Running "the newest"
+// would then mean running nothing, so it is not a version.
+func TestADirectoryWithNoBinaryIsNotAVersion(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(dir, "half-delivered"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	deliver(t, dir, "good", time.Now().Add(-time.Hour))
+
+	got, err := Versions(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 1 || got[0].Version != "good" {
+		t.Fatalf("versions are %+v, want only the one with a binary", got)
+	}
+}
+
+// Nothing delivered is not a fault. A machine whose host was placed by hand has no versions
+// directory at all, and that must read as "no successor" rather than as an error that stops a
+// reconcile.
+func TestNoVersionsDirectoryIsNotAnError(t *testing.T) {
+	got, err := Versions(filepath.Join(t.TempDir(), "absent"))
+	if err != nil {
+		t.Fatalf("an absent versions directory should not be an error: %v", err)
+	}
+	if len(got) != 0 {
+		t.Fatalf("versions are %+v, want none", got)
+	}
+}
+
+func TestTheNewestVersionIsTheSuccessorAndTheRunningOneIsNot(t *testing.T) {
+	dir := t.TempDir()
+	base := time.Now().Add(-time.Hour)
+	deliver(t, dir, "one", base)
+	deliver(t, dir, "two", base.Add(time.Minute))
+
+	next, yes, err := Successor(dir, "one")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !yes || next.Version != "two" {
+		t.Fatalf("successor is %+v (%v), want two", next, yes)
+	}
+
+	if _, yes, err := Successor(dir, "two"); err != nil || yes {
+		t.Fatalf("the newest version is its own successor (%v, %v)", yes, err)
+	}
+}
+
+// A host put there by hand, before any of this existed, is not among the delivered versions. What the
+// mesh delivered is what it asked for, so that is a successor — otherwise the first delivery to such a
+// machine would be ignored for ever, which is every machine in this mesh today.
+func TestAHostThatWasNeverDeliveredHasASuccessor(t *testing.T) {
+	dir := t.TempDir()
+	deliver(t, dir, "delivered", time.Now().Add(-time.Hour))
+
+	next, yes, err := Successor(dir, "copied-by-hand")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !yes || next.Version != "delivered" {
+		t.Fatalf("successor is %+v (%v), want the delivered one", next, yes)
+	}
+}
+
+// The running version and its predecessor are kept, and nothing else. The predecessor is exactly what
+// a rollback starts; everything older has no reader.
+func TestRetireKeepsTheRunningVersionAndItsPredecessor(t *testing.T) {
+	dir := t.TempDir()
+	base := time.Now().Add(-4 * time.Hour)
+	for i, v := range []string{"one", "two", "three", "four"} {
+		deliver(t, dir, v, base.Add(time.Duration(i)*time.Hour))
+	}
+
+	removed, err := Retire(dir, "four")
+	if err != nil {
+		t.Fatal(err)
+	}
+	sort.Strings(removed)
+	if !reflect.DeepEqual(removed, []string{"one", "two"}) {
+		t.Fatalf("retired %v, want one and two — three is the predecessor a rollback needs", removed)
+	}
+	for _, kept := range []string{"three", "four"} {
+		if _, err := os.Stat(filepath.Join(dir, kept, BinaryName)); err != nil {
+			t.Fatalf("%s was retired and a rollback now has nowhere to go: %v", kept, err)
+		}
+	}
+}
+
+// **Never the running version, whatever it is asked.** A host that deleted its own image would run
+// until it stopped and then be unstartable, and the launcher's rollback reads a version rather than a
+// process.
+func TestRetireNeverRemovesTheRunningVersion(t *testing.T) {
+	dir := t.TempDir()
+	base := time.Now().Add(-2 * time.Hour)
+	deliver(t, dir, "older", base)
+	deliver(t, dir, "newer", base.Add(time.Hour))
+
+	// Asked while running the OLDER one, which is what a machine looks like between a delivery and
+	// the moment it stands aside.
+	if _, err := Retire(dir, "older"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "older", BinaryName)); err != nil {
+		t.Fatalf("the running version was retired: %v", err)
+	}
+}
+
+// A machine running a hand-placed host keeps the newest delivered version, because that is what a
+// rollback would reach for. Retiring it would leave the machine with no way back at all.
+func TestRetireKeepsTheNewestWhenTheRunningVersionWasNeverDelivered(t *testing.T) {
+	dir := t.TempDir()
+	base := time.Now().Add(-3 * time.Hour)
+	deliver(t, dir, "old", base)
+	deliver(t, dir, "new", base.Add(time.Hour))
+
+	removed, err := Retire(dir, "copied-by-hand")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(removed, []string{"old"}) {
+		t.Fatalf("retired %v, want only old — new is the rollback target", removed)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "new", BinaryName)); err != nil {
+		t.Fatalf("the only delivered version was retired: %v", err)
+	}
+}

packaging/launch_test.sh

@@ -187,5 +187,70 @@ sleep 1
 check "a crash is counted" "unlike a clean exit, which is not" "$(count)" "1"
 kill -TERM "$LP" 2>/dev/null; sleep 1; pkill -f "$MESH_HOST_BIN" 2>/dev/null || true
 
+# --- which version it runs (novox/hq ADR 0141) ------------------------------------------------
+#
+# Versions live side by side in directories named for them. The launcher picks one every time round
+# the loop, never once: standing aside for a successor is a clean exit, and the next turn has to run
+# what is on disk NOW — resolved once, the same binary would restart for ever and no upgrade would
+# ever take.
+
+# deliver a version as the mesh would, recording which one ran so a test can assert the choice.
+deliver() {
+	mkdir -p "$MESH_HOST_LIBEXEC/versions/$1"
+	cat > "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host" <<STUB
+#!/bin/sh
+echo "$1" >> "\$MESH_HOST_STATE_DIR/which.ran"
+exit "\${STUB_HOST_EXIT:-1}"
+STUB
+	chmod +x "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host"
+	# When it arrived is what "newest" means, so it is set rather than left to the clock.
+	touch -d "$2" "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host" "$MESH_HOST_LIBEXEC/versions/$1"
+}
+which_ran() { cat "$MESH_HOST_STATE_DIR/which.ran" 2>/dev/null || echo NONE; }
+
+# Newest is when it arrived, not how its name sorts: "1.10" orders before "1.9" by name, so ordering
+# by name would run an older host and call it an upgrade.
+setup
+deliver 1.10 "2 hours ago"
+deliver 1.9 "1 hour ago"
+"$LAUNCH" >/dev/null 2>&1 || true
+check "runs the newest delivered version" "newest is when it arrived, not how the name sorts" \
+	"$(which_ran)" "1.9"
+
+# A pin from a rollback beats the newest, or the launcher would start the failing binary again and
+# the rollback would flap.
+setup
+deliver 1.9 "2 hours ago"
+deliver 2.0 "1 hour ago"
+echo 1.9 > "$MESH_HOST_STATE_DIR/rollback-pinned"
+"$LAUNCH" >/dev/null 2>&1 || true
+check "a pinned version beats the newest" "otherwise a rollback starts the binary it just rejected" \
+	"$(which_ran)" "1.9"
+
+# A pin naming a version that is not there is ignored rather than fatal: the machine choosing for
+# itself is better than a machine that starts nothing.
+setup
+deliver 2.0 "1 hour ago"
+echo 1.9 > "$MESH_HOST_STATE_DIR/rollback-pinned"
+"$LAUNCH" >/dev/null 2>&1 || true
+check "an undeliverable pin is ignored" "a machine that starts nothing is worse than one that chooses" \
+	"$(which_ran)" "2.0"
+
+# An interrupted delivery leaves a directory with no binary in it. Treating it as the newest would
+# mean running nothing.
+setup
+deliver 1.9 "2 hours ago"
+mkdir -p "$MESH_HOST_LIBEXEC/versions/2.0-half"
+touch -d "1 minute ago" "$MESH_HOST_LIBEXEC/versions/2.0-half"
+"$LAUNCH" >/dev/null 2>&1 || true
+check "skips a version with no binary" "a directory is not a version; the binary is" \
+	"$(which_ran)" "1.9"
+
+# Nothing delivered: the host placed by hand, which is how the first one always arrives. Without this
+# the change would strand every machine in the mesh on the day it ships.
+setup
+"$LAUNCH" >/dev/null 2>&1 || true
+check "falls back to the host placed by hand" "every first host arrives this way" "$(started)" "yes"
+
 printf '\nlaunch: %d passed, %d failed\n' "$PASS" "$FAIL"
 [ "$FAIL" -eq 0 ]

packaging/launcher_is_delivered_test.go

@@ -0,0 +1,48 @@
+package packaging_test
+
+import (
+	"encoding/json"
+	"os"
+	"testing"
+)
+
+// The mesh delivers the launcher, so the manifest carries a copy of it (novox/hq 04-ISSUES/142).
+//
+// **Two copies of one script is a drift waiting to happen**, and the only reason to accept it is that
+// a file resource is written atomically — temp file, then rename — while an archive writes in place
+// with truncate. The running launcher keeps the inode it was started from and the next start picks up
+// the new one; unpacking an archive over it would truncate the file a running shell is reading.
+//
+// So: two copies, and this is the check that they are the same one.
+func TestTheManifestCarriesTheLauncherExactly(t *testing.T) {
+	onDisk, err := os.ReadFile("nox-mesh-host-launch")
+	if err != nil {
+		t.Fatal(err)
+	}
+	raw, err := os.ReadFile("../module.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var manifest struct {
+		Resources []struct {
+			ID      string `json:"id"`
+			Content string `json:"content"`
+		} `json:"resources"`
+	}
+	if err := json.Unmarshal(raw, &manifest); err != nil {
+		t.Fatal(err)
+	}
+	for _, r := range manifest.Resources {
+		if r.ID != "launcher" {
+			continue
+		}
+		if r.Content != string(onDisk) {
+			t.Fatal("the launcher the mesh would deliver is not the launcher in this repository. " +
+				"Copy packaging/nox-mesh-host-launch into module.json's `launcher` resource — the " +
+				"machines run what the manifest says, and this file is what gets reviewed")
+		}
+		return
+	}
+	t.Fatal("module.json declares no `launcher` resource, so nothing delivers the launcher and a " +
+		"delivered host version is never started")
+}

packaging/nox-mesh-host-launch

@@ -16,16 +16,51 @@ set -u
 
 STATE_DIR="${MESH_HOST_STATE_DIR:-/var/lib/mesh-host}"
 LIBEXEC="${MESH_HOST_LIBEXEC:-/usr/lib/nox-mesh-host}"
-HOST="${MESH_HOST_BIN:-/usr/bin/nox-mesh-host}"
+# The host that was placed by hand, used only when nothing has been delivered. The first host on a
+# machine always arrives this way; every one after it is delivered (novox/hq ADR 0141).
+FALLBACK="${MESH_HOST_BIN:-/usr/bin/nox-mesh-host}"
+VERSIONS="$LIBEXEC/versions"
+BINARY="nox-mesh-host"
 LIMIT="${MESH_HOST_START_LIMIT:-3}"
 BACKOFF="${MESH_HOST_BACKOFF:-5}"
 ONCE="${MESH_HOST_RUN_ONCE:-}"   # tests run one iteration; nothing else sets this
 
 ATTEMPTS="$STATE_DIR/start-attempts"
 HALTED="$STATE_DIR/halted"
+PINNED="$STATE_DIR/rollback-pinned"
 
 say() { echo "nox-mesh-host-launch: $*" >&2; }
 
+# Which host to run: the version a rollback pinned, or the most recently delivered one, or the one
+# placed by hand when nothing has been delivered (novox/hq ADR 0141).
+#
+# **Asked every time round the loop, not once.** Standing aside for a successor is a clean exit, and
+# the next turn has to run what is on disk NOW — resolving this once would restart the same binary
+# for ever and the upgrade would never take.
+#
+# Newest by when it arrived, never by how its name sorts: a version string is whatever the source was
+# described as, and those do not sort — "1.10" orders before "1.9". Ordering by name would start an
+# older host and call it an upgrade.
+pick_host() {
+	if [ -s "$PINNED" ]; then
+		pinned="$(tr -d '[:space:]' < "$PINNED" 2>/dev/null || true)"
+		if [ -n "$pinned" ] && [ -x "$VERSIONS/$pinned/$BINARY" ]; then
+			echo "$VERSIONS/$pinned/$BINARY"
+			return 0
+		fi
+		say "the pinned version '$pinned' is not delivered; ignoring the pin"
+	fi
+	# A directory with no executable in it is not a version: an interrupted delivery leaves one, and
+	# running "the newest" would then mean running nothing.
+	for candidate in $(ls -1t "$VERSIONS" 2>/dev/null || true); do
+		if [ -x "$VERSIONS/$candidate/$BINARY" ]; then
+			echo "$VERSIONS/$candidate/$BINARY"
+			return 0
+		fi
+	done
+	echo "$FALLBACK"
+}
+
 child=
 stopping=
 
@@ -95,6 +130,14 @@ while :; do
 		fi
 	fi
 
+	HOST="$(pick_host)"
+	if [ ! -x "$HOST" ]; then
+		say "no host to run: nothing delivered under $VERSIONS and $FALLBACK is not executable."
+		printf 'no host binary\n' > "$HALTED"
+		exit 0
+	fi
+	say "running $HOST"
+
 	"$HOST" run &
 	child=$!
 	status=0

packaging/nox-mesh-host-rollback

@@ -1,20 +1,29 @@
 #!/bin/sh
 # Put the host back on the last version that worked.
 #
-# novox/hq ADR 0005. This runs when nox-mesh-host will not start, so it shares no code with it
-# and calls none of it: a binary that cannot start cannot be its own recovery. POSIX sh, no
+# novox/hq ADR 0005 and ADR 0141. This runs when nox-mesh-host will not start, so it shares no code
+# with it and calls none of it: a binary that cannot start cannot be its own recovery. POSIX sh, no
 # bashisms, nothing that has to be installed.
 #
-# It is deliberately dull. Everything it does is one of: read a file, run the package manager,
-# ask the service manager to try again.
+# It is deliberately dull. Everything it does is one of: read a file, look at a directory, write a
+# file.
+#
+# **It used to reinstall a package.** It read the known-good version and asked one operating system's
+# package manager for it, out of that package manager's cache. Two things were wrong with that. No
+# machine in this mesh had the host installed as a package, so the recovery could not run on any of
+# them; and the host is built per operating system (ADR 0005), so a recovery written in one package
+# manager's terms could not run on two of the three. Versions now live side by side in directories
+# named for them, so going back is choosing a directory — which is the same on every machine.
 set -eu
 
 STATE_DIR="${MESH_HOST_STATE_DIR:-/var/lib/mesh-host}"
-PKG_CACHE="${MESH_HOST_PKG_CACHE:-/var/cache/pacman/pkg}"
-PACKAGE="${MESH_HOST_PACKAGE:-nox-mesh-host}"
+LIBEXEC="${MESH_HOST_LIBEXEC:-/usr/lib/nox-mesh-host}"
+VERSIONS="$LIBEXEC/versions"
+BINARY="nox-mesh-host"
 
 KNOWN_GOOD="$STATE_DIR/known-good"
 ATTEMPTED="$STATE_DIR/rollback-attempted"
+PINNED="$STATE_DIR/rollback-pinned"
 
 say() { echo "nox-mesh-host-rollback: $*" >&2; }
 
@@ -43,23 +52,24 @@ if [ -z "$VERSION" ]; then
 	exit 0
 fi
 
-PKG="$(ls "$PKG_CACHE"/"$PACKAGE"-"$VERSION"-*.pkg.tar.* 2>/dev/null | head -n 1 || true)"
-if [ -z "$PKG" ]; then
-	say "known-good is $VERSION and no package for it is in $PKG_CACHE."
-	say "the cache was cleaned, or that version was never installed from here."
+# The version that last worked may be the one that was placed by hand, which is not delivered and has
+# no directory. Nothing to choose, and saying so is better than pinning a version that is not there —
+# the launcher would ignore the pin and start the newest again, which is the binary that is failing.
+if [ ! -x "$VERSIONS/$VERSION/$BINARY" ]; then
+	say "known-good is $VERSION and no such version is delivered under $VERSIONS."
+	say "it was retired, or that host was placed by hand and never delivered."
 	say "cannot roll back. this node needs a person."
 	exit 1
 fi
 
-say "rolling back to $VERSION ($PKG)"
+say "rolling back to $VERSION ($VERSIONS/$VERSION/$BINARY)"
 printf '%s\n' "$VERSION" > "$ATTEMPTED"
 
-if ! pacman -U --noconfirm "$PKG"; then
-	say "the package manager refused to install $PKG."
-	exit 1
-fi
+# The pin is what stops the launcher starting the newest again. Written last, so a failure above
+# leaves the machine choosing for itself rather than pinned to something this script did not verify.
+printf '%s\n' "$VERSION" > "$PINNED"
 
-# Deliberately does NOT start anything. The launcher called this and will exec the host next,
-# so starting it here would run two. novox/hq ADR 0005 moved that responsibility; this script
-# installs a version and says so, and nothing else.
-say "rolled back to $VERSION. the launcher will start it."
+# Deliberately does NOT start anything. The launcher called this and will run the host next, so
+# starting it here would run two. novox/hq ADR 0005 moved that responsibility; this script chooses a
+# version and says so, and nothing else.
+say "pinned $VERSION. the launcher will start it."

packaging/rollback_test.sh

@@ -1,9 +1,15 @@
 #!/bin/sh
 # Tests for nox-mesh-host-rollback.
 #
-# It runs on a machine where the host will not start, which is the one moment nobody can afford
-# it to be wrong — and the one moment it is hardest to debug. So it is tested here, against a
-# real filesystem, with a stub package manager that records what it was asked to do.
+# It runs on a machine where the host will not start, which is the one moment nobody can afford it to
+# be wrong — and the one moment it is hardest to debug. So it is tested here, against a real
+# filesystem holding real delivered versions.
+#
+# **These used to stub a package manager.** The script reinstalled the known-good version with
+# `pacman -U` out of the package cache, which no machine in this mesh used and which two of the three
+# operating systems the host is built for do not have (novox/hq ADR 0141). Going back is now choosing
+# a directory, so there is nothing to stub: the thing under test is the filesystem, and a fake would
+# only assert that the fake behaves as expected (novox/hq ADR 0017).
 set -eu
 cd "$(dirname "$0")"
 SCRIPT="$PWD/nox-mesh-host-rollback"
@@ -12,26 +18,15 @@ PASS=0; FAIL=0
 setup() {
 	WORK="$(mktemp -d)"
 	export MESH_HOST_STATE_DIR="$WORK/state"
-	export MESH_HOST_PKG_CACHE="$WORK/cache"
-	export MESH_HOST_PACKAGE="nox-mesh-host"
-	mkdir -p "$MESH_HOST_STATE_DIR" "$MESH_HOST_PKG_CACHE" "$WORK/bin"
+	export MESH_HOST_LIBEXEC="$WORK/libexec"
+	mkdir -p "$MESH_HOST_STATE_DIR" "$MESH_HOST_LIBEXEC/versions"
+}
 
-	# Stubs on PATH. Not mocks of the script's own logic — the boundary is real commands, and
-	# these record the calls so a test can assert what the script asked the machine to do.
-	cat > "$WORK/bin/pacman" <<'STUB'
-#!/bin/sh
-echo "$@" >> "$MESH_HOST_STATE_DIR/pacman.calls"
-[ -n "${STUB_PACMAN_FAILS:-}" ] && exit 1
-exit 0
-STUB
-	cat > "$WORK/bin/systemctl" <<'STUB'
-#!/bin/sh
-echo "$@" >> "$MESH_HOST_STATE_DIR/systemctl.calls"
-exit 0
-STUB
-	chmod +x "$WORK/bin/pacman" "$WORK/bin/systemctl"
-	PATH="$WORK/bin:$PATH"; export PATH
-	unset STUB_PACMAN_FAILS || true
+# deliver a version the way the mesh would: a directory named for it, with the binary inside.
+deliver() {
+	mkdir -p "$MESH_HOST_LIBEXEC/versions/$1"
+	printf '#!/bin/sh\nexit 0\n' > "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host"
+	chmod +x "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host"
 }
 
 check() { # name, condition-description, actual, expected
@@ -41,32 +36,38 @@ check() { # name, condition-description, actual, expected
 
 # --- a normal rollback ---------------------------------------------------------------------
 setup
-echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
-touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst"
-"$SCRIPT" >/dev/null 2>&1
-check "installs the known-good version" "pacman is asked to install the cached package" \
-	"$(grep -c 'nox-mesh-host-1.4.2' "$MESH_HOST_STATE_DIR/pacman.calls" 2>/dev/null || echo 0)" "1"
-# It installs and stops. The launcher execs the host next, and starting it here would run two
-# (novox/hq ADR 0005).
-check "does not start anything itself" "the launcher owns starting" \
-	"$([ -f "$MESH_HOST_STATE_DIR/systemctl.calls" ] && echo started || echo not-started)" "not-started"
+deliver 1.4.2
+deliver 1.5.0
+echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good"
+RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$?
+check "pins the known-good version" "the launcher reads the pin and runs that version instead of the newest" \
+	"$(cat "$MESH_HOST_STATE_DIR/rollback-pinned" 2>/dev/null || echo MISSING)" "1.4.2"
 check "records that it rolled back" "the attempted marker holds the version" \
 	"$(cat "$MESH_HOST_STATE_DIR/rollback-attempted" 2>/dev/null || echo MISSING)" "1.4.2"
+check "succeeds" "a rollback that found its version is not a failure" "$RC" "0"
+# It chooses and stops. The launcher runs the host next, and starting it here would run two
+# (novox/hq ADR 0005).
+check "does not start anything itself" "the launcher owns starting" \
+	"$(ls "$MESH_HOST_STATE_DIR" | grep -c started || true)" "0"
+# The version it rolled back FROM is left alone: it is the newest, and retiring it is the running
+# host's job after a reconcile it completes, never a recovery's.
+check "leaves the failing version on disk" "a recovery deletes nothing" \
+	"$([ -x "$MESH_HOST_LIBEXEC/versions/1.5.0/nox-mesh-host" ] && echo present || echo gone)" "present"
 
 # --- it rolls back only once ---------------------------------------------------------------
 setup
-echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
-echo "1.4.2" > "$MESH_HOST_STATE_DIR/rollback-attempted"
-touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst"
-"$SCRIPT" >/dev/null 2>&1
+deliver 1.4.2
+echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good"
+echo 1.4.2 > "$MESH_HOST_STATE_DIR/rollback-attempted"
+"$SCRIPT" >/dev/null 2>&1 || true
 check "does not roll back twice" "a second failure is the machine, not the binary" \
-	"$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called"
+	"$([ -e "$MESH_HOST_STATE_DIR/rollback-pinned" ] && echo pinned || echo untouched)" "untouched"
 
 # --- nothing to roll back to ---------------------------------------------------------------
 setup
-set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e
+RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$?
 check "no known-good: does nothing" "a host that never reconciled has no version to return to" \
-	"$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called"
+	"$([ -e "$MESH_HOST_STATE_DIR/rollback-attempted" ] && echo attempted || echo untouched)" "untouched"
 # The exit code is asserted from a real run, not from a literal. An earlier version of this
 # compared "0" to "0" and could not fail — which hid an injected fault that made the script die
 # here instead of returning cleanly.
@@ -74,23 +75,29 @@ check "no known-good: exits zero" "an installation failure is not a rollback fai
 
 setup
 printf '   \n' > "$MESH_HOST_STATE_DIR/known-good"
-"$SCRIPT" >/dev/null 2>&1
-check "blank known-good: refuses to guess" "installing nothing and reporting success is the fault this prevents" \
-	"$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called"
+"$SCRIPT" >/dev/null 2>&1 || true
+check "blank known-good: refuses to guess" "pinning nothing and reporting success is the fault this prevents" \
+	"$([ -e "$MESH_HOST_STATE_DIR/rollback-pinned" ] && echo pinned || echo untouched)" "untouched"
 
-# --- the cache was cleaned ------------------------------------------------------------------
+# --- the known-good version is not delivered -------------------------------------------------
+# It was retired, or that host was placed on the machine by hand and never delivered — which is how
+# every first host arrives. Pinning it anyway would have the launcher ignore the pin and start the
+# newest again, which is the binary that is failing.
 setup
-echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
-set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e
-check "missing package: fails loudly" "cannot roll back, and says so rather than reporting success" "$RC" "1"
+deliver 1.5.0
+echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good"
+RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$?
+check "version not delivered: fails loudly" "cannot roll back, and says so rather than reporting success" "$RC" "1"
+check "version not delivered: pins nothing" "a pin the launcher would ignore is worse than none" \
+	"$([ -e "$MESH_HOST_STATE_DIR/rollback-pinned" ] && echo pinned || echo untouched)" "untouched"
 
-# --- the package manager refuses -------------------------------------------------------------
+# --- a version directory with no binary in it ------------------------------------------------
+# An interrupted delivery leaves one. Pinning it would start nothing.
 setup
-echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
-touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst"
-STUB_PACMAN_FAILS=1 ; export STUB_PACMAN_FAILS
-set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e
-check "pacman fails: exits non-zero" "a failed rollback is a failure the launcher must see" "$RC" "1"
+mkdir -p "$MESH_HOST_LIBEXEC/versions/1.4.2"
+echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good"
+RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$?
+check "half-delivered version: fails loudly" "a directory is not a version; the binary is" "$RC" "1"
 
 printf '\nrollback: %d passed, %d failed\n' "$PASS" "$FAIL"
 [ "$FAIL" -eq 0 ]