Merge pull request 'A converged machine can speak unasked' (#48) from fix/a-converged-machine-can-speak-unasked into main

A reconcile is otherwise silent, and the condition deciding when it speaks asked
only what an adopted node reports: what it holds, and the firewall it found. A
converged node has neither, so it could speak only in reply to a declaration —
and the mesh composes no declaration for a node that has not said which links
face outside (ADR 0140). A machine waiting for a push that was waiting for the
machine.

Measured, not predicted: after the control plane learnt to read the links, the
control node recorded its own and the three converged machines sat silent while
their filters were refused.

The condition is now a named predicate, because as an inline expression nothing
could test it — which is why the gap shipped.

cmd/mesh-host/main.go

@@ -33,6 +33,7 @@ import (
 	"github.com/novox/mesh-host/internal/identity"
 	"github.com/novox/mesh-host/internal/inventory"
 	"github.com/novox/mesh-host/internal/link"
+	"github.com/novox/mesh-host/internal/outward"
 	"github.com/novox/mesh-host/internal/profile"
 	"github.com/novox/mesh-host/internal/reachable"
 	"github.com/novox/mesh-host/internal/store"
@@ -600,6 +601,20 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
 					"  a rollback would have nothing to return to.\n", version, err)
 		}
 	}
+	// And retire what is older than this version's predecessor, on the same evidence known-good is
+	// written on (novox/hq ADR 0141). The predecessor stays, because it is exactly what a rollback
+	// starts; everything before it has no reader. Never this version, whatever the answer.
+	//
+	// A failure is said and does not fail the apply, for the reason above: what is lost is disk, and
+	// hiding it would make a machine quietly fill up.
+	if version != "" {
+		if retired, err := upgrade.Retire(upgrade.VersionsDir(""), version); err != nil {
+			fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not retire an older host: %v\n", err)
+		} else if len(retired) > 0 {
+			fmt.Fprintf(os.Stderr, "mesh-host: retired the host version(s) %s\n",
+				strings.Join(retired, ", "))
+		}
+	}
 	// And tell the launcher this start worked. Without it the counter only climbs, and a node
 	// that has been up for months rolls itself back on its third ordinary restart.
 	if err := upgrade.ClearAttempts(upgrade.AttemptsPath(opts.state)); err != nil {
@@ -978,12 +993,31 @@ func runLink(ctx context.Context, opts options) error {
 	sched := apply.NewScheduler(apply.SystemClock(), apply.ExecRunner, say)
 	go sched.Run(ctx)
 
+	// **Standing aside for a successor happens between reconciles and nowhere else** (novox/hq ADR
+	// 0141). A host that stood aside mid-apply is the half-configured machine this host exists to
+	// prevent, so the question is asked after an apply has finished and the answer is a clean exit —
+	// which the launcher already reads as "run whatever is on disk now".
+	aside, standAside := context.WithCancel(ctx)
+	defer standAside()
+	stoodAside := false
+
 	applier := func(ctx context.Context, raw, signature []byte) link.Report {
 		report := applyAndKeep(ctx, opts, raw, &store.Declared{Declaration: raw, Signature: signature}, sched, say)
 		// **A declaration may carry this machine's membership for another bus.** It arrives as a
 		// sealed file like any secret, and is read after the rest has applied so the bus it names is
 		// standing before this machine leaves the one it is on (novox/hq design 28, task 5.2).
 		adoptDeliveredMembership(identity.Path(opts.state), &mine, say)
+
+		switch next, waiting, err := upgrade.Successor(upgrade.VersionsDir(""), version); {
+		case err != nil:
+			// Said, not fatal. A host that cannot read the delivered versions is still running this
+			// machine correctly; what it has lost is the ability to be replaced.
+			say(fmt.Sprintf("cannot tell whether a newer host is delivered: %v", err))
+		case waiting:
+			say(fmt.Sprintf("host %s is delivered; standing aside so the launcher runs it", next.Version))
+			stoodAside = true
+			standAside()
+		}
 		return report
 	}
 
@@ -1014,7 +1048,7 @@ func runLink(ctx context.Context, opts options) error {
 		}}
 	})
 
-	return link.HoldRoused(ctx, link.Membership{
+	held := link.HoldRoused(aside, link.Membership{
 		Node:        mine.Node,
 		Broker:      mine.Membership.Broker,
 		Fingerprint: mine.Membership.Fingerprint,
@@ -1022,6 +1056,13 @@ func runLink(ctx context.Context, opts options) error {
 		Transport:   mine.Membership.Transport,
 		Signer:      mine.Membership.Signer,
 	}, applier, say, opts.timeout, rousedBySignal(ctx), outbox)
+	// **Cleanly**, or the launcher counts standing aside as a crash and rolls the new host back
+	// before it has run once. The context this returns on was cancelled deliberately, so its error
+	// is not a fault to report.
+	if stoodAside {
+		return nil
+	}
+	return held
 }
 
 // adoptionWatch remembers what the node last said about what it holds and its firewall, so a
@@ -1036,7 +1077,12 @@ type adoptionWatch struct {
 // is what the controller previews a flip from, so a port that opens or closes between deliveries
 // must reach it too (novox/hq ADR 0100).
 func adoptionFingerprint(r link.Report) string {
-	parts := []string{"firewall=" + r.Firewall}
+	// **Which links face outside is part of it, though it is not about adoption** (novox/hq ADR
+	// 0140). The mesh composes no filter for a machine that has not said, so a machine whose links
+	// changed — or which has only just learnt to say — has to say so without being asked. Left out,
+	// it could only speak when a declaration arrived, and a declaration cannot be composed until it
+	// has spoken: a machine waiting for a push that is waiting for the machine.
+	parts := []string{"firewall=" + r.Firewall, "outward=" + strings.Join(r.Outward, ",")}
 	for _, h := range r.Held {
 		parts = append(parts, "held "+h.ID+"="+h.Changed)
 	}
@@ -1044,7 +1090,7 @@ func adoptionFingerprint(r link.Report) string {
 		parts = append(parts, fmt.Sprintf("reach %s %s:%d %s %v %d", reach.Protocol, reach.Address,
 			reach.Port, reach.By, reach.Published, reach.ContainerPort))
 	}
-	sort.Strings(parts[1:])
+	sort.Strings(parts[2:])
 	return strings.Join(parts, "\n")
 }
 
@@ -1152,7 +1198,17 @@ func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, s
 		// A reconcile is otherwise silent. On an adopted node it speaks when what it holds or
 		// its firewall changed, because that is how a predecessor still writing is caught
 		// (novox/hq ADR 0100); publish decides whether anything did.
-		if publish != nil && report.Refused == "" && (len(report.Held) > 0 || report.Firewall != "") {
+		//
+		// **And on any node, when it can say which links face outside** (novox/hq ADR 0140). A
+		// converged node holds nothing and found no firewall, so this gate closed on it and the
+		// node could speak only in reply to a declaration — while the mesh composes no declaration
+		// for a node that has not said which links face outside. A machine waiting for a push that
+		// was waiting for the machine, and measured: three converged machines sat silent while the
+		// control plane refused to send them a filter.
+		//
+		// Offered, not published: whether it is news is still the watch's to decide, so an
+		// unchanged answer costs one comparison every reconcile and nothing on the bus.
+		if publish != nil && worthSaying(report) {
 			publish(report)
 		}
 		switch {
@@ -1164,6 +1220,23 @@ func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, s
 	}
 }
 
+// worthSaying is whether a reconcile's report carries anything the mesh needs to hear unasked.
+//
+// **Named rather than written into the loop**, so the rule can be tested. It was a condition inline
+// and it was wrong in a way nothing could catch: it asked only what an adopted node reports, so a
+// converged node — which holds nothing and found no firewall — could speak only in reply to a
+// declaration, while the mesh composes no declaration for a node that has not said which links face
+// outside (novox/hq ADR 0140). Three machines sat silent waiting for a push that was waiting for them.
+//
+// A refused report says nothing about the machine, so it is not news; the refusal is said on the
+// console where a person reads it.
+func worthSaying(report link.Report) bool {
+	if report.Refused != "" {
+		return false
+	}
+	return len(report.Held) > 0 || report.Firewall != "" || len(report.Outward) > 0
+}
+
 // applyDeclared applies a declaration that has already been proved to come from the mesh.
 //
 // Signature checking happens before this is called, in the link. By the time anything here runs,
@@ -1262,7 +1335,16 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
 		sched.Sync(declared, held)
 	}
 
-	report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw)}
+	report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw), Host: version}
+	// Which of this machine's links face outside, for the filter the mesh writes around them
+	// (novox/hq ADR 0140). Reported whatever the node's mode: a converged node's filter needs it,
+	// and an adopted one becomes converged without a further round trip. A machine that cannot read
+	// its own routing table says nothing rather than guessing, and is sent no filter.
+	if links, err := outward.Links(""); err != nil {
+		fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not read which links face outside: %v\n", err)
+	} else {
+		report.Outward = links
+	}
 	// What this node found and holds, its firewall, and what is reachable on it — so an adopted
 	// node never reads as converged (novox/hq ADR 0100).
 	for _, h := range updated.Held {

cmd/mesh-host/main_test.go

@@ -501,3 +501,70 @@ func TestReconcileAfterAControllerDeclarationDoesNotReapplyTheBundle(t *testing.
 		t.Errorf("with nothing said, reconcile did not reach for the carried bundle: %v", err)
 	}
 }
+
+// **A machine says which links face outside without being asked.**
+//
+// The mesh composes no filter for a machine that has not said (novox/hq ADR 0140), and a machine only
+// speaks unasked when this fingerprint changes. Left out of it, a machine that has just learnt to say
+// could speak only when a declaration arrived — and a declaration cannot be composed until it has
+// spoken. A machine waiting for a push that is waiting for the machine.
+func TestANewOutwardLinkIsSaidUnasked(t *testing.T) {
+	w := &adoptionWatch{}
+	first := link.Report{Firewall: "none"}
+	if !w.differs(first) {
+		t.Fatal("the first report should differ from nothing")
+	}
+	w.said(first)
+
+	// Only the links changed, and nothing about adoption.
+	learnt := link.Report{Firewall: "none", Outward: []string{"eth0"}}
+	if !w.differs(learnt) {
+		t.Fatal("a machine that has just learnt which links face outside would never say so, " +
+			"and could then never be sent a filter")
+	}
+	w.said(learnt)
+	if w.differs(link.Report{Firewall: "none", Outward: []string{"eth0"}}) {
+		t.Fatal("the same links are reported as a change, so the machine would speak on every reconcile")
+	}
+
+	// And a link that changes — a laptop moving from a cable to a radio — is said too, because the
+	// filter is written around the old one until it is.
+	if !w.differs(link.Report{Firewall: "none", Outward: []string{"wlan0"}}) {
+		t.Fatal("a changed outward link is not said, so the filter stays written around the old one")
+	}
+}
+
+// **A converged machine can say which links face outside, unasked.**
+//
+// A reconcile is otherwise silent, and the condition deciding when it speaks asked only what an
+// adopted node reports — what it holds, and the firewall it found. A converged node has neither, so
+// it could speak only in reply to a declaration, and the mesh composes no declaration for a node
+// that has not said which links face outside (novox/hq ADR 0140). Measured: three converged machines
+// sat silent while the control plane refused to send them a filter.
+func TestAConvergedMachineSaysItsOutwardLinksUnasked(t *testing.T) {
+	// A converged node's reconcile: nothing held, no found firewall, and the links it can see.
+	converged := link.Report{Outward: []string{"eth0"}}
+	if !worthSaying(converged) {
+		t.Fatal("a converged machine cannot say which links face outside, so it can never be " +
+			"sent a filter — a machine waiting for a push that is waiting for the machine")
+	}
+
+	// An adopted node's reasons still hold, because that is how a predecessor still writing is caught.
+	if !worthSaying(link.Report{Firewall: "ufw"}) {
+		t.Fatal("an adopted machine no longer says which firewall it found")
+	}
+	if !worthSaying(link.Report{Held: []link.Held{{ID: "a-file"}}}) {
+		t.Fatal("an adopted machine no longer says what it holds")
+	}
+
+	// And a reconcile with nothing to say stays silent, or every machine speaks every five minutes
+	// about nothing.
+	if worthSaying(link.Report{}) {
+		t.Fatal("a reconcile with nothing to say speaks anyway")
+	}
+
+	// A refused report says nothing about the machine; the refusal is for the console.
+	if worthSaying(link.Report{Outward: []string{"eth0"}, Refused: "not for this node"}) {
+		t.Fatal("a refused report is offered as news about the machine")
+	}
+}

examples/foundation-first-node-nats.lock

@@ -152,7 +152,7 @@
       "type": "file",
       "path": "/var/lib/mesh-bus-conf/accounts.conf",
       "mode": "0600",
-      "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n  MESH {\n    jetstream: enabled\n    users = [\n      { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n        publish: { allow: [\"$JS.API.>\", \"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"_INBOX.enrol.>\", \"mesh.control.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\"] }\n        subscribe: { allow: [\"$JS.API.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.seat.mesh-build-machine.event.built\"] }\n        allow_responses: { max: 1, ttl: \"1m\" }\n      } }\n    ]\n  }\n}\n"
+      "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n  MESH {\n    jetstream: enabled\n    users = [\n      { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n        publish: { allow: [\"$JS.API.>\", \"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"_INBOX.enrol.>\", \"mesh.control.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\"] }\n        subscribe: { allow: [\"$JS.API.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.seat.mesh-build-machine.event.built\"] }\n        allow_responses: { max: 1, ttl: \"1m\" }\n      } }\n    ]\n  }\n}\n"
     },
     {
       "id": "broker",

internal/apply/apply.go

@@ -320,6 +320,9 @@ func ApplyKeeping(
 	// is a different thing — one is "this machine could not do it", the other is "this was never
 	// a declaration", and they are fixed in different places.
 	var failures []*Error
+	// Modules whose own step did not complete. What follows *within such a module* is not attempted;
+	// the rest of the machine is (novox/hq ADR 0136).
+	gated := map[string]bool{}
 	for i, resource := range ordered {
 		if !orphansRemoved && i == guardFirst {
 			// **Only a guard that is up may let the filter go.** Removing the derived filter's
@@ -337,6 +340,17 @@ func ApplyKeeping(
 				return report, known, err
 			}
 		}
+		// **A module whose step did not complete is skipped from there on** (novox/hq ADR 0136).
+		// Reported rather than passed over in silence: "not attempted" and "nothing to do" are
+		// different answers, and only one of them is somebody's to fix.
+		if module, ours := moduleOf(resource.Identity()); ours && gated[module] {
+			report.Outcomes = append(report.Outcomes, Outcome{
+				ID: resource.Identity(), Type: string(resource.Kind()), Target: resource.Target(),
+				Action: "skipped", Detail: "a step this module declares did not complete",
+			})
+			continue
+		}
+
 		// **On an adopted node, what is found is kept until its module is taken** (novox/hq ADR
 		// 0100, ADR 0103). Before anything is applied: whatever of a module not yet taken is
 		// present with no record of this host making it — or would reach what is — is held as it
@@ -465,9 +479,26 @@ func ApplyKeeping(
 				gates = true
 			}
 			if gates {
+				failed.Gated = true
+				// **A module's step gates that module, not the machine** (novox/hq ADR 0136).
+				//
+				// Stopping the whole apply is what this loop's own comment above calls holding a
+				// machine hostage, and it was already rejected for every other shape (04-ISSUES/011).
+				// A step exists to make something true before the next thing in *its module* needs it
+				// — a store seeded before the broker starts, a schema prepared before the version
+				// that needs it runs — so that is exactly how far the gate reaches. Everything else
+				// on the machine is independent state and is attempted.
+				//
+				// An action still gates the machine: the bootstrap is a row of them, each making the
+				// next possible, and they belong to no module.
+				if module, ours := moduleOf(resource.Identity()); ours {
+					gated[module] = true
+					log(fmt.Sprintf("  gated %s.*: a step it declares did not complete, so the rest "+
+						"of it was not attempted", module))
+					continue
+				}
 				failed.Done = report
 				failed.Others = len(failures) - 1
-				failed.Gated = true
 				return report, known, failed
 			}
 			continue
@@ -1470,6 +1501,20 @@ func containerSpecReading(r *declaration.Container, declares, reads map[string]s
 	for _, a := range r.Args {
 		b.WriteString("arg " + a + "\n")
 	}
+	// **The mesh's names are part of what a container is** (novox/hq 04-ISSUES/135). A container
+	// resolves every other machine and every public name through the entries the mesh gives it at
+	// creation, and nothing re-reads them afterwards — so a container left alone when the roster
+	// moved is one that cannot reach anything by name, for ever, while every check reports it
+	// running. That is exactly what happened when this mesh's overlay range changed: one container
+	// whose image and files never changed kept an address five days out of date and restarted
+	// 2286 times against a database it could no longer find.
+	//
+	// Sorted, so the digest does not move for a reordering nobody made.
+	hosts := append([]string(nil), r.Hosts...)
+	sort.Strings(hosts)
+	for _, h := range hosts {
+		b.WriteString("host " + h + "\n")
+	}
 	// The resolver and address are part of what was declared: a container whose dns or ip moved
 	// is a different container, or the fields could never reach one that already ran — which is
 	// exactly how their first deployment silently changed nothing.
@@ -2247,3 +2292,25 @@ func meshMadeUnits(known store.State) map[string]bool {
 	}
 	return made
 }
+
+// moduleOf is the module a declared resource belongs to.
+//
+// The mesh composes a module's resource ids as `<module>.<its own id>`, and **a module's name may
+// contain a dot** — `novox.be` is one on this mesh — while a resource's own id never does. So the
+// owner is everything before the *last* dot; reading to the first one would make `novox.be.server`
+// belong to a module called "novox", and a gate would then skip whatever else happened to start that
+// way.
+//
+// False for what the mesh declares in its own right: the foundation's resources carry no dot at all,
+// and the adoption's are named for the mesh rather than for a module. Both belong to no module, and
+// their gate is therefore the machine's.
+func moduleOf(identity string) (string, bool) {
+	if strings.HasPrefix(identity, declaration.AdoptionPrefix) {
+		return "", false
+	}
+	at := strings.LastIndex(identity, ".")
+	if at <= 0 {
+		return "", false
+	}
+	return identity[:at], true
+}

internal/apply/mesh_names_test.go

@@ -0,0 +1,54 @@
+package apply
+
+import (
+	"testing"
+
+	"github.com/novox/mesh-host/internal/declaration"
+)
+
+// **A container's mesh names are part of what it is** (novox/hq 04-ISSUES/135).
+//
+// A container resolves every machine and every public name through the entries it was given when it
+// was created, and nothing re-reads them. So a container the host leaves alone because nothing else
+// about it changed is a container that cannot reach anything by name — for ever, while every check
+// reports it running. That is what happened when this mesh's overlay range moved: one container kept
+// an address five days out of date and restarted 2286 times against a database it could no longer
+// find, and the host compared everything about it except that.
+func TestAContainersMeshNamesAreComparedLikeTheRestOfIt(t *testing.T) {
+	was := &declaration.Container{
+		Name: "umami", Image: "ghcr.io/example/umami@sha256:" + zeros(64),
+		Hosts: []string{"novox.internal:10.42.0.1", "umami.novox.be:10.42.0.1"},
+	}
+	moved := &declaration.Container{
+		Name: was.Name, Image: was.Image,
+		Hosts: []string{"novox.internal:10.10.0.1", "umami.novox.be:10.10.0.1"},
+	}
+	if containerSpecReading(was, nil, nil) == containerSpecReading(moved, nil, nil) {
+		t.Fatal("a container whose mesh names moved compares equal, so it is never recreated")
+	}
+
+	// And the order they arrive in is not a change: the digest must not move for a reordering
+	// nobody made.
+	reordered := &declaration.Container{
+		Name: moved.Name, Image: moved.Image,
+		Hosts: []string{moved.Hosts[1], moved.Hosts[0]},
+	}
+	if containerSpecReading(moved, nil, nil) != containerSpecReading(reordered, nil, nil) {
+		t.Fatal("the same names in another order read as a different container")
+	}
+
+	// A container the mesh gives no names is unaffected, so nothing is recreated for a field it
+	// does not set.
+	plain := &declaration.Container{Name: "plex", Image: was.Image}
+	if containerSpecReading(plain, nil, nil) == containerSpecReading(was, nil, nil) {
+		return // different for other reasons, which is fine
+	}
+}
+
+func zeros(n int) string {
+	out := make([]byte, n)
+	for i := range out {
+		out[i] = '0'
+	}
+	return string(out)
+}

internal/apply/runonce_test.go

@@ -316,3 +316,85 @@ func TestAContainerNamingARunOnceStepIsRecreatedWhenItRan(t *testing.T) {
 		t.Errorf("the recreation did not name the step as its reason: %+v", server)
 	}
 }
+
+func TestAFailedStepGatesItsModuleAndNotTheMachine(t *testing.T) {
+	// **The blast radius of a step is its module** (novox/hq ADR 0136). A step exists to make
+	// something true before the next thing in its own module needs it — a store seeded before the
+	// broker starts, a schema prepared before the version that needs it runs. Stopping the whole
+	// apply is what this host's own loop calls holding a machine hostage, and it was already
+	// rejected for every other shape (04-ISSUES/011): a module whose database is briefly
+	// unreachable must not stop every module declared after it.
+	var startedNames []string
+	run := func(ctx context.Context, name string, args ...string) (string, error) {
+		switch args[0] {
+		case "info":
+			return "27.0\n", nil
+		case "container":
+			return "false\t\n", errors.New("no such container")
+		case "run":
+			startedNames = append(startedNames, nameOf(args))
+			if nameOf(args) == "catalogue-prepare" {
+				return "", errors.New("exit status 1") // the schema could not be reached
+			}
+			return "deadbeef\n", nil
+		case "rm":
+			return "", nil
+		}
+		return "", nil
+	}
+	d := parseTrusted(t, `{"declaration":1,"resources":[
+	  {"id":"mesh-catalog.runtime-prepare","type":"container","name":"catalogue-prepare","image":"`+pinned+`","run-once":true},
+	  {"id":"mesh-catalog.runtime","type":"container","name":"catalogue","image":"`+pinned+`"},
+	  {"id":"gitea.server","type":"container","name":"forge","image":"`+pinned+`"}
+	]}`)
+
+	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
+	if err == nil {
+		t.Fatal("a failed step was not reported as a failure")
+	}
+	started := map[string]bool{}
+	for _, n := range startedNames {
+		started[n] = true
+	}
+	if started["catalogue"] {
+		t.Error("the module's own workload ran although its step did not complete")
+	}
+	if !started["forge"] {
+		t.Error("another module was not attempted, so one module's step held the machine hostage")
+	}
+	// And the machine's own account says which was not attempted, rather than leaving it to be
+	// inferred from silence.
+	var skipped string
+	for _, o := range report.Outcomes {
+		if o.Action == "skipped" {
+			skipped = o.ID
+		}
+	}
+	if skipped != "mesh-catalog.runtime" {
+		t.Errorf("the report does not say what was not attempted: %q", skipped)
+	}
+}
+
+func TestAResourcesOwnerIsReadToTheLastDot(t *testing.T) {
+	// **A module's name may contain a dot.** `novox.be` is one on this mesh, so reading a resource's
+	// owner to the first dot would make its resources belong to something called "novox" — and a gate
+	// would skip whatever else happened to start that way. A resource's own id never contains one,
+	// which is what makes the last dot the boundary.
+	for identity, want := range map[string]string{
+		"novox.be.server":              "novox.be",
+		"mesh-catalog.runtime-prepare": "mesh-catalog",
+		"gitea.admin-bootstrap":        "gitea",
+	} {
+		got, ours := moduleOf(identity)
+		if !ours || got != want {
+			t.Errorf("%q belongs to %q (%v), want %q", identity, got, ours, want)
+		}
+	}
+	// What the mesh declares in its own right belongs to no module: the foundation's resources carry
+	// no dot, and the adoption's are the mesh's.
+	for _, identity := range []string{"container-runtime", "store-ready", "adoption.guard", ".server"} {
+		if _, ours := moduleOf(identity); ours {
+			t.Errorf("%q was read as a module's", identity)
+		}
+	}
+}

internal/link/messages.go

@@ -110,6 +110,27 @@ type Report struct {
 	// and the mesh's up in its place, and where the found configuration's original was kept.
 	Tunnel *CarriedTunnel `json:"tunnel,omitempty"`
 
+	// Host is the version of the host that produced this report (novox/hq ADR 0141).
+	//
+	// Without it nothing can say a machine is behind, so "every machine current with its source"
+	// could not include the host — the one component the mesh did not deliver. It is a fact the
+	// machine states about itself, like the firewall it found and the links that face outside.
+	Host string `json:"host,omitempty"`
+
+	// Outward is the links on this machine that face outside it — the ones carrying a default
+	// route (novox/hq ADR 0140). Every node reports it, adopted or converged, because a converged
+	// node's filter is written around it.
+	//
+	// **It replaces a list of addresses.** The filter used to block everything passing through the
+	// machine and then allow the machine's own containers back by naming the ranges they sit on.
+	// A range describes one machine and goes stale in silence; the link carrying the default route
+	// is read afresh on every report and does not change when a module is added or removed.
+	//
+	// Empty means this machine has no route off itself. The mesh then composes no filter for it and
+	// leaves the one it has, rather than writing a rule around a link with no name — a rule set
+	// that does not load is a machine filtering nothing while its unit reports success.
+	Outward []string `json:"outward,omitempty"`
+
 	// Rekey is this node taking a found tunnel's key as its overlay key after enrolment (novox/hq
 	// ADR 0105). Not an account of the machine: a report carrying one says nothing else.
 	Rekey *Rekey `json:"rekey,omitempty"`

internal/outward/links.go

@@ -0,0 +1,147 @@
+// Package outward reads which of this machine's links face outside it (novox/hq ADR 0140).
+//
+// The filter the mesh derives constrains traffic arriving from outside the machine and says nothing
+// about traffic that did not. To write that rule the mesh has to know which links "outside" arrives
+// on, and that is a thing only the machine can say — so it says it, once per report, the way it
+// already reports the kind of firewall it found and the tunnel it carried.
+//
+// **It replaces a list of addresses.** The filter used to allow the machine's own containers back
+// through by naming the address ranges they sit on: two ranges fixed in the control plane's source
+// and the rest typed by an operator. A range describes one machine and goes stale silently
+// (novox/hq 04-ISSUES/137 and /141). A link that carries the default route is a fact the machine
+// reads afresh every time, and it does not change when a module is added or removed.
+//
+// It reads the kernel's routing tables directly rather than asking a program. A module naming a
+// program the machine does not have is how the mesh already reported success while doing nothing
+// (novox/hq 04-ISSUES/136), and every machine has /proc.
+package outward
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// ProcNet is where the kernel publishes its routing tables. A parameter so a test can hold a
+// routing table without one.
+const ProcNet = "/proc/net"
+
+// Links are the interfaces carrying a default route, for both address families, sorted and without
+// repeats.
+//
+// A machine may have more than one: a laptop with a cable and a radio has two, and both face
+// outside. A machine with none — no route off itself — returns nothing, and the mesh refuses to
+// compose a filter for it rather than writing a rule around a link with no name, which would be a
+// rule set that does not load and a machine filtering nothing while its unit reports success.
+func Links(procNet string) ([]string, error) {
+	if procNet == "" {
+		procNet = ProcNet
+	}
+	seen := map[string]bool{}
+
+	four, err := defaultsV4(filepath.Join(procNet, "route"))
+	if err != nil {
+		return nil, err
+	}
+	six, err := defaultsV6(filepath.Join(procNet, "ipv6_route"))
+	if err != nil {
+		return nil, err
+	}
+	for _, name := range append(four, six...) {
+		if name != "" && name != "lo" {
+			seen[name] = true
+		}
+	}
+
+	out := make([]string, 0, len(seen))
+	for name := range seen {
+		out = append(out, name)
+	}
+	sort.Strings(out)
+	return out, nil
+}
+
+// defaultsV4 reads /proc/net/route, whose columns are
+//
+//	Iface Destination Gateway Flags RefCnt Use Metric Mask ...
+//
+// with addresses in hexadecimal. A default route is destination zero with mask zero — the mask
+// matters, because a route to the zero address with a real mask is not a default route.
+func defaultsV4(path string) ([]string, error) {
+	lines, err := rows(path)
+	if err != nil {
+		return nil, err
+	}
+	var out []string
+	for _, fields := range lines {
+		if len(fields) < 8 {
+			continue
+		}
+		if isZeroHex(fields[1]) && isZeroHex(fields[7]) {
+			out = append(out, fields[0])
+		}
+	}
+	return out, nil
+}
+
+// defaultsV6 reads /proc/net/ipv6_route, whose columns are
+//
+//	dest destprefix src srcprefix nexthop metric refcnt use flags iface
+//
+// A default route is the zero destination with a zero prefix length.
+func defaultsV6(path string) ([]string, error) {
+	lines, err := rows(path)
+	if err != nil {
+		return nil, err
+	}
+	var out []string
+	for _, fields := range lines {
+		if len(fields) < 10 {
+			continue
+		}
+		if isZeroHex(fields[0]) && isZeroHex(fields[1]) {
+			out = append(out, fields[9])
+		}
+	}
+	return out, nil
+}
+
+// rows reads a routing table into fields per line, skipping a header and blank lines. A table that
+// is not there is not an error: a machine without the second address family has no file for it,
+// and that is not a machine that cannot be filtered.
+func rows(path string) ([][]string, error) {
+	file, err := os.Open(path)
+	if os.IsNotExist(err) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("cannot read the routing table at %s: %w", path, err)
+	}
+	defer file.Close()
+
+	var out [][]string
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" || strings.HasPrefix(line, "Iface") {
+			continue
+		}
+		out = append(out, strings.Fields(line))
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("cannot read the routing table at %s: %w", path, err)
+	}
+	return out, nil
+}
+
+// isZeroHex is whether a hexadecimal field is all zeroes, whatever its width — the v4 table writes
+// eight digits and the v6 table thirty-two, and a prefix length is two.
+func isZeroHex(field string) bool {
+	if field == "" {
+		return false
+	}
+	return strings.Trim(strings.ToLower(field), "0") == ""
+}

internal/outward/links_test.go

@@ -0,0 +1,120 @@
+package outward
+
+import (
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+)
+
+// A routing table as the kernel writes it: a default route, a route to the zero address that is not
+// one, and a route on the loopback. Only the default route's link faces outside.
+const routeV4 = `Iface	Destination	Gateway 	Flags	RefCnt	Use	Metric	Mask		MTU	Window	IRTT
+enp9s0	00000000	01FEA8C0	0003	0	0	100	00000000	0	0	0
+docker0	000011AC	00000000	0001	0	0	0	0000FFFF	0	0	0
+enp9s0	00000000	00000000	0001	0	0	100	00FFFFFF	0	0	0
+lo	00000000	00000000	0003	0	0	0	00000000	0	0	0
+`
+
+const routeV6 = `00000000000000000000000000000000 00 00000000000000000000000000000000 00 fe800000000000000000000000000001 00000400 00000001 00000000 00000003     wlan0
+fd0000000000000000000000000000000 40 00000000000000000000000000000000 00 00000000000000000000000000000000 00000100 00000000 00000000 00000001    enp9s0
+`
+
+func write(t *testing.T, dir, name, body string) {
+	t.Helper()
+	if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestLinksAreTheOnesCarryingADefaultRoute(t *testing.T) {
+	dir := t.TempDir()
+	write(t, dir, "route", routeV4)
+	write(t, dir, "ipv6_route", routeV6)
+
+	got, err := Links(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// The cable from the v4 table and the radio from the v6 one. Not docker0, whose route is not a
+	// default; not the loopback, which faces nothing; and not the v6 route with a real prefix.
+	want := []string{"enp9s0", "wlan0"}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("outward links are %v, want %v", got, want)
+	}
+}
+
+// A route to the zero address with a real mask is not a default route. Trusting the destination
+// alone would name every link with such a route as facing outside, and a filter that treats an
+// internal bridge as outward constrains this machine's own guests — the fault ADR 0140 removes.
+func TestAZeroDestinationWithAMaskIsNotADefaultRoute(t *testing.T) {
+	dir := t.TempDir()
+	write(t, dir, "route", `Iface	Destination	Gateway 	Flags	RefCnt	Use	Metric	Mask		MTU	Window	IRTT
+br-abc	00000000	00000000	0001	0	0	0	00FFFFFF	0	0	0
+`)
+	got, err := Links(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 0 {
+		t.Fatalf("outward links are %v, want none", got)
+	}
+}
+
+// A machine with no route off itself says so, rather than guessing. The mesh refuses to compose a
+// filter for it; a rule written around a link with no name does not load, and a rule set that does
+// not load is a machine filtering nothing while its unit reports success.
+func TestNoDefaultRouteIsNoLinks(t *testing.T) {
+	dir := t.TempDir()
+	write(t, dir, "route", "Iface\tDestination\tGateway \tFlags\tRefCnt\tUse\tMetric\tMask\t\tMTU\tWindow\tIRTT\n")
+	got, err := Links(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 0 {
+		t.Fatalf("outward links are %v, want none", got)
+	}
+}
+
+// A machine without the second address family has no file for it. That is not a machine that cannot
+// be filtered, so a missing table is read as no routes rather than as a failure.
+func TestAMissingTableIsNotAFailure(t *testing.T) {
+	dir := t.TempDir()
+	write(t, dir, "route", routeV4)
+	got, err := Links(dir)
+	if err != nil {
+		t.Fatalf("a missing v6 table should not fail: %v", err)
+	}
+	if !reflect.DeepEqual(got, []string{"enp9s0"}) {
+		t.Fatalf("outward links are %v, want [enp9s0]", got)
+	}
+}
+
+// The same link carrying a default route in both families is reported once.
+func TestALinkIsReportedOnce(t *testing.T) {
+	dir := t.TempDir()
+	write(t, dir, "route", routeV4)
+	write(t, dir, "ipv6_route",
+		"00000000000000000000000000000000 00 00000000000000000000000000000000 00 "+
+			"fe800000000000000000000000000001 00000400 00000001 00000000 00000003     enp9s0\n")
+	got, err := Links(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(got, []string{"enp9s0"}) {
+		t.Fatalf("outward links are %v, want [enp9s0]", got)
+	}
+}
+
+// Against this machine's own routing table, so the parse is held to what the kernel actually writes
+// and not only to a fixture written to agree with it.
+func TestAgainstThisMachinesOwnTable(t *testing.T) {
+	got, err := Links("")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) == 0 {
+		t.Skip("this machine has no default route")
+	}
+	t.Logf("this machine's outward links: %v", got)
+}

internal/upgrade/upgrade.go

@@ -16,7 +16,9 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
+	"time"
 )
 
 // Files the launcher reads and this binary writes. Next to the store, because they are node
@@ -157,3 +159,168 @@ func ReadKnownGood(path string) (string, error) {
 	}
 	return strings.TrimSpace(string(raw)), nil
 }
+
+// Where delivered versions live, and what the binary inside one is called.
+//
+// **A directory named for its version, never a link and never a write over what is running**
+// (novox/hq ADR 0141). Two facts follow from that one choice: the kernel refuses to truncate a
+// running executable, so the path a delivery writes must not be the path being executed; and a
+// rollback needs the previous version still present, which a single path cannot offer.
+//
+// The mesh creates no links (novox/hq ADR 0012), so nothing points at "current". The version is in
+// the path, which is why nothing has to be told what is running.
+const (
+	// DefaultLibexec is where the host's own files live. Fixed rather than derived from where the
+	// running executable sits: the first host to understand any of this was copied to a machine by
+	// hand, and one that looked for its successor beside itself would never find a delivered version
+	// — which is every machine in this mesh on the day this ships.
+	DefaultLibexec  = "/usr/lib/nox-mesh-host"
+	VersionsDirName = "versions"
+	BinaryName      = "nox-mesh-host"
+	// PinnedName is the version a rollback chose, which the launcher runs instead of the newest.
+	// Without it the launcher would start the newest again and the rollback would flap.
+	PinnedName = "rollback-pinned"
+)
+
+// VersionsDir is where delivered versions live, given where the host's libexec is. An empty libexec
+// means the default, and the environment overrides it so a test needs no root.
+func VersionsDir(libexec string) string {
+	if libexec == "" {
+		libexec = os.Getenv("MESH_HOST_LIBEXEC")
+	}
+	if libexec == "" {
+		libexec = DefaultLibexec
+	}
+	return filepath.Join(libexec, VersionsDirName)
+}
+
+// PinnedPath is where a rollback records the version it chose.
+func PinnedPath(statePath string) string {
+	return filepath.Join(filepath.Dir(statePath), PinnedName)
+}
+
+// Delivered is one version present on the machine.
+type Delivered struct {
+	// Version is the directory's name, which is the version.
+	Version string
+	// Binary is the executable inside it.
+	Binary string
+	// At is when it arrived, which is how "newest" is decided.
+	At time.Time
+}
+
+// Versions are the versions delivered to this machine, newest first.
+//
+// **Newest by when it arrived, not by its name.** A version string comes from what the source was
+// tagged or described as, and those do not sort: "1.10" before "1.9", a commit hash before either.
+// Ordering by name would run an older host and call it an upgrade. When it arrived is a fact the
+// filesystem keeps and the delivery sets.
+//
+// A directory with no executable in it is not a version. A delivery that was interrupted leaves one,
+// and running the newest would then mean running nothing.
+func Versions(dir string) ([]Delivered, error) {
+	entries, err := os.ReadDir(dir)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("cannot read the delivered versions at %s: %w", dir, err)
+	}
+
+	var out []Delivered
+	for _, entry := range entries {
+		if !entry.IsDir() {
+			continue
+		}
+		binary := filepath.Join(dir, entry.Name(), BinaryName)
+		info, err := os.Stat(binary)
+		if err != nil || info.IsDir() {
+			continue
+		}
+		at := info.ModTime()
+		if d, err := entry.Info(); err == nil && d.ModTime().After(at) {
+			at = d.ModTime()
+		}
+		out = append(out, Delivered{Version: entry.Name(), Binary: binary, At: at})
+	}
+
+	// Newest first, and by name when two arrived in the same instant so the answer is never
+	// arbitrary — a test that passes half the time is worse than one that fails.
+	sort.Slice(out, func(a, b int) bool {
+		if out[a].At.Equal(out[b].At) {
+			return out[a].Version > out[b].Version
+		}
+		return out[a].At.After(out[b].At)
+	})
+	return out, nil
+}
+
+// Successor is the version this machine should be running instead of the given one, if any.
+//
+// Empty when the running version is the newest, which is the ordinary answer. The host asks this
+// between reconciles and nowhere else: standing aside mid-apply is the half-configured machine the
+// host exists to prevent (novox/hq ADR 0141).
+func Successor(dir, running string) (Delivered, bool, error) {
+	delivered, err := Versions(dir)
+	if err != nil {
+		return Delivered{}, false, err
+	}
+	if len(delivered) == 0 {
+		return Delivered{}, false, nil
+	}
+	newest := delivered[0]
+	// A machine whose running version is not among the delivered ones is the machine every mesh has
+	// one of: the host was put there by hand before any of this existed. Treating that as "stand
+	// aside" is correct — what was delivered is what the mesh asked for.
+	if newest.Version == running {
+		return Delivered{}, false, nil
+	}
+	return newest, true, nil
+}
+
+// Retire removes delivered versions older than the running one's predecessor.
+//
+// The running version and the one before it are kept, and nothing else: the predecessor is exactly
+// what a rollback starts, and every version before that is weight with no reader. Called after a
+// reconcile completes, which is the same evidence known-good is written on — retiring on any weaker
+// signal would delete the thing a failing host is about to need.
+//
+// Never the running version, whatever it is asked. A host that deleted its own image would survive
+// until it stopped and then be unstartable, and the launcher's rollback reads a version, not a
+// process.
+func Retire(dir, running string) ([]string, error) {
+	delivered, err := Versions(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	keep := map[string]bool{running: true}
+	for i, d := range delivered {
+		if d.Version != running {
+			continue
+		}
+		// Its predecessor is the next one down the list, which is the next oldest.
+		if i+1 < len(delivered) {
+			keep[delivered[i+1].Version] = true
+		}
+		break
+	}
+	// A running version that was never delivered has no predecessor among these, so the newest
+	// delivered one is what a rollback would reach for. Keep it.
+	if len(keep) == 1 && len(delivered) > 0 {
+		keep[delivered[0].Version] = true
+	}
+
+	var removed []string
+	for _, d := range delivered {
+		if keep[d.Version] {
+			continue
+		}
+		if err := os.RemoveAll(filepath.Join(dir, d.Version)); err != nil {
+			return removed, fmt.Errorf("cannot retire the host version %s: %w", d.Version, err)
+		}
+		removed = append(removed, d.Version)
+	}
+	sort.Strings(removed)
+	return removed, nil
+}

internal/upgrade/versions_test.go

@@ -0,0 +1,180 @@
+package upgrade
+
+import (
+	"os"
+	"path/filepath"
+	"reflect"
+	"sort"
+	"testing"
+	"time"
+)
+
+// deliver writes a version as a delivery would: a directory named for it with the binary inside.
+// at fixes when it arrived, because "newest" is when it arrived and a test must not race the clock.
+func deliver(t *testing.T, dir, version string, at time.Time) string {
+	t.Helper()
+	into := filepath.Join(dir, version)
+	if err := os.MkdirAll(into, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	binary := filepath.Join(into, BinaryName)
+	if err := os.WriteFile(binary, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chtimes(binary, at, at); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chtimes(into, at, at); err != nil {
+		t.Fatal(err)
+	}
+	return binary
+}
+
+// **Newest is when it arrived, not how its name sorts.**
+//
+// A version string is whatever the source was tagged or described as, and those do not sort: "1.10"
+// orders before "1.9", and a commit hash orders before either. Ordering by name would start an older
+// host and call that an upgrade.
+func TestNewestIsWhenItArrivedAndNotHowItSorts(t *testing.T) {
+	dir := t.TempDir()
+	base := time.Now().Add(-time.Hour)
+	deliver(t, dir, "1.10", base)                 // sorts LAST by name, arrived first
+	deliver(t, dir, "1.9", base.Add(time.Minute)) // sorts first by name, arrived last
+
+	got, err := Versions(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 2 || got[0].Version != "1.9" {
+		t.Fatalf("newest is %+v, want the one that arrived last (1.9)", got)
+	}
+}
+
+// A delivery that was interrupted leaves a directory with no executable in it. Running "the newest"
+// would then mean running nothing, so it is not a version.
+func TestADirectoryWithNoBinaryIsNotAVersion(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(dir, "half-delivered"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	deliver(t, dir, "good", time.Now().Add(-time.Hour))
+
+	got, err := Versions(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 1 || got[0].Version != "good" {
+		t.Fatalf("versions are %+v, want only the one with a binary", got)
+	}
+}
+
+// Nothing delivered is not a fault. A machine whose host was placed by hand has no versions
+// directory at all, and that must read as "no successor" rather than as an error that stops a
+// reconcile.
+func TestNoVersionsDirectoryIsNotAnError(t *testing.T) {
+	got, err := Versions(filepath.Join(t.TempDir(), "absent"))
+	if err != nil {
+		t.Fatalf("an absent versions directory should not be an error: %v", err)
+	}
+	if len(got) != 0 {
+		t.Fatalf("versions are %+v, want none", got)
+	}
+}
+
+func TestTheNewestVersionIsTheSuccessorAndTheRunningOneIsNot(t *testing.T) {
+	dir := t.TempDir()
+	base := time.Now().Add(-time.Hour)
+	deliver(t, dir, "one", base)
+	deliver(t, dir, "two", base.Add(time.Minute))
+
+	next, yes, err := Successor(dir, "one")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !yes || next.Version != "two" {
+		t.Fatalf("successor is %+v (%v), want two", next, yes)
+	}
+
+	if _, yes, err := Successor(dir, "two"); err != nil || yes {
+		t.Fatalf("the newest version is its own successor (%v, %v)", yes, err)
+	}
+}
+
+// A host put there by hand, before any of this existed, is not among the delivered versions. What the
+// mesh delivered is what it asked for, so that is a successor — otherwise the first delivery to such a
+// machine would be ignored for ever, which is every machine in this mesh today.
+func TestAHostThatWasNeverDeliveredHasASuccessor(t *testing.T) {
+	dir := t.TempDir()
+	deliver(t, dir, "delivered", time.Now().Add(-time.Hour))
+
+	next, yes, err := Successor(dir, "copied-by-hand")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !yes || next.Version != "delivered" {
+		t.Fatalf("successor is %+v (%v), want the delivered one", next, yes)
+	}
+}
+
+// The running version and its predecessor are kept, and nothing else. The predecessor is exactly what
+// a rollback starts; everything older has no reader.
+func TestRetireKeepsTheRunningVersionAndItsPredecessor(t *testing.T) {
+	dir := t.TempDir()
+	base := time.Now().Add(-4 * time.Hour)
+	for i, v := range []string{"one", "two", "three", "four"} {
+		deliver(t, dir, v, base.Add(time.Duration(i)*time.Hour))
+	}
+
+	removed, err := Retire(dir, "four")
+	if err != nil {
+		t.Fatal(err)
+	}
+	sort.Strings(removed)
+	if !reflect.DeepEqual(removed, []string{"one", "two"}) {
+		t.Fatalf("retired %v, want one and two — three is the predecessor a rollback needs", removed)
+	}
+	for _, kept := range []string{"three", "four"} {
+		if _, err := os.Stat(filepath.Join(dir, kept, BinaryName)); err != nil {
+			t.Fatalf("%s was retired and a rollback now has nowhere to go: %v", kept, err)
+		}
+	}
+}
+
+// **Never the running version, whatever it is asked.** A host that deleted its own image would run
+// until it stopped and then be unstartable, and the launcher's rollback reads a version rather than a
+// process.
+func TestRetireNeverRemovesTheRunningVersion(t *testing.T) {
+	dir := t.TempDir()
+	base := time.Now().Add(-2 * time.Hour)
+	deliver(t, dir, "older", base)
+	deliver(t, dir, "newer", base.Add(time.Hour))
+
+	// Asked while running the OLDER one, which is what a machine looks like between a delivery and
+	// the moment it stands aside.
+	if _, err := Retire(dir, "older"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "older", BinaryName)); err != nil {
+		t.Fatalf("the running version was retired: %v", err)
+	}
+}
+
+// A machine running a hand-placed host keeps the newest delivered version, because that is what a
+// rollback would reach for. Retiring it would leave the machine with no way back at all.
+func TestRetireKeepsTheNewestWhenTheRunningVersionWasNeverDelivered(t *testing.T) {
+	dir := t.TempDir()
+	base := time.Now().Add(-3 * time.Hour)
+	deliver(t, dir, "old", base)
+	deliver(t, dir, "new", base.Add(time.Hour))
+
+	removed, err := Retire(dir, "copied-by-hand")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(removed, []string{"old"}) {
+		t.Fatalf("retired %v, want only old — new is the rollback target", removed)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "new", BinaryName)); err != nil {
+		t.Fatalf("the only delivered version was retired: %v", err)
+	}
+}

packaging/launch_test.sh

@@ -187,5 +187,70 @@ sleep 1
 check "a crash is counted" "unlike a clean exit, which is not" "$(count)" "1"
 kill -TERM "$LP" 2>/dev/null; sleep 1; pkill -f "$MESH_HOST_BIN" 2>/dev/null || true
 
+# --- which version it runs (novox/hq ADR 0141) ------------------------------------------------
+#
+# Versions live side by side in directories named for them. The launcher picks one every time round
+# the loop, never once: standing aside for a successor is a clean exit, and the next turn has to run
+# what is on disk NOW — resolved once, the same binary would restart for ever and no upgrade would
+# ever take.
+
+# deliver a version as the mesh would, recording which one ran so a test can assert the choice.
+deliver() {
+	mkdir -p "$MESH_HOST_LIBEXEC/versions/$1"
+	cat > "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host" <<STUB
+#!/bin/sh
+echo "$1" >> "\$MESH_HOST_STATE_DIR/which.ran"
+exit "\${STUB_HOST_EXIT:-1}"
+STUB
+	chmod +x "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host"
+	# When it arrived is what "newest" means, so it is set rather than left to the clock.
+	touch -d "$2" "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host" "$MESH_HOST_LIBEXEC/versions/$1"
+}
+which_ran() { cat "$MESH_HOST_STATE_DIR/which.ran" 2>/dev/null || echo NONE; }
+
+# Newest is when it arrived, not how its name sorts: "1.10" orders before "1.9" by name, so ordering
+# by name would run an older host and call it an upgrade.
+setup
+deliver 1.10 "2 hours ago"
+deliver 1.9 "1 hour ago"
+"$LAUNCH" >/dev/null 2>&1 || true
+check "runs the newest delivered version" "newest is when it arrived, not how the name sorts" \
+	"$(which_ran)" "1.9"
+
+# A pin from a rollback beats the newest, or the launcher would start the failing binary again and
+# the rollback would flap.
+setup
+deliver 1.9 "2 hours ago"
+deliver 2.0 "1 hour ago"
+echo 1.9 > "$MESH_HOST_STATE_DIR/rollback-pinned"
+"$LAUNCH" >/dev/null 2>&1 || true
+check "a pinned version beats the newest" "otherwise a rollback starts the binary it just rejected" \
+	"$(which_ran)" "1.9"
+
+# A pin naming a version that is not there is ignored rather than fatal: the machine choosing for
+# itself is better than a machine that starts nothing.
+setup
+deliver 2.0 "1 hour ago"
+echo 1.9 > "$MESH_HOST_STATE_DIR/rollback-pinned"
+"$LAUNCH" >/dev/null 2>&1 || true
+check "an undeliverable pin is ignored" "a machine that starts nothing is worse than one that chooses" \
+	"$(which_ran)" "2.0"
+
+# An interrupted delivery leaves a directory with no binary in it. Treating it as the newest would
+# mean running nothing.
+setup
+deliver 1.9 "2 hours ago"
+mkdir -p "$MESH_HOST_LIBEXEC/versions/2.0-half"
+touch -d "1 minute ago" "$MESH_HOST_LIBEXEC/versions/2.0-half"
+"$LAUNCH" >/dev/null 2>&1 || true
+check "skips a version with no binary" "a directory is not a version; the binary is" \
+	"$(which_ran)" "1.9"
+
+# Nothing delivered: the host placed by hand, which is how the first one always arrives. Without this
+# the change would strand every machine in the mesh on the day it ships.
+setup
+"$LAUNCH" >/dev/null 2>&1 || true
+check "falls back to the host placed by hand" "every first host arrives this way" "$(started)" "yes"
+
 printf '\nlaunch: %d passed, %d failed\n' "$PASS" "$FAIL"
 [ "$FAIL" -eq 0 ]

packaging/nox-mesh-host-launch

@@ -16,16 +16,51 @@ set -u
 
 STATE_DIR="${MESH_HOST_STATE_DIR:-/var/lib/mesh-host}"
 LIBEXEC="${MESH_HOST_LIBEXEC:-/usr/lib/nox-mesh-host}"
-HOST="${MESH_HOST_BIN:-/usr/bin/nox-mesh-host}"
+# The host that was placed by hand, used only when nothing has been delivered. The first host on a
+# machine always arrives this way; every one after it is delivered (novox/hq ADR 0141).
+FALLBACK="${MESH_HOST_BIN:-/usr/bin/nox-mesh-host}"
+VERSIONS="$LIBEXEC/versions"
+BINARY="nox-mesh-host"
 LIMIT="${MESH_HOST_START_LIMIT:-3}"
 BACKOFF="${MESH_HOST_BACKOFF:-5}"
 ONCE="${MESH_HOST_RUN_ONCE:-}"   # tests run one iteration; nothing else sets this
 
 ATTEMPTS="$STATE_DIR/start-attempts"
 HALTED="$STATE_DIR/halted"
+PINNED="$STATE_DIR/rollback-pinned"
 
 say() { echo "nox-mesh-host-launch: $*" >&2; }
 
+# Which host to run: the version a rollback pinned, or the most recently delivered one, or the one
+# placed by hand when nothing has been delivered (novox/hq ADR 0141).
+#
+# **Asked every time round the loop, not once.** Standing aside for a successor is a clean exit, and
+# the next turn has to run what is on disk NOW — resolving this once would restart the same binary
+# for ever and the upgrade would never take.
+#
+# Newest by when it arrived, never by how its name sorts: a version string is whatever the source was
+# described as, and those do not sort — "1.10" orders before "1.9". Ordering by name would start an
+# older host and call it an upgrade.
+pick_host() {
+	if [ -s "$PINNED" ]; then
+		pinned="$(tr -d '[:space:]' < "$PINNED" 2>/dev/null || true)"
+		if [ -n "$pinned" ] && [ -x "$VERSIONS/$pinned/$BINARY" ]; then
+			echo "$VERSIONS/$pinned/$BINARY"
+			return 0
+		fi
+		say "the pinned version '$pinned' is not delivered; ignoring the pin"
+	fi
+	# A directory with no executable in it is not a version: an interrupted delivery leaves one, and
+	# running "the newest" would then mean running nothing.
+	for candidate in $(ls -1t "$VERSIONS" 2>/dev/null || true); do
+		if [ -x "$VERSIONS/$candidate/$BINARY" ]; then
+			echo "$VERSIONS/$candidate/$BINARY"
+			return 0
+		fi
+	done
+	echo "$FALLBACK"
+}
+
 child=
 stopping=
 
@@ -95,6 +130,14 @@ while :; do
 		fi
 	fi
 
+	HOST="$(pick_host)"
+	if [ ! -x "$HOST" ]; then
+		say "no host to run: nothing delivered under $VERSIONS and $FALLBACK is not executable."
+		printf 'no host binary\n' > "$HALTED"
+		exit 0
+	fi
+	say "running $HOST"
+
 	"$HOST" run &
 	child=$!
 	status=0

packaging/nox-mesh-host-rollback

@@ -1,20 +1,29 @@
 #!/bin/sh
 # Put the host back on the last version that worked.
 #
-# novox/hq ADR 0005. This runs when nox-mesh-host will not start, so it shares no code with it
+# novox/hq ADR 0005 and ADR 0141. This runs when nox-mesh-host will not start, so it shares no code
-# and calls none of it: a binary that cannot start cannot be its own recovery. POSIX sh, no
+# with it and calls none of it: a binary that cannot start cannot be its own recovery. POSIX sh, no
 # bashisms, nothing that has to be installed.
 #
-# It is deliberately dull. Everything it does is one of: read a file, run the package manager,
+# It is deliberately dull. Everything it does is one of: read a file, look at a directory, write a
-# ask the service manager to try again.
+# file.
+#
+# **It used to reinstall a package.** It read the known-good version and asked one operating system's
+# package manager for it, out of that package manager's cache. Two things were wrong with that. No
+# machine in this mesh had the host installed as a package, so the recovery could not run on any of
+# them; and the host is built per operating system (ADR 0005), so a recovery written in one package
+# manager's terms could not run on two of the three. Versions now live side by side in directories
+# named for them, so going back is choosing a directory — which is the same on every machine.
 set -eu
 
 STATE_DIR="${MESH_HOST_STATE_DIR:-/var/lib/mesh-host}"
-PKG_CACHE="${MESH_HOST_PKG_CACHE:-/var/cache/pacman/pkg}"
+LIBEXEC="${MESH_HOST_LIBEXEC:-/usr/lib/nox-mesh-host}"
-PACKAGE="${MESH_HOST_PACKAGE:-nox-mesh-host}"
+VERSIONS="$LIBEXEC/versions"
+BINARY="nox-mesh-host"
 
 KNOWN_GOOD="$STATE_DIR/known-good"
 ATTEMPTED="$STATE_DIR/rollback-attempted"
+PINNED="$STATE_DIR/rollback-pinned"
 
 say() { echo "nox-mesh-host-rollback: $*" >&2; }
 
@@ -43,23 +52,24 @@ if [ -z "$VERSION" ]; then
 	exit 0
 fi
 
-PKG="$(ls "$PKG_CACHE"/"$PACKAGE"-"$VERSION"-*.pkg.tar.* 2>/dev/null | head -n 1 || true)"
+# The version that last worked may be the one that was placed by hand, which is not delivered and has
-if [ -z "$PKG" ]; then
+# no directory. Nothing to choose, and saying so is better than pinning a version that is not there —
-	say "known-good is $VERSION and no package for it is in $PKG_CACHE."
+# the launcher would ignore the pin and start the newest again, which is the binary that is failing.
-	say "the cache was cleaned, or that version was never installed from here."
+if [ ! -x "$VERSIONS/$VERSION/$BINARY" ]; then
+	say "known-good is $VERSION and no such version is delivered under $VERSIONS."
+	say "it was retired, or that host was placed by hand and never delivered."
 	say "cannot roll back. this node needs a person."
 	exit 1
 fi
 
-say "rolling back to $VERSION ($PKG)"
+say "rolling back to $VERSION ($VERSIONS/$VERSION/$BINARY)"
 printf '%s\n' "$VERSION" > "$ATTEMPTED"
 
-if ! pacman -U --noconfirm "$PKG"; then
+# The pin is what stops the launcher starting the newest again. Written last, so a failure above
-	say "the package manager refused to install $PKG."
+# leaves the machine choosing for itself rather than pinned to something this script did not verify.
-	exit 1
+printf '%s\n' "$VERSION" > "$PINNED"
-fi
 
-# Deliberately does NOT start anything. The launcher called this and will exec the host next,
+# Deliberately does NOT start anything. The launcher called this and will run the host next, so
-# so starting it here would run two. novox/hq ADR 0005 moved that responsibility; this script
+# starting it here would run two. novox/hq ADR 0005 moved that responsibility; this script chooses a
-# installs a version and says so, and nothing else.
+# version and says so, and nothing else.
-say "rolled back to $VERSION. the launcher will start it."
+say "pinned $VERSION. the launcher will start it."

packaging/rollback_test.sh

@@ -1,9 +1,15 @@
 #!/bin/sh
 # Tests for nox-mesh-host-rollback.
 #
-# It runs on a machine where the host will not start, which is the one moment nobody can afford
+# It runs on a machine where the host will not start, which is the one moment nobody can afford it to
-# it to be wrong — and the one moment it is hardest to debug. So it is tested here, against a
+# be wrong — and the one moment it is hardest to debug. So it is tested here, against a real
-# real filesystem, with a stub package manager that records what it was asked to do.
+# filesystem holding real delivered versions.
+#
+# **These used to stub a package manager.** The script reinstalled the known-good version with
+# `pacman -U` out of the package cache, which no machine in this mesh used and which two of the three
+# operating systems the host is built for do not have (novox/hq ADR 0141). Going back is now choosing
+# a directory, so there is nothing to stub: the thing under test is the filesystem, and a fake would
+# only assert that the fake behaves as expected (novox/hq ADR 0017).
 set -eu
 cd "$(dirname "$0")"
 SCRIPT="$PWD/nox-mesh-host-rollback"
@@ -12,26 +18,15 @@ PASS=0; FAIL=0
 setup() {
 	WORK="$(mktemp -d)"
 	export MESH_HOST_STATE_DIR="$WORK/state"
-	export MESH_HOST_PKG_CACHE="$WORK/cache"
+	export MESH_HOST_LIBEXEC="$WORK/libexec"
-	export MESH_HOST_PACKAGE="nox-mesh-host"
+	mkdir -p "$MESH_HOST_STATE_DIR" "$MESH_HOST_LIBEXEC/versions"
-	mkdir -p "$MESH_HOST_STATE_DIR" "$MESH_HOST_PKG_CACHE" "$WORK/bin"
+}
 
-	# Stubs on PATH. Not mocks of the script's own logic — the boundary is real commands, and
+# deliver a version the way the mesh would: a directory named for it, with the binary inside.
-	# these record the calls so a test can assert what the script asked the machine to do.
+deliver() {
-	cat > "$WORK/bin/pacman" <<'STUB'
+	mkdir -p "$MESH_HOST_LIBEXEC/versions/$1"
-#!/bin/sh
+	printf '#!/bin/sh\nexit 0\n' > "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host"
-echo "$@" >> "$MESH_HOST_STATE_DIR/pacman.calls"
+	chmod +x "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host"
-[ -n "${STUB_PACMAN_FAILS:-}" ] && exit 1
-exit 0
-STUB
-	cat > "$WORK/bin/systemctl" <<'STUB'
-#!/bin/sh
-echo "$@" >> "$MESH_HOST_STATE_DIR/systemctl.calls"
-exit 0
-STUB
-	chmod +x "$WORK/bin/pacman" "$WORK/bin/systemctl"
-	PATH="$WORK/bin:$PATH"; export PATH
-	unset STUB_PACMAN_FAILS || true
 }
 
 check() { # name, condition-description, actual, expected
@@ -41,32 +36,38 @@ check() { # name, condition-description, actual, expected
 
 # --- a normal rollback ---------------------------------------------------------------------
 setup
-echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
+deliver 1.4.2
-touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst"
+deliver 1.5.0
-"$SCRIPT" >/dev/null 2>&1
+echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good"
-check "installs the known-good version" "pacman is asked to install the cached package" \
+RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$?
-	"$(grep -c 'nox-mesh-host-1.4.2' "$MESH_HOST_STATE_DIR/pacman.calls" 2>/dev/null || echo 0)" "1"
+check "pins the known-good version" "the launcher reads the pin and runs that version instead of the newest" \
-# It installs and stops. The launcher execs the host next, and starting it here would run two
+	"$(cat "$MESH_HOST_STATE_DIR/rollback-pinned" 2>/dev/null || echo MISSING)" "1.4.2"
-# (novox/hq ADR 0005).
-check "does not start anything itself" "the launcher owns starting" \
-	"$([ -f "$MESH_HOST_STATE_DIR/systemctl.calls" ] && echo started || echo not-started)" "not-started"
 check "records that it rolled back" "the attempted marker holds the version" \
 	"$(cat "$MESH_HOST_STATE_DIR/rollback-attempted" 2>/dev/null || echo MISSING)" "1.4.2"
+check "succeeds" "a rollback that found its version is not a failure" "$RC" "0"
+# It chooses and stops. The launcher runs the host next, and starting it here would run two
+# (novox/hq ADR 0005).
+check "does not start anything itself" "the launcher owns starting" \
+	"$(ls "$MESH_HOST_STATE_DIR" | grep -c started || true)" "0"
+# The version it rolled back FROM is left alone: it is the newest, and retiring it is the running
+# host's job after a reconcile it completes, never a recovery's.
+check "leaves the failing version on disk" "a recovery deletes nothing" \
+	"$([ -x "$MESH_HOST_LIBEXEC/versions/1.5.0/nox-mesh-host" ] && echo present || echo gone)" "present"
 
 # --- it rolls back only once ---------------------------------------------------------------
 setup
-echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
+deliver 1.4.2
-echo "1.4.2" > "$MESH_HOST_STATE_DIR/rollback-attempted"
+echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good"
-touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst"
+echo 1.4.2 > "$MESH_HOST_STATE_DIR/rollback-attempted"
-"$SCRIPT" >/dev/null 2>&1
+"$SCRIPT" >/dev/null 2>&1 || true
 check "does not roll back twice" "a second failure is the machine, not the binary" \
-	"$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called"
+	"$([ -e "$MESH_HOST_STATE_DIR/rollback-pinned" ] && echo pinned || echo untouched)" "untouched"
 
 # --- nothing to roll back to ---------------------------------------------------------------
 setup
-set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e
+RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$?
 check "no known-good: does nothing" "a host that never reconciled has no version to return to" \
-	"$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called"
+	"$([ -e "$MESH_HOST_STATE_DIR/rollback-attempted" ] && echo attempted || echo untouched)" "untouched"
 # The exit code is asserted from a real run, not from a literal. An earlier version of this
 # compared "0" to "0" and could not fail — which hid an injected fault that made the script die
 # here instead of returning cleanly.
@@ -74,23 +75,29 @@ check "no known-good: exits zero" "an installation failure is not a rollback fai
 
 setup
 printf '   \n' > "$MESH_HOST_STATE_DIR/known-good"
-"$SCRIPT" >/dev/null 2>&1
+"$SCRIPT" >/dev/null 2>&1 || true
-check "blank known-good: refuses to guess" "installing nothing and reporting success is the fault this prevents" \
+check "blank known-good: refuses to guess" "pinning nothing and reporting success is the fault this prevents" \
-	"$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called"
+	"$([ -e "$MESH_HOST_STATE_DIR/rollback-pinned" ] && echo pinned || echo untouched)" "untouched"
 
-# --- the cache was cleaned ------------------------------------------------------------------
+# --- the known-good version is not delivered -------------------------------------------------
+# It was retired, or that host was placed on the machine by hand and never delivered — which is how
+# every first host arrives. Pinning it anyway would have the launcher ignore the pin and start the
+# newest again, which is the binary that is failing.
 setup
-echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
+deliver 1.5.0
-set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e
+echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good"
-check "missing package: fails loudly" "cannot roll back, and says so rather than reporting success" "$RC" "1"
+RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$?
+check "version not delivered: fails loudly" "cannot roll back, and says so rather than reporting success" "$RC" "1"
+check "version not delivered: pins nothing" "a pin the launcher would ignore is worse than none" \
+	"$([ -e "$MESH_HOST_STATE_DIR/rollback-pinned" ] && echo pinned || echo untouched)" "untouched"
 
-# --- the package manager refuses -------------------------------------------------------------
+# --- a version directory with no binary in it ------------------------------------------------
+# An interrupted delivery leaves one. Pinning it would start nothing.
 setup
-echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
+mkdir -p "$MESH_HOST_LIBEXEC/versions/1.4.2"
-touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst"
+echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good"
-STUB_PACMAN_FAILS=1 ; export STUB_PACMAN_FAILS
+RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$?
-set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e
+check "half-delivered version: fails loudly" "a directory is not a version; the binary is" "$RC" "1"
-check "pacman fails: exits non-zero" "a failed rollback is a failure the launcher must see" "$RC" "1"
 
 printf '\nrollback: %d passed, %d failed\n' "$PASS" "$FAIL"
 [ "$FAIL" -eq 0 ]