Merge pull request 'A converged machine can speak unasked' (#48) from fix/a-converged-machine-can-speak-unasked into main

A reconcile is otherwise silent, and the condition deciding when it speaks asked
only what an adopted node reports: what it holds, and the firewall it found. A
converged node has neither, so it could speak only in reply to a declaration —
and the mesh composes no declaration for a node that has not said which links
face outside (ADR 0140). A machine waiting for a push that was waiting for the
machine.

Measured, not predicted: after the control plane learnt to read the links, the
control node recorded its own and the three converged machines sat silent while
their filters were refused.

The condition is now a named predicate, because as an inline expression nothing
could test it — which is why the gap shipped.

cmd/mesh-host/main.go

@@ -601,6 +601,20 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
 					"  a rollback would have nothing to return to.\n", version, err)
 		}
 	}
+	// And retire what is older than this version's predecessor, on the same evidence known-good is
+	// written on (novox/hq ADR 0141). The predecessor stays, because it is exactly what a rollback
+	// starts; everything before it has no reader. Never this version, whatever the answer.
+	//
+	// A failure is said and does not fail the apply, for the reason above: what is lost is disk, and
+	// hiding it would make a machine quietly fill up.
+	if version != "" {
+		if retired, err := upgrade.Retire(upgrade.VersionsDir(""), version); err != nil {
+			fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not retire an older host: %v\n", err)
+		} else if len(retired) > 0 {
+			fmt.Fprintf(os.Stderr, "mesh-host: retired the host version(s) %s\n",
+				strings.Join(retired, ", "))
+		}
+	}
 	// And tell the launcher this start worked. Without it the counter only climbs, and a node
 	// that has been up for months rolls itself back on its third ordinary restart.
 	if err := upgrade.ClearAttempts(upgrade.AttemptsPath(opts.state)); err != nil {
@@ -979,12 +993,31 @@ func runLink(ctx context.Context, opts options) error {
 	sched := apply.NewScheduler(apply.SystemClock(), apply.ExecRunner, say)
 	go sched.Run(ctx)
 
+	// **Standing aside for a successor happens between reconciles and nowhere else** (novox/hq ADR
+	// 0141). A host that stood aside mid-apply is the half-configured machine this host exists to
+	// prevent, so the question is asked after an apply has finished and the answer is a clean exit —
+	// which the launcher already reads as "run whatever is on disk now".
+	aside, standAside := context.WithCancel(ctx)
+	defer standAside()
+	stoodAside := false
+
 	applier := func(ctx context.Context, raw, signature []byte) link.Report {
 		report := applyAndKeep(ctx, opts, raw, &store.Declared{Declaration: raw, Signature: signature}, sched, say)
 		// **A declaration may carry this machine's membership for another bus.** It arrives as a
 		// sealed file like any secret, and is read after the rest has applied so the bus it names is
 		// standing before this machine leaves the one it is on (novox/hq design 28, task 5.2).
 		adoptDeliveredMembership(identity.Path(opts.state), &mine, say)
+
+		switch next, waiting, err := upgrade.Successor(upgrade.VersionsDir(""), version); {
+		case err != nil:
+			// Said, not fatal. A host that cannot read the delivered versions is still running this
+			// machine correctly; what it has lost is the ability to be replaced.
+			say(fmt.Sprintf("cannot tell whether a newer host is delivered: %v", err))
+		case waiting:
+			say(fmt.Sprintf("host %s is delivered; standing aside so the launcher runs it", next.Version))
+			stoodAside = true
+			standAside()
+		}
 		return report
 	}
 
@@ -1015,7 +1048,7 @@ func runLink(ctx context.Context, opts options) error {
 		}}
 	})
 
-	return link.HoldRoused(ctx, link.Membership{
+	held := link.HoldRoused(aside, link.Membership{
 		Node:        mine.Node,
 		Broker:      mine.Membership.Broker,
 		Fingerprint: mine.Membership.Fingerprint,
@@ -1023,6 +1056,13 @@ func runLink(ctx context.Context, opts options) error {
 		Transport:   mine.Membership.Transport,
 		Signer:      mine.Membership.Signer,
 	}, applier, say, opts.timeout, rousedBySignal(ctx), outbox)
+	// **Cleanly**, or the launcher counts standing aside as a crash and rolls the new host back
+	// before it has run once. The context this returns on was cancelled deliberately, so its error
+	// is not a fault to report.
+	if stoodAside {
+		return nil
+	}
+	return held
 }
 
 // adoptionWatch remembers what the node last said about what it holds and its firewall, so a
@@ -1037,7 +1077,12 @@ type adoptionWatch struct {
 // is what the controller previews a flip from, so a port that opens or closes between deliveries
 // must reach it too (novox/hq ADR 0100).
 func adoptionFingerprint(r link.Report) string {
-	parts := []string{"firewall=" + r.Firewall}
+	// **Which links face outside is part of it, though it is not about adoption** (novox/hq ADR
+	// 0140). The mesh composes no filter for a machine that has not said, so a machine whose links
+	// changed — or which has only just learnt to say — has to say so without being asked. Left out,
+	// it could only speak when a declaration arrived, and a declaration cannot be composed until it
+	// has spoken: a machine waiting for a push that is waiting for the machine.
+	parts := []string{"firewall=" + r.Firewall, "outward=" + strings.Join(r.Outward, ",")}
 	for _, h := range r.Held {
 		parts = append(parts, "held "+h.ID+"="+h.Changed)
 	}
@@ -1045,7 +1090,7 @@ func adoptionFingerprint(r link.Report) string {
 		parts = append(parts, fmt.Sprintf("reach %s %s:%d %s %v %d", reach.Protocol, reach.Address,
 			reach.Port, reach.By, reach.Published, reach.ContainerPort))
 	}
-	sort.Strings(parts[1:])
+	sort.Strings(parts[2:])
 	return strings.Join(parts, "\n")
 }
 
@@ -1153,7 +1198,17 @@ func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, s
 		// A reconcile is otherwise silent. On an adopted node it speaks when what it holds or
 		// its firewall changed, because that is how a predecessor still writing is caught
 		// (novox/hq ADR 0100); publish decides whether anything did.
-		if publish != nil && report.Refused == "" && (len(report.Held) > 0 || report.Firewall != "") {
+		//
+		// **And on any node, when it can say which links face outside** (novox/hq ADR 0140). A
+		// converged node holds nothing and found no firewall, so this gate closed on it and the
+		// node could speak only in reply to a declaration — while the mesh composes no declaration
+		// for a node that has not said which links face outside. A machine waiting for a push that
+		// was waiting for the machine, and measured: three converged machines sat silent while the
+		// control plane refused to send them a filter.
+		//
+		// Offered, not published: whether it is news is still the watch's to decide, so an
+		// unchanged answer costs one comparison every reconcile and nothing on the bus.
+		if publish != nil && worthSaying(report) {
 			publish(report)
 		}
 		switch {
@@ -1165,6 +1220,23 @@ func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, s
 	}
 }
 
+// worthSaying is whether a reconcile's report carries anything the mesh needs to hear unasked.
+//
+// **Named rather than written into the loop**, so the rule can be tested. It was a condition inline
+// and it was wrong in a way nothing could catch: it asked only what an adopted node reports, so a
+// converged node — which holds nothing and found no firewall — could speak only in reply to a
+// declaration, while the mesh composes no declaration for a node that has not said which links face
+// outside (novox/hq ADR 0140). Three machines sat silent waiting for a push that was waiting for them.
+//
+// A refused report says nothing about the machine, so it is not news; the refusal is said on the
+// console where a person reads it.
+func worthSaying(report link.Report) bool {
+	if report.Refused != "" {
+		return false
+	}
+	return len(report.Held) > 0 || report.Firewall != "" || len(report.Outward) > 0
+}
+
 // applyDeclared applies a declaration that has already been proved to come from the mesh.
 //
 // Signature checking happens before this is called, in the link. By the time anything here runs,
@@ -1263,7 +1335,7 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
 		sched.Sync(declared, held)
 	}
 
-	report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw)}
+	report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw), Host: version}
 	// Which of this machine's links face outside, for the filter the mesh writes around them
 	// (novox/hq ADR 0140). Reported whatever the node's mode: a converged node's filter needs it,
 	// and an adopted one becomes converged without a further round trip. A machine that cannot read

cmd/mesh-host/main_test.go

@@ -501,3 +501,70 @@ func TestReconcileAfterAControllerDeclarationDoesNotReapplyTheBundle(t *testing.
 		t.Errorf("with nothing said, reconcile did not reach for the carried bundle: %v", err)
 	}
 }
+
+// **A machine says which links face outside without being asked.**
+//
+// The mesh composes no filter for a machine that has not said (novox/hq ADR 0140), and a machine only
+// speaks unasked when this fingerprint changes. Left out of it, a machine that has just learnt to say
+// could speak only when a declaration arrived — and a declaration cannot be composed until it has
+// spoken. A machine waiting for a push that is waiting for the machine.
+func TestANewOutwardLinkIsSaidUnasked(t *testing.T) {
+	w := &adoptionWatch{}
+	first := link.Report{Firewall: "none"}
+	if !w.differs(first) {
+		t.Fatal("the first report should differ from nothing")
+	}
+	w.said(first)
+
+	// Only the links changed, and nothing about adoption.
+	learnt := link.Report{Firewall: "none", Outward: []string{"eth0"}}
+	if !w.differs(learnt) {
+		t.Fatal("a machine that has just learnt which links face outside would never say so, " +
+			"and could then never be sent a filter")
+	}
+	w.said(learnt)
+	if w.differs(link.Report{Firewall: "none", Outward: []string{"eth0"}}) {
+		t.Fatal("the same links are reported as a change, so the machine would speak on every reconcile")
+	}
+
+	// And a link that changes — a laptop moving from a cable to a radio — is said too, because the
+	// filter is written around the old one until it is.
+	if !w.differs(link.Report{Firewall: "none", Outward: []string{"wlan0"}}) {
+		t.Fatal("a changed outward link is not said, so the filter stays written around the old one")
+	}
+}
+
+// **A converged machine can say which links face outside, unasked.**
+//
+// A reconcile is otherwise silent, and the condition deciding when it speaks asked only what an
+// adopted node reports — what it holds, and the firewall it found. A converged node has neither, so
+// it could speak only in reply to a declaration, and the mesh composes no declaration for a node
+// that has not said which links face outside (novox/hq ADR 0140). Measured: three converged machines
+// sat silent while the control plane refused to send them a filter.
+func TestAConvergedMachineSaysItsOutwardLinksUnasked(t *testing.T) {
+	// A converged node's reconcile: nothing held, no found firewall, and the links it can see.
+	converged := link.Report{Outward: []string{"eth0"}}
+	if !worthSaying(converged) {
+		t.Fatal("a converged machine cannot say which links face outside, so it can never be " +
+			"sent a filter — a machine waiting for a push that is waiting for the machine")
+	}
+
+	// An adopted node's reasons still hold, because that is how a predecessor still writing is caught.
+	if !worthSaying(link.Report{Firewall: "ufw"}) {
+		t.Fatal("an adopted machine no longer says which firewall it found")
+	}
+	if !worthSaying(link.Report{Held: []link.Held{{ID: "a-file"}}}) {
+		t.Fatal("an adopted machine no longer says what it holds")
+	}
+
+	// And a reconcile with nothing to say stays silent, or every machine speaks every five minutes
+	// about nothing.
+	if worthSaying(link.Report{}) {
+		t.Fatal("a reconcile with nothing to say speaks anyway")
+	}
+
+	// A refused report says nothing about the machine; the refusal is for the console.
+	if worthSaying(link.Report{Outward: []string{"eth0"}, Refused: "not for this node"}) {
+		t.Fatal("a refused report is offered as news about the machine")
+	}
+}

internal/link/messages.go

@@ -110,6 +110,13 @@ type Report struct {
 	// and the mesh's up in its place, and where the found configuration's original was kept.
 	Tunnel *CarriedTunnel `json:"tunnel,omitempty"`
 
+	// Host is the version of the host that produced this report (novox/hq ADR 0141).
+	//
+	// Without it nothing can say a machine is behind, so "every machine current with its source"
+	// could not include the host — the one component the mesh did not deliver. It is a fact the
+	// machine states about itself, like the firewall it found and the links that face outside.
+	Host string `json:"host,omitempty"`
+
 	// Outward is the links on this machine that face outside it — the ones carrying a default
 	// route (novox/hq ADR 0140). Every node reports it, adopted or converged, because a converged
 	// node's filter is written around it.

internal/upgrade/upgrade.go

@@ -16,7 +16,9 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
+	"time"
 )
 
 // Files the launcher reads and this binary writes. Next to the store, because they are node
@@ -157,3 +159,168 @@ func ReadKnownGood(path string) (string, error) {
 	}
 	return strings.TrimSpace(string(raw)), nil
 }
+
+// Where delivered versions live, and what the binary inside one is called.
+//
+// **A directory named for its version, never a link and never a write over what is running**
+// (novox/hq ADR 0141). Two facts follow from that one choice: the kernel refuses to truncate a
+// running executable, so the path a delivery writes must not be the path being executed; and a
+// rollback needs the previous version still present, which a single path cannot offer.
+//
+// The mesh creates no links (novox/hq ADR 0012), so nothing points at "current". The version is in
+// the path, which is why nothing has to be told what is running.
+const (
+	// DefaultLibexec is where the host's own files live. Fixed rather than derived from where the
+	// running executable sits: the first host to understand any of this was copied to a machine by
+	// hand, and one that looked for its successor beside itself would never find a delivered version
+	// — which is every machine in this mesh on the day this ships.
+	DefaultLibexec  = "/usr/lib/nox-mesh-host"
+	VersionsDirName = "versions"
+	BinaryName      = "nox-mesh-host"
+	// PinnedName is the version a rollback chose, which the launcher runs instead of the newest.
+	// Without it the launcher would start the newest again and the rollback would flap.
+	PinnedName = "rollback-pinned"
+)
+
+// VersionsDir is where delivered versions live, given where the host's libexec is. An empty libexec
+// means the default, and the environment overrides it so a test needs no root.
+func VersionsDir(libexec string) string {
+	if libexec == "" {
+		libexec = os.Getenv("MESH_HOST_LIBEXEC")
+	}
+	if libexec == "" {
+		libexec = DefaultLibexec
+	}
+	return filepath.Join(libexec, VersionsDirName)
+}
+
+// PinnedPath is where a rollback records the version it chose.
+func PinnedPath(statePath string) string {
+	return filepath.Join(filepath.Dir(statePath), PinnedName)
+}
+
+// Delivered is one version present on the machine.
+type Delivered struct {
+	// Version is the directory's name, which is the version.
+	Version string
+	// Binary is the executable inside it.
+	Binary string
+	// At is when it arrived, which is how "newest" is decided.
+	At time.Time
+}
+
+// Versions are the versions delivered to this machine, newest first.
+//
+// **Newest by when it arrived, not by its name.** A version string comes from what the source was
+// tagged or described as, and those do not sort: "1.10" before "1.9", a commit hash before either.
+// Ordering by name would run an older host and call it an upgrade. When it arrived is a fact the
+// filesystem keeps and the delivery sets.
+//
+// A directory with no executable in it is not a version. A delivery that was interrupted leaves one,
+// and running the newest would then mean running nothing.
+func Versions(dir string) ([]Delivered, error) {
+	entries, err := os.ReadDir(dir)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("cannot read the delivered versions at %s: %w", dir, err)
+	}
+
+	var out []Delivered
+	for _, entry := range entries {
+		if !entry.IsDir() {
+			continue
+		}
+		binary := filepath.Join(dir, entry.Name(), BinaryName)
+		info, err := os.Stat(binary)
+		if err != nil || info.IsDir() {
+			continue
+		}
+		at := info.ModTime()
+		if d, err := entry.Info(); err == nil && d.ModTime().After(at) {
+			at = d.ModTime()
+		}
+		out = append(out, Delivered{Version: entry.Name(), Binary: binary, At: at})
+	}
+
+	// Newest first, and by name when two arrived in the same instant so the answer is never
+	// arbitrary — a test that passes half the time is worse than one that fails.
+	sort.Slice(out, func(a, b int) bool {
+		if out[a].At.Equal(out[b].At) {
+			return out[a].Version > out[b].Version
+		}
+		return out[a].At.After(out[b].At)
+	})
+	return out, nil
+}
+
+// Successor is the version this machine should be running instead of the given one, if any.
+//
+// Empty when the running version is the newest, which is the ordinary answer. The host asks this
+// between reconciles and nowhere else: standing aside mid-apply is the half-configured machine the
+// host exists to prevent (novox/hq ADR 0141).
+func Successor(dir, running string) (Delivered, bool, error) {
+	delivered, err := Versions(dir)
+	if err != nil {
+		return Delivered{}, false, err
+	}
+	if len(delivered) == 0 {
+		return Delivered{}, false, nil
+	}
+	newest := delivered[0]
+	// A machine whose running version is not among the delivered ones is the machine every mesh has
+	// one of: the host was put there by hand before any of this existed. Treating that as "stand
+	// aside" is correct — what was delivered is what the mesh asked for.
+	if newest.Version == running {
+		return Delivered{}, false, nil
+	}
+	return newest, true, nil
+}
+
+// Retire removes delivered versions older than the running one's predecessor.
+//
+// The running version and the one before it are kept, and nothing else: the predecessor is exactly
+// what a rollback starts, and every version before that is weight with no reader. Called after a
+// reconcile completes, which is the same evidence known-good is written on — retiring on any weaker
+// signal would delete the thing a failing host is about to need.
+//
+// Never the running version, whatever it is asked. A host that deleted its own image would survive
+// until it stopped and then be unstartable, and the launcher's rollback reads a version, not a
+// process.
+func Retire(dir, running string) ([]string, error) {
+	delivered, err := Versions(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	keep := map[string]bool{running: true}
+	for i, d := range delivered {
+		if d.Version != running {
+			continue
+		}
+		// Its predecessor is the next one down the list, which is the next oldest.
+		if i+1 < len(delivered) {
+			keep[delivered[i+1].Version] = true
+		}
+		break
+	}
+	// A running version that was never delivered has no predecessor among these, so the newest
+	// delivered one is what a rollback would reach for. Keep it.
+	if len(keep) == 1 && len(delivered) > 0 {
+		keep[delivered[0].Version] = true
+	}
+
+	var removed []string
+	for _, d := range delivered {
+		if keep[d.Version] {
+			continue
+		}
+		if err := os.RemoveAll(filepath.Join(dir, d.Version)); err != nil {
+			return removed, fmt.Errorf("cannot retire the host version %s: %w", d.Version, err)
+		}
+		removed = append(removed, d.Version)
+	}
+	sort.Strings(removed)
+	return removed, nil
+}

internal/upgrade/versions_test.go

@@ -0,0 +1,180 @@
+package upgrade
+
+import (
+	"os"
+	"path/filepath"
+	"reflect"
+	"sort"
+	"testing"
+	"time"
+)
+
+// deliver writes a version as a delivery would: a directory named for it with the binary inside.
+// at fixes when it arrived, because "newest" is when it arrived and a test must not race the clock.
+func deliver(t *testing.T, dir, version string, at time.Time) string {
+	t.Helper()
+	into := filepath.Join(dir, version)
+	if err := os.MkdirAll(into, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	binary := filepath.Join(into, BinaryName)
+	if err := os.WriteFile(binary, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chtimes(binary, at, at); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chtimes(into, at, at); err != nil {
+		t.Fatal(err)
+	}
+	return binary
+}
+
+// **Newest is when it arrived, not how its name sorts.**
+//
+// A version string is whatever the source was tagged or described as, and those do not sort: "1.10"
+// orders before "1.9", and a commit hash orders before either. Ordering by name would start an older
+// host and call that an upgrade.
+func TestNewestIsWhenItArrivedAndNotHowItSorts(t *testing.T) {
+	dir := t.TempDir()
+	base := time.Now().Add(-time.Hour)
+	deliver(t, dir, "1.10", base)                 // sorts LAST by name, arrived first
+	deliver(t, dir, "1.9", base.Add(time.Minute)) // sorts first by name, arrived last
+
+	got, err := Versions(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 2 || got[0].Version != "1.9" {
+		t.Fatalf("newest is %+v, want the one that arrived last (1.9)", got)
+	}
+}
+
+// A delivery that was interrupted leaves a directory with no executable in it. Running "the newest"
+// would then mean running nothing, so it is not a version.
+func TestADirectoryWithNoBinaryIsNotAVersion(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(dir, "half-delivered"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	deliver(t, dir, "good", time.Now().Add(-time.Hour))
+
+	got, err := Versions(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 1 || got[0].Version != "good" {
+		t.Fatalf("versions are %+v, want only the one with a binary", got)
+	}
+}
+
+// Nothing delivered is not a fault. A machine whose host was placed by hand has no versions
+// directory at all, and that must read as "no successor" rather than as an error that stops a
+// reconcile.
+func TestNoVersionsDirectoryIsNotAnError(t *testing.T) {
+	got, err := Versions(filepath.Join(t.TempDir(), "absent"))
+	if err != nil {
+		t.Fatalf("an absent versions directory should not be an error: %v", err)
+	}
+	if len(got) != 0 {
+		t.Fatalf("versions are %+v, want none", got)
+	}
+}
+
+func TestTheNewestVersionIsTheSuccessorAndTheRunningOneIsNot(t *testing.T) {
+	dir := t.TempDir()
+	base := time.Now().Add(-time.Hour)
+	deliver(t, dir, "one", base)
+	deliver(t, dir, "two", base.Add(time.Minute))
+
+	next, yes, err := Successor(dir, "one")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !yes || next.Version != "two" {
+		t.Fatalf("successor is %+v (%v), want two", next, yes)
+	}
+
+	if _, yes, err := Successor(dir, "two"); err != nil || yes {
+		t.Fatalf("the newest version is its own successor (%v, %v)", yes, err)
+	}
+}
+
+// A host put there by hand, before any of this existed, is not among the delivered versions. What the
+// mesh delivered is what it asked for, so that is a successor — otherwise the first delivery to such a
+// machine would be ignored for ever, which is every machine in this mesh today.
+func TestAHostThatWasNeverDeliveredHasASuccessor(t *testing.T) {
+	dir := t.TempDir()
+	deliver(t, dir, "delivered", time.Now().Add(-time.Hour))
+
+	next, yes, err := Successor(dir, "copied-by-hand")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !yes || next.Version != "delivered" {
+		t.Fatalf("successor is %+v (%v), want the delivered one", next, yes)
+	}
+}
+
+// The running version and its predecessor are kept, and nothing else. The predecessor is exactly what
+// a rollback starts; everything older has no reader.
+func TestRetireKeepsTheRunningVersionAndItsPredecessor(t *testing.T) {
+	dir := t.TempDir()
+	base := time.Now().Add(-4 * time.Hour)
+	for i, v := range []string{"one", "two", "three", "four"} {
+		deliver(t, dir, v, base.Add(time.Duration(i)*time.Hour))
+	}
+
+	removed, err := Retire(dir, "four")
+	if err != nil {
+		t.Fatal(err)
+	}
+	sort.Strings(removed)
+	if !reflect.DeepEqual(removed, []string{"one", "two"}) {
+		t.Fatalf("retired %v, want one and two — three is the predecessor a rollback needs", removed)
+	}
+	for _, kept := range []string{"three", "four"} {
+		if _, err := os.Stat(filepath.Join(dir, kept, BinaryName)); err != nil {
+			t.Fatalf("%s was retired and a rollback now has nowhere to go: %v", kept, err)
+		}
+	}
+}
+
+// **Never the running version, whatever it is asked.** A host that deleted its own image would run
+// until it stopped and then be unstartable, and the launcher's rollback reads a version rather than a
+// process.
+func TestRetireNeverRemovesTheRunningVersion(t *testing.T) {
+	dir := t.TempDir()
+	base := time.Now().Add(-2 * time.Hour)
+	deliver(t, dir, "older", base)
+	deliver(t, dir, "newer", base.Add(time.Hour))
+
+	// Asked while running the OLDER one, which is what a machine looks like between a delivery and
+	// the moment it stands aside.
+	if _, err := Retire(dir, "older"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "older", BinaryName)); err != nil {
+		t.Fatalf("the running version was retired: %v", err)
+	}
+}
+
+// A machine running a hand-placed host keeps the newest delivered version, because that is what a
+// rollback would reach for. Retiring it would leave the machine with no way back at all.
+func TestRetireKeepsTheNewestWhenTheRunningVersionWasNeverDelivered(t *testing.T) {
+	dir := t.TempDir()
+	base := time.Now().Add(-3 * time.Hour)
+	deliver(t, dir, "old", base)
+	deliver(t, dir, "new", base.Add(time.Hour))
+
+	removed, err := Retire(dir, "copied-by-hand")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(removed, []string{"old"}) {
+		t.Fatalf("retired %v, want only old — new is the rollback target", removed)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "new", BinaryName)); err != nil {
+		t.Fatalf("the only delivered version was retired: %v", err)
+	}
+}

packaging/launch_test.sh

@@ -187,5 +187,70 @@ sleep 1
 check "a crash is counted" "unlike a clean exit, which is not" "$(count)" "1"
 kill -TERM "$LP" 2>/dev/null; sleep 1; pkill -f "$MESH_HOST_BIN" 2>/dev/null || true
 
+# --- which version it runs (novox/hq ADR 0141) ------------------------------------------------
+#
+# Versions live side by side in directories named for them. The launcher picks one every time round
+# the loop, never once: standing aside for a successor is a clean exit, and the next turn has to run
+# what is on disk NOW — resolved once, the same binary would restart for ever and no upgrade would
+# ever take.
+
+# deliver a version as the mesh would, recording which one ran so a test can assert the choice.
+deliver() {
+	mkdir -p "$MESH_HOST_LIBEXEC/versions/$1"
+	cat > "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host" <<STUB
+#!/bin/sh
+echo "$1" >> "\$MESH_HOST_STATE_DIR/which.ran"
+exit "\${STUB_HOST_EXIT:-1}"
+STUB
+	chmod +x "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host"
+	# When it arrived is what "newest" means, so it is set rather than left to the clock.
+	touch -d "$2" "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host" "$MESH_HOST_LIBEXEC/versions/$1"
+}
+which_ran() { cat "$MESH_HOST_STATE_DIR/which.ran" 2>/dev/null || echo NONE; }
+
+# Newest is when it arrived, not how its name sorts: "1.10" orders before "1.9" by name, so ordering
+# by name would run an older host and call it an upgrade.
+setup
+deliver 1.10 "2 hours ago"
+deliver 1.9 "1 hour ago"
+"$LAUNCH" >/dev/null 2>&1 || true
+check "runs the newest delivered version" "newest is when it arrived, not how the name sorts" \
+	"$(which_ran)" "1.9"
+
+# A pin from a rollback beats the newest, or the launcher would start the failing binary again and
+# the rollback would flap.
+setup
+deliver 1.9 "2 hours ago"
+deliver 2.0 "1 hour ago"
+echo 1.9 > "$MESH_HOST_STATE_DIR/rollback-pinned"
+"$LAUNCH" >/dev/null 2>&1 || true
+check "a pinned version beats the newest" "otherwise a rollback starts the binary it just rejected" \
+	"$(which_ran)" "1.9"
+
+# A pin naming a version that is not there is ignored rather than fatal: the machine choosing for
+# itself is better than a machine that starts nothing.
+setup
+deliver 2.0 "1 hour ago"
+echo 1.9 > "$MESH_HOST_STATE_DIR/rollback-pinned"
+"$LAUNCH" >/dev/null 2>&1 || true
+check "an undeliverable pin is ignored" "a machine that starts nothing is worse than one that chooses" \
+	"$(which_ran)" "2.0"
+
+# An interrupted delivery leaves a directory with no binary in it. Treating it as the newest would
+# mean running nothing.
+setup
+deliver 1.9 "2 hours ago"
+mkdir -p "$MESH_HOST_LIBEXEC/versions/2.0-half"
+touch -d "1 minute ago" "$MESH_HOST_LIBEXEC/versions/2.0-half"
+"$LAUNCH" >/dev/null 2>&1 || true
+check "skips a version with no binary" "a directory is not a version; the binary is" \
+	"$(which_ran)" "1.9"
+
+# Nothing delivered: the host placed by hand, which is how the first one always arrives. Without this
+# the change would strand every machine in the mesh on the day it ships.
+setup
+"$LAUNCH" >/dev/null 2>&1 || true
+check "falls back to the host placed by hand" "every first host arrives this way" "$(started)" "yes"
+
 printf '\nlaunch: %d passed, %d failed\n' "$PASS" "$FAIL"
 [ "$FAIL" -eq 0 ]

packaging/nox-mesh-host-launch

@@ -16,16 +16,51 @@ set -u
 
 STATE_DIR="${MESH_HOST_STATE_DIR:-/var/lib/mesh-host}"
 LIBEXEC="${MESH_HOST_LIBEXEC:-/usr/lib/nox-mesh-host}"
-HOST="${MESH_HOST_BIN:-/usr/bin/nox-mesh-host}"
+# The host that was placed by hand, used only when nothing has been delivered. The first host on a
+# machine always arrives this way; every one after it is delivered (novox/hq ADR 0141).
+FALLBACK="${MESH_HOST_BIN:-/usr/bin/nox-mesh-host}"
+VERSIONS="$LIBEXEC/versions"
+BINARY="nox-mesh-host"
 LIMIT="${MESH_HOST_START_LIMIT:-3}"
 BACKOFF="${MESH_HOST_BACKOFF:-5}"
 ONCE="${MESH_HOST_RUN_ONCE:-}"   # tests run one iteration; nothing else sets this
 
 ATTEMPTS="$STATE_DIR/start-attempts"
 HALTED="$STATE_DIR/halted"
+PINNED="$STATE_DIR/rollback-pinned"
 
 say() { echo "nox-mesh-host-launch: $*" >&2; }
 
+# Which host to run: the version a rollback pinned, or the most recently delivered one, or the one
+# placed by hand when nothing has been delivered (novox/hq ADR 0141).
+#
+# **Asked every time round the loop, not once.** Standing aside for a successor is a clean exit, and
+# the next turn has to run what is on disk NOW — resolving this once would restart the same binary
+# for ever and the upgrade would never take.
+#
+# Newest by when it arrived, never by how its name sorts: a version string is whatever the source was
+# described as, and those do not sort — "1.10" orders before "1.9". Ordering by name would start an
+# older host and call it an upgrade.
+pick_host() {
+	if [ -s "$PINNED" ]; then
+		pinned="$(tr -d '[:space:]' < "$PINNED" 2>/dev/null || true)"
+		if [ -n "$pinned" ] && [ -x "$VERSIONS/$pinned/$BINARY" ]; then
+			echo "$VERSIONS/$pinned/$BINARY"
+			return 0
+		fi
+		say "the pinned version '$pinned' is not delivered; ignoring the pin"
+	fi
+	# A directory with no executable in it is not a version: an interrupted delivery leaves one, and
+	# running "the newest" would then mean running nothing.
+	for candidate in $(ls -1t "$VERSIONS" 2>/dev/null || true); do
+		if [ -x "$VERSIONS/$candidate/$BINARY" ]; then
+			echo "$VERSIONS/$candidate/$BINARY"
+			return 0
+		fi
+	done
+	echo "$FALLBACK"
+}
+
 child=
 stopping=
 
@@ -95,6 +130,14 @@ while :; do
 		fi
 	fi
 
+	HOST="$(pick_host)"
+	if [ ! -x "$HOST" ]; then
+		say "no host to run: nothing delivered under $VERSIONS and $FALLBACK is not executable."
+		printf 'no host binary\n' > "$HALTED"
+		exit 0
+	fi
+	say "running $HOST"
+
 	"$HOST" run &
 	child=$!
 	status=0

packaging/nox-mesh-host-rollback

@@ -1,20 +1,29 @@
 #!/bin/sh
 # Put the host back on the last version that worked.
 #
-# novox/hq ADR 0005. This runs when nox-mesh-host will not start, so it shares no code with it
-# and calls none of it: a binary that cannot start cannot be its own recovery. POSIX sh, no
+# novox/hq ADR 0005 and ADR 0141. This runs when nox-mesh-host will not start, so it shares no code
+# with it and calls none of it: a binary that cannot start cannot be its own recovery. POSIX sh, no
 # bashisms, nothing that has to be installed.
 #
-# It is deliberately dull. Everything it does is one of: read a file, run the package manager,
-# ask the service manager to try again.
+# It is deliberately dull. Everything it does is one of: read a file, look at a directory, write a
+# file.
+#
+# **It used to reinstall a package.** It read the known-good version and asked one operating system's
+# package manager for it, out of that package manager's cache. Two things were wrong with that. No
+# machine in this mesh had the host installed as a package, so the recovery could not run on any of
+# them; and the host is built per operating system (ADR 0005), so a recovery written in one package
+# manager's terms could not run on two of the three. Versions now live side by side in directories
+# named for them, so going back is choosing a directory — which is the same on every machine.
 set -eu
 
 STATE_DIR="${MESH_HOST_STATE_DIR:-/var/lib/mesh-host}"
-PKG_CACHE="${MESH_HOST_PKG_CACHE:-/var/cache/pacman/pkg}"
-PACKAGE="${MESH_HOST_PACKAGE:-nox-mesh-host}"
+LIBEXEC="${MESH_HOST_LIBEXEC:-/usr/lib/nox-mesh-host}"
+VERSIONS="$LIBEXEC/versions"
+BINARY="nox-mesh-host"
 
 KNOWN_GOOD="$STATE_DIR/known-good"
 ATTEMPTED="$STATE_DIR/rollback-attempted"
+PINNED="$STATE_DIR/rollback-pinned"
 
 say() { echo "nox-mesh-host-rollback: $*" >&2; }
 
@@ -43,23 +52,24 @@ if [ -z "$VERSION" ]; then
 	exit 0
 fi
 
-PKG="$(ls "$PKG_CACHE"/"$PACKAGE"-"$VERSION"-*.pkg.tar.* 2>/dev/null | head -n 1 || true)"
-if [ -z "$PKG" ]; then
-	say "known-good is $VERSION and no package for it is in $PKG_CACHE."
-	say "the cache was cleaned, or that version was never installed from here."
+# The version that last worked may be the one that was placed by hand, which is not delivered and has
+# no directory. Nothing to choose, and saying so is better than pinning a version that is not there —
+# the launcher would ignore the pin and start the newest again, which is the binary that is failing.
+if [ ! -x "$VERSIONS/$VERSION/$BINARY" ]; then
+	say "known-good is $VERSION and no such version is delivered under $VERSIONS."
+	say "it was retired, or that host was placed by hand and never delivered."
 	say "cannot roll back. this node needs a person."
 	exit 1
 fi
 
-say "rolling back to $VERSION ($PKG)"
+say "rolling back to $VERSION ($VERSIONS/$VERSION/$BINARY)"
 printf '%s\n' "$VERSION" > "$ATTEMPTED"
 
-if ! pacman -U --noconfirm "$PKG"; then
-	say "the package manager refused to install $PKG."
-	exit 1
-fi
+# The pin is what stops the launcher starting the newest again. Written last, so a failure above
+# leaves the machine choosing for itself rather than pinned to something this script did not verify.
+printf '%s\n' "$VERSION" > "$PINNED"
 
-# Deliberately does NOT start anything. The launcher called this and will exec the host next,
-# so starting it here would run two. novox/hq ADR 0005 moved that responsibility; this script
-# installs a version and says so, and nothing else.
-say "rolled back to $VERSION. the launcher will start it."
+# Deliberately does NOT start anything. The launcher called this and will run the host next, so
+# starting it here would run two. novox/hq ADR 0005 moved that responsibility; this script chooses a
+# version and says so, and nothing else.
+say "pinned $VERSION. the launcher will start it."

packaging/rollback_test.sh

@@ -1,9 +1,15 @@
 #!/bin/sh
 # Tests for nox-mesh-host-rollback.
 #
-# It runs on a machine where the host will not start, which is the one moment nobody can afford
-# it to be wrong — and the one moment it is hardest to debug. So it is tested here, against a
-# real filesystem, with a stub package manager that records what it was asked to do.
+# It runs on a machine where the host will not start, which is the one moment nobody can afford it to
+# be wrong — and the one moment it is hardest to debug. So it is tested here, against a real
+# filesystem holding real delivered versions.
+#
+# **These used to stub a package manager.** The script reinstalled the known-good version with
+# `pacman -U` out of the package cache, which no machine in this mesh used and which two of the three
+# operating systems the host is built for do not have (novox/hq ADR 0141). Going back is now choosing
+# a directory, so there is nothing to stub: the thing under test is the filesystem, and a fake would
+# only assert that the fake behaves as expected (novox/hq ADR 0017).
 set -eu
 cd "$(dirname "$0")"
 SCRIPT="$PWD/nox-mesh-host-rollback"
@@ -12,26 +18,15 @@ PASS=0; FAIL=0
 setup() {
 	WORK="$(mktemp -d)"
 	export MESH_HOST_STATE_DIR="$WORK/state"
-	export MESH_HOST_PKG_CACHE="$WORK/cache"
-	export MESH_HOST_PACKAGE="nox-mesh-host"
-	mkdir -p "$MESH_HOST_STATE_DIR" "$MESH_HOST_PKG_CACHE" "$WORK/bin"
+	export MESH_HOST_LIBEXEC="$WORK/libexec"
+	mkdir -p "$MESH_HOST_STATE_DIR" "$MESH_HOST_LIBEXEC/versions"
+}
 
-	# Stubs on PATH. Not mocks of the script's own logic — the boundary is real commands, and
-	# these record the calls so a test can assert what the script asked the machine to do.
-	cat > "$WORK/bin/pacman" <<'STUB'
-#!/bin/sh
-echo "$@" >> "$MESH_HOST_STATE_DIR/pacman.calls"
-[ -n "${STUB_PACMAN_FAILS:-}" ] && exit 1
-exit 0
-STUB
-	cat > "$WORK/bin/systemctl" <<'STUB'
-#!/bin/sh
-echo "$@" >> "$MESH_HOST_STATE_DIR/systemctl.calls"
-exit 0
-STUB
-	chmod +x "$WORK/bin/pacman" "$WORK/bin/systemctl"
-	PATH="$WORK/bin:$PATH"; export PATH
-	unset STUB_PACMAN_FAILS || true
+# deliver a version the way the mesh would: a directory named for it, with the binary inside.
+deliver() {
+	mkdir -p "$MESH_HOST_LIBEXEC/versions/$1"
+	printf '#!/bin/sh\nexit 0\n' > "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host"
+	chmod +x "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host"
 }
 
 check() { # name, condition-description, actual, expected
@@ -41,32 +36,38 @@ check() { # name, condition-description, actual, expected
 
 # --- a normal rollback ---------------------------------------------------------------------
 setup
-echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
-touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst"
-"$SCRIPT" >/dev/null 2>&1
-check "installs the known-good version" "pacman is asked to install the cached package" \
-	"$(grep -c 'nox-mesh-host-1.4.2' "$MESH_HOST_STATE_DIR/pacman.calls" 2>/dev/null || echo 0)" "1"
-# It installs and stops. The launcher execs the host next, and starting it here would run two
-# (novox/hq ADR 0005).
-check "does not start anything itself" "the launcher owns starting" \
-	"$([ -f "$MESH_HOST_STATE_DIR/systemctl.calls" ] && echo started || echo not-started)" "not-started"
+deliver 1.4.2
+deliver 1.5.0
+echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good"
+RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$?
+check "pins the known-good version" "the launcher reads the pin and runs that version instead of the newest" \
+	"$(cat "$MESH_HOST_STATE_DIR/rollback-pinned" 2>/dev/null || echo MISSING)" "1.4.2"
 check "records that it rolled back" "the attempted marker holds the version" \
 	"$(cat "$MESH_HOST_STATE_DIR/rollback-attempted" 2>/dev/null || echo MISSING)" "1.4.2"
+check "succeeds" "a rollback that found its version is not a failure" "$RC" "0"
+# It chooses and stops. The launcher runs the host next, and starting it here would run two
+# (novox/hq ADR 0005).
+check "does not start anything itself" "the launcher owns starting" \
+	"$(ls "$MESH_HOST_STATE_DIR" | grep -c started || true)" "0"
+# The version it rolled back FROM is left alone: it is the newest, and retiring it is the running
+# host's job after a reconcile it completes, never a recovery's.
+check "leaves the failing version on disk" "a recovery deletes nothing" \
+	"$([ -x "$MESH_HOST_LIBEXEC/versions/1.5.0/nox-mesh-host" ] && echo present || echo gone)" "present"
 
 # --- it rolls back only once ---------------------------------------------------------------
 setup
-echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
-echo "1.4.2" > "$MESH_HOST_STATE_DIR/rollback-attempted"
-touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst"
-"$SCRIPT" >/dev/null 2>&1
+deliver 1.4.2
+echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good"
+echo 1.4.2 > "$MESH_HOST_STATE_DIR/rollback-attempted"
+"$SCRIPT" >/dev/null 2>&1 || true
 check "does not roll back twice" "a second failure is the machine, not the binary" \
-	"$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called"
+	"$([ -e "$MESH_HOST_STATE_DIR/rollback-pinned" ] && echo pinned || echo untouched)" "untouched"
 
 # --- nothing to roll back to ---------------------------------------------------------------
 setup
-set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e
+RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$?
 check "no known-good: does nothing" "a host that never reconciled has no version to return to" \
-	"$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called"
+	"$([ -e "$MESH_HOST_STATE_DIR/rollback-attempted" ] && echo attempted || echo untouched)" "untouched"
 # The exit code is asserted from a real run, not from a literal. An earlier version of this
 # compared "0" to "0" and could not fail — which hid an injected fault that made the script die
 # here instead of returning cleanly.
@@ -74,23 +75,29 @@ check "no known-good: exits zero" "an installation failure is not a rollback fai
 
 setup
 printf '   \n' > "$MESH_HOST_STATE_DIR/known-good"
-"$SCRIPT" >/dev/null 2>&1
-check "blank known-good: refuses to guess" "installing nothing and reporting success is the fault this prevents" \
-	"$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called"
+"$SCRIPT" >/dev/null 2>&1 || true
+check "blank known-good: refuses to guess" "pinning nothing and reporting success is the fault this prevents" \
+	"$([ -e "$MESH_HOST_STATE_DIR/rollback-pinned" ] && echo pinned || echo untouched)" "untouched"
 
-# --- the cache was cleaned ------------------------------------------------------------------
+# --- the known-good version is not delivered -------------------------------------------------
+# It was retired, or that host was placed on the machine by hand and never delivered — which is how
+# every first host arrives. Pinning it anyway would have the launcher ignore the pin and start the
+# newest again, which is the binary that is failing.
 setup
-echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
-set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e
-check "missing package: fails loudly" "cannot roll back, and says so rather than reporting success" "$RC" "1"
+deliver 1.5.0
+echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good"
+RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$?
+check "version not delivered: fails loudly" "cannot roll back, and says so rather than reporting success" "$RC" "1"
+check "version not delivered: pins nothing" "a pin the launcher would ignore is worse than none" \
+	"$([ -e "$MESH_HOST_STATE_DIR/rollback-pinned" ] && echo pinned || echo untouched)" "untouched"
 
-# --- the package manager refuses -------------------------------------------------------------
+# --- a version directory with no binary in it ------------------------------------------------
+# An interrupted delivery leaves one. Pinning it would start nothing.
 setup
-echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
-touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst"
-STUB_PACMAN_FAILS=1 ; export STUB_PACMAN_FAILS
-set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e
-check "pacman fails: exits non-zero" "a failed rollback is a failure the launcher must see" "$RC" "1"
+mkdir -p "$MESH_HOST_LIBEXEC/versions/1.4.2"
+echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good"
+RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$?
+check "half-delivered version: fails loudly" "a directory is not a version; the binary is" "$RC" "1"
 
 printf '\nrollback: %d passed, %d failed\n' "$PASS" "$FAIL"
 [ "$FAIL" -eq 0 ]