From fbf0fb7d63d0b8becbd49bd853c7199af813b77b Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 29 Sep 2026 00:29:36 +0200 Subject: [PATCH] The host delivers its own successor, and versions live side by side MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The supervision was already right: a clean exit means the host stood aside, and the launcher's next turn runs what is on disk. Two things made it dead code — nothing told the running host a successor was waiting, and the rollback resolved its known-good version through pacman, which no machine here uses and which two of three operating systems do not have. Keeping a version rather than a path was the clue. Versions now live in directories named for them: - the launcher picks the newest delivered one every time round the loop, or the one a rollback pinned, or the host placed by hand when nothing is delivered; - the running host stands aside between reconciles, never inside one, by exiting cleanly — and returns nil so the launcher does not count it as a crash; - a completed reconcile retires what is older than the predecessor, keeping the predecessor because that is what a rollback starts, and never the running one; - rollback pins the predecessor instead of reinstalling a package: no package manager, no cache anyone may clean, same script on every operating system; - the report says which host version produced it, so 'behind' is answerable. Newest is when it arrived, never how the name sorts: '1.10' orders before '1.9', and ordering by name would start an older host and call it an upgrade. novox/hq ADR 0141. The delivery half — a module carrying the next host — follows; until then nothing delivers a version and every machine takes the fallback, which is what it does today. --- cmd/mesh-host/main.go | 44 +++++++- internal/link/messages.go | 7 ++ internal/upgrade/upgrade.go | 167 +++++++++++++++++++++++++++ internal/upgrade/versions_test.go | 180 ++++++++++++++++++++++++++++++ packaging/launch_test.sh | 65 +++++++++++ packaging/nox-mesh-host-launch | 45 +++++++- packaging/nox-mesh-host-rollback | 48 ++++---- packaging/rollback_test.sh | 109 +++++++++--------- 8 files changed, 592 insertions(+), 73 deletions(-) create mode 100644 internal/upgrade/versions_test.go diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 7c56afc..30d91ef 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -601,6 +601,20 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw " a rollback would have nothing to return to.\n", version, err) } } + // And retire what is older than this version's predecessor, on the same evidence known-good is + // written on (novox/hq ADR 0141). The predecessor stays, because it is exactly what a rollback + // starts; everything before it has no reader. Never this version, whatever the answer. + // + // A failure is said and does not fail the apply, for the reason above: what is lost is disk, and + // hiding it would make a machine quietly fill up. + if version != "" { + if retired, err := upgrade.Retire(upgrade.VersionsDir(""), version); err != nil { + fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not retire an older host: %v\n", err) + } else if len(retired) > 0 { + fmt.Fprintf(os.Stderr, "mesh-host: retired the host version(s) %s\n", + strings.Join(retired, ", ")) + } + } // And tell the launcher this start worked. Without it the counter only climbs, and a node // that has been up for months rolls itself back on its third ordinary restart. if err := upgrade.ClearAttempts(upgrade.AttemptsPath(opts.state)); err != nil { @@ -979,12 +993,31 @@ func runLink(ctx context.Context, opts options) error { sched := apply.NewScheduler(apply.SystemClock(), apply.ExecRunner, say) go sched.Run(ctx) + // **Standing aside for a successor happens between reconciles and nowhere else** (novox/hq ADR + // 0141). A host that stood aside mid-apply is the half-configured machine this host exists to + // prevent, so the question is asked after an apply has finished and the answer is a clean exit — + // which the launcher already reads as "run whatever is on disk now". + aside, standAside := context.WithCancel(ctx) + defer standAside() + stoodAside := false + applier := func(ctx context.Context, raw, signature []byte) link.Report { report := applyAndKeep(ctx, opts, raw, &store.Declared{Declaration: raw, Signature: signature}, sched, say) // **A declaration may carry this machine's membership for another bus.** It arrives as a // sealed file like any secret, and is read after the rest has applied so the bus it names is // standing before this machine leaves the one it is on (novox/hq design 28, task 5.2). adoptDeliveredMembership(identity.Path(opts.state), &mine, say) + + switch next, waiting, err := upgrade.Successor(upgrade.VersionsDir(""), version); { + case err != nil: + // Said, not fatal. A host that cannot read the delivered versions is still running this + // machine correctly; what it has lost is the ability to be replaced. + say(fmt.Sprintf("cannot tell whether a newer host is delivered: %v", err)) + case waiting: + say(fmt.Sprintf("host %s is delivered; standing aside so the launcher runs it", next.Version)) + stoodAside = true + standAside() + } return report } @@ -1015,7 +1048,7 @@ func runLink(ctx context.Context, opts options) error { }} }) - return link.HoldRoused(ctx, link.Membership{ + held := link.HoldRoused(aside, link.Membership{ Node: mine.Node, Broker: mine.Membership.Broker, Fingerprint: mine.Membership.Fingerprint, @@ -1023,6 +1056,13 @@ func runLink(ctx context.Context, opts options) error { Transport: mine.Membership.Transport, Signer: mine.Membership.Signer, }, applier, say, opts.timeout, rousedBySignal(ctx), outbox) + // **Cleanly**, or the launcher counts standing aside as a crash and rolls the new host back + // before it has run once. The context this returns on was cancelled deliberately, so its error + // is not a fault to report. + if stoodAside { + return nil + } + return held } // adoptionWatch remembers what the node last said about what it holds and its firewall, so a @@ -1263,7 +1303,7 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D sched.Sync(declared, held) } - report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw)} + report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw), Host: version} // Which of this machine's links face outside, for the filter the mesh writes around them // (novox/hq ADR 0140). Reported whatever the node's mode: a converged node's filter needs it, // and an adopted one becomes converged without a further round trip. A machine that cannot read diff --git a/internal/link/messages.go b/internal/link/messages.go index f560fa4..ca9d0ce 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -110,6 +110,13 @@ type Report struct { // and the mesh's up in its place, and where the found configuration's original was kept. Tunnel *CarriedTunnel `json:"tunnel,omitempty"` + // Host is the version of the host that produced this report (novox/hq ADR 0141). + // + // Without it nothing can say a machine is behind, so "every machine current with its source" + // could not include the host — the one component the mesh did not deliver. It is a fact the + // machine states about itself, like the firewall it found and the links that face outside. + Host string `json:"host,omitempty"` + // Outward is the links on this machine that face outside it — the ones carrying a default // route (novox/hq ADR 0140). Every node reports it, adopted or converged, because a converged // node's filter is written around it. diff --git a/internal/upgrade/upgrade.go b/internal/upgrade/upgrade.go index a8d2647..acde450 100644 --- a/internal/upgrade/upgrade.go +++ b/internal/upgrade/upgrade.go @@ -16,7 +16,9 @@ import ( "fmt" "os" "path/filepath" + "sort" "strings" + "time" ) // Files the launcher reads and this binary writes. Next to the store, because they are node @@ -157,3 +159,168 @@ func ReadKnownGood(path string) (string, error) { } return strings.TrimSpace(string(raw)), nil } + +// Where delivered versions live, and what the binary inside one is called. +// +// **A directory named for its version, never a link and never a write over what is running** +// (novox/hq ADR 0141). Two facts follow from that one choice: the kernel refuses to truncate a +// running executable, so the path a delivery writes must not be the path being executed; and a +// rollback needs the previous version still present, which a single path cannot offer. +// +// The mesh creates no links (novox/hq ADR 0012), so nothing points at "current". The version is in +// the path, which is why nothing has to be told what is running. +const ( + // DefaultLibexec is where the host's own files live. Fixed rather than derived from where the + // running executable sits: the first host to understand any of this was copied to a machine by + // hand, and one that looked for its successor beside itself would never find a delivered version + // — which is every machine in this mesh on the day this ships. + DefaultLibexec = "/usr/lib/nox-mesh-host" + VersionsDirName = "versions" + BinaryName = "nox-mesh-host" + // PinnedName is the version a rollback chose, which the launcher runs instead of the newest. + // Without it the launcher would start the newest again and the rollback would flap. + PinnedName = "rollback-pinned" +) + +// VersionsDir is where delivered versions live, given where the host's libexec is. An empty libexec +// means the default, and the environment overrides it so a test needs no root. +func VersionsDir(libexec string) string { + if libexec == "" { + libexec = os.Getenv("MESH_HOST_LIBEXEC") + } + if libexec == "" { + libexec = DefaultLibexec + } + return filepath.Join(libexec, VersionsDirName) +} + +// PinnedPath is where a rollback records the version it chose. +func PinnedPath(statePath string) string { + return filepath.Join(filepath.Dir(statePath), PinnedName) +} + +// Delivered is one version present on the machine. +type Delivered struct { + // Version is the directory's name, which is the version. + Version string + // Binary is the executable inside it. + Binary string + // At is when it arrived, which is how "newest" is decided. + At time.Time +} + +// Versions are the versions delivered to this machine, newest first. +// +// **Newest by when it arrived, not by its name.** A version string comes from what the source was +// tagged or described as, and those do not sort: "1.10" before "1.9", a commit hash before either. +// Ordering by name would run an older host and call it an upgrade. When it arrived is a fact the +// filesystem keeps and the delivery sets. +// +// A directory with no executable in it is not a version. A delivery that was interrupted leaves one, +// and running the newest would then mean running nothing. +func Versions(dir string) ([]Delivered, error) { + entries, err := os.ReadDir(dir) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("cannot read the delivered versions at %s: %w", dir, err) + } + + var out []Delivered + for _, entry := range entries { + if !entry.IsDir() { + continue + } + binary := filepath.Join(dir, entry.Name(), BinaryName) + info, err := os.Stat(binary) + if err != nil || info.IsDir() { + continue + } + at := info.ModTime() + if d, err := entry.Info(); err == nil && d.ModTime().After(at) { + at = d.ModTime() + } + out = append(out, Delivered{Version: entry.Name(), Binary: binary, At: at}) + } + + // Newest first, and by name when two arrived in the same instant so the answer is never + // arbitrary — a test that passes half the time is worse than one that fails. + sort.Slice(out, func(a, b int) bool { + if out[a].At.Equal(out[b].At) { + return out[a].Version > out[b].Version + } + return out[a].At.After(out[b].At) + }) + return out, nil +} + +// Successor is the version this machine should be running instead of the given one, if any. +// +// Empty when the running version is the newest, which is the ordinary answer. The host asks this +// between reconciles and nowhere else: standing aside mid-apply is the half-configured machine the +// host exists to prevent (novox/hq ADR 0141). +func Successor(dir, running string) (Delivered, bool, error) { + delivered, err := Versions(dir) + if err != nil { + return Delivered{}, false, err + } + if len(delivered) == 0 { + return Delivered{}, false, nil + } + newest := delivered[0] + // A machine whose running version is not among the delivered ones is the machine every mesh has + // one of: the host was put there by hand before any of this existed. Treating that as "stand + // aside" is correct — what was delivered is what the mesh asked for. + if newest.Version == running { + return Delivered{}, false, nil + } + return newest, true, nil +} + +// Retire removes delivered versions older than the running one's predecessor. +// +// The running version and the one before it are kept, and nothing else: the predecessor is exactly +// what a rollback starts, and every version before that is weight with no reader. Called after a +// reconcile completes, which is the same evidence known-good is written on — retiring on any weaker +// signal would delete the thing a failing host is about to need. +// +// Never the running version, whatever it is asked. A host that deleted its own image would survive +// until it stopped and then be unstartable, and the launcher's rollback reads a version, not a +// process. +func Retire(dir, running string) ([]string, error) { + delivered, err := Versions(dir) + if err != nil { + return nil, err + } + + keep := map[string]bool{running: true} + for i, d := range delivered { + if d.Version != running { + continue + } + // Its predecessor is the next one down the list, which is the next oldest. + if i+1 < len(delivered) { + keep[delivered[i+1].Version] = true + } + break + } + // A running version that was never delivered has no predecessor among these, so the newest + // delivered one is what a rollback would reach for. Keep it. + if len(keep) == 1 && len(delivered) > 0 { + keep[delivered[0].Version] = true + } + + var removed []string + for _, d := range delivered { + if keep[d.Version] { + continue + } + if err := os.RemoveAll(filepath.Join(dir, d.Version)); err != nil { + return removed, fmt.Errorf("cannot retire the host version %s: %w", d.Version, err) + } + removed = append(removed, d.Version) + } + sort.Strings(removed) + return removed, nil +} diff --git a/internal/upgrade/versions_test.go b/internal/upgrade/versions_test.go new file mode 100644 index 0000000..03ebc62 --- /dev/null +++ b/internal/upgrade/versions_test.go @@ -0,0 +1,180 @@ +package upgrade + +import ( + "os" + "path/filepath" + "reflect" + "sort" + "testing" + "time" +) + +// deliver writes a version as a delivery would: a directory named for it with the binary inside. +// at fixes when it arrived, because "newest" is when it arrived and a test must not race the clock. +func deliver(t *testing.T, dir, version string, at time.Time) string { + t.Helper() + into := filepath.Join(dir, version) + if err := os.MkdirAll(into, 0o755); err != nil { + t.Fatal(err) + } + binary := filepath.Join(into, BinaryName) + if err := os.WriteFile(binary, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(binary, at, at); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(into, at, at); err != nil { + t.Fatal(err) + } + return binary +} + +// **Newest is when it arrived, not how its name sorts.** +// +// A version string is whatever the source was tagged or described as, and those do not sort: "1.10" +// orders before "1.9", and a commit hash orders before either. Ordering by name would start an older +// host and call that an upgrade. +func TestNewestIsWhenItArrivedAndNotHowItSorts(t *testing.T) { + dir := t.TempDir() + base := time.Now().Add(-time.Hour) + deliver(t, dir, "1.10", base) // sorts LAST by name, arrived first + deliver(t, dir, "1.9", base.Add(time.Minute)) // sorts first by name, arrived last + + got, err := Versions(dir) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || got[0].Version != "1.9" { + t.Fatalf("newest is %+v, want the one that arrived last (1.9)", got) + } +} + +// A delivery that was interrupted leaves a directory with no executable in it. Running "the newest" +// would then mean running nothing, so it is not a version. +func TestADirectoryWithNoBinaryIsNotAVersion(t *testing.T) { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "half-delivered"), 0o755); err != nil { + t.Fatal(err) + } + deliver(t, dir, "good", time.Now().Add(-time.Hour)) + + got, err := Versions(dir) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].Version != "good" { + t.Fatalf("versions are %+v, want only the one with a binary", got) + } +} + +// Nothing delivered is not a fault. A machine whose host was placed by hand has no versions +// directory at all, and that must read as "no successor" rather than as an error that stops a +// reconcile. +func TestNoVersionsDirectoryIsNotAnError(t *testing.T) { + got, err := Versions(filepath.Join(t.TempDir(), "absent")) + if err != nil { + t.Fatalf("an absent versions directory should not be an error: %v", err) + } + if len(got) != 0 { + t.Fatalf("versions are %+v, want none", got) + } +} + +func TestTheNewestVersionIsTheSuccessorAndTheRunningOneIsNot(t *testing.T) { + dir := t.TempDir() + base := time.Now().Add(-time.Hour) + deliver(t, dir, "one", base) + deliver(t, dir, "two", base.Add(time.Minute)) + + next, yes, err := Successor(dir, "one") + if err != nil { + t.Fatal(err) + } + if !yes || next.Version != "two" { + t.Fatalf("successor is %+v (%v), want two", next, yes) + } + + if _, yes, err := Successor(dir, "two"); err != nil || yes { + t.Fatalf("the newest version is its own successor (%v, %v)", yes, err) + } +} + +// A host put there by hand, before any of this existed, is not among the delivered versions. What the +// mesh delivered is what it asked for, so that is a successor — otherwise the first delivery to such a +// machine would be ignored for ever, which is every machine in this mesh today. +func TestAHostThatWasNeverDeliveredHasASuccessor(t *testing.T) { + dir := t.TempDir() + deliver(t, dir, "delivered", time.Now().Add(-time.Hour)) + + next, yes, err := Successor(dir, "copied-by-hand") + if err != nil { + t.Fatal(err) + } + if !yes || next.Version != "delivered" { + t.Fatalf("successor is %+v (%v), want the delivered one", next, yes) + } +} + +// The running version and its predecessor are kept, and nothing else. The predecessor is exactly what +// a rollback starts; everything older has no reader. +func TestRetireKeepsTheRunningVersionAndItsPredecessor(t *testing.T) { + dir := t.TempDir() + base := time.Now().Add(-4 * time.Hour) + for i, v := range []string{"one", "two", "three", "four"} { + deliver(t, dir, v, base.Add(time.Duration(i)*time.Hour)) + } + + removed, err := Retire(dir, "four") + if err != nil { + t.Fatal(err) + } + sort.Strings(removed) + if !reflect.DeepEqual(removed, []string{"one", "two"}) { + t.Fatalf("retired %v, want one and two — three is the predecessor a rollback needs", removed) + } + for _, kept := range []string{"three", "four"} { + if _, err := os.Stat(filepath.Join(dir, kept, BinaryName)); err != nil { + t.Fatalf("%s was retired and a rollback now has nowhere to go: %v", kept, err) + } + } +} + +// **Never the running version, whatever it is asked.** A host that deleted its own image would run +// until it stopped and then be unstartable, and the launcher's rollback reads a version rather than a +// process. +func TestRetireNeverRemovesTheRunningVersion(t *testing.T) { + dir := t.TempDir() + base := time.Now().Add(-2 * time.Hour) + deliver(t, dir, "older", base) + deliver(t, dir, "newer", base.Add(time.Hour)) + + // Asked while running the OLDER one, which is what a machine looks like between a delivery and + // the moment it stands aside. + if _, err := Retire(dir, "older"); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dir, "older", BinaryName)); err != nil { + t.Fatalf("the running version was retired: %v", err) + } +} + +// A machine running a hand-placed host keeps the newest delivered version, because that is what a +// rollback would reach for. Retiring it would leave the machine with no way back at all. +func TestRetireKeepsTheNewestWhenTheRunningVersionWasNeverDelivered(t *testing.T) { + dir := t.TempDir() + base := time.Now().Add(-3 * time.Hour) + deliver(t, dir, "old", base) + deliver(t, dir, "new", base.Add(time.Hour)) + + removed, err := Retire(dir, "copied-by-hand") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(removed, []string{"old"}) { + t.Fatalf("retired %v, want only old — new is the rollback target", removed) + } + if _, err := os.Stat(filepath.Join(dir, "new", BinaryName)); err != nil { + t.Fatalf("the only delivered version was retired: %v", err) + } +} diff --git a/packaging/launch_test.sh b/packaging/launch_test.sh index 014e246..ab8fbc2 100755 --- a/packaging/launch_test.sh +++ b/packaging/launch_test.sh @@ -187,5 +187,70 @@ sleep 1 check "a crash is counted" "unlike a clean exit, which is not" "$(count)" "1" kill -TERM "$LP" 2>/dev/null; sleep 1; pkill -f "$MESH_HOST_BIN" 2>/dev/null || true +# --- which version it runs (novox/hq ADR 0141) ------------------------------------------------ +# +# Versions live side by side in directories named for them. The launcher picks one every time round +# the loop, never once: standing aside for a successor is a clean exit, and the next turn has to run +# what is on disk NOW — resolved once, the same binary would restart for ever and no upgrade would +# ever take. + +# deliver a version as the mesh would, recording which one ran so a test can assert the choice. +deliver() { + mkdir -p "$MESH_HOST_LIBEXEC/versions/$1" + cat > "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host" <> "\$MESH_HOST_STATE_DIR/which.ran" +exit "\${STUB_HOST_EXIT:-1}" +STUB + chmod +x "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host" + # When it arrived is what "newest" means, so it is set rather than left to the clock. + touch -d "$2" "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host" "$MESH_HOST_LIBEXEC/versions/$1" +} +which_ran() { cat "$MESH_HOST_STATE_DIR/which.ran" 2>/dev/null || echo NONE; } + +# Newest is when it arrived, not how its name sorts: "1.10" orders before "1.9" by name, so ordering +# by name would run an older host and call it an upgrade. +setup +deliver 1.10 "2 hours ago" +deliver 1.9 "1 hour ago" +"$LAUNCH" >/dev/null 2>&1 || true +check "runs the newest delivered version" "newest is when it arrived, not how the name sorts" \ + "$(which_ran)" "1.9" + +# A pin from a rollback beats the newest, or the launcher would start the failing binary again and +# the rollback would flap. +setup +deliver 1.9 "2 hours ago" +deliver 2.0 "1 hour ago" +echo 1.9 > "$MESH_HOST_STATE_DIR/rollback-pinned" +"$LAUNCH" >/dev/null 2>&1 || true +check "a pinned version beats the newest" "otherwise a rollback starts the binary it just rejected" \ + "$(which_ran)" "1.9" + +# A pin naming a version that is not there is ignored rather than fatal: the machine choosing for +# itself is better than a machine that starts nothing. +setup +deliver 2.0 "1 hour ago" +echo 1.9 > "$MESH_HOST_STATE_DIR/rollback-pinned" +"$LAUNCH" >/dev/null 2>&1 || true +check "an undeliverable pin is ignored" "a machine that starts nothing is worse than one that chooses" \ + "$(which_ran)" "2.0" + +# An interrupted delivery leaves a directory with no binary in it. Treating it as the newest would +# mean running nothing. +setup +deliver 1.9 "2 hours ago" +mkdir -p "$MESH_HOST_LIBEXEC/versions/2.0-half" +touch -d "1 minute ago" "$MESH_HOST_LIBEXEC/versions/2.0-half" +"$LAUNCH" >/dev/null 2>&1 || true +check "skips a version with no binary" "a directory is not a version; the binary is" \ + "$(which_ran)" "1.9" + +# Nothing delivered: the host placed by hand, which is how the first one always arrives. Without this +# the change would strand every machine in the mesh on the day it ships. +setup +"$LAUNCH" >/dev/null 2>&1 || true +check "falls back to the host placed by hand" "every first host arrives this way" "$(started)" "yes" + printf '\nlaunch: %d passed, %d failed\n' "$PASS" "$FAIL" [ "$FAIL" -eq 0 ] diff --git a/packaging/nox-mesh-host-launch b/packaging/nox-mesh-host-launch index b2a5495..abf911b 100755 --- a/packaging/nox-mesh-host-launch +++ b/packaging/nox-mesh-host-launch @@ -16,16 +16,51 @@ set -u STATE_DIR="${MESH_HOST_STATE_DIR:-/var/lib/mesh-host}" LIBEXEC="${MESH_HOST_LIBEXEC:-/usr/lib/nox-mesh-host}" -HOST="${MESH_HOST_BIN:-/usr/bin/nox-mesh-host}" +# The host that was placed by hand, used only when nothing has been delivered. The first host on a +# machine always arrives this way; every one after it is delivered (novox/hq ADR 0141). +FALLBACK="${MESH_HOST_BIN:-/usr/bin/nox-mesh-host}" +VERSIONS="$LIBEXEC/versions" +BINARY="nox-mesh-host" LIMIT="${MESH_HOST_START_LIMIT:-3}" BACKOFF="${MESH_HOST_BACKOFF:-5}" ONCE="${MESH_HOST_RUN_ONCE:-}" # tests run one iteration; nothing else sets this ATTEMPTS="$STATE_DIR/start-attempts" HALTED="$STATE_DIR/halted" +PINNED="$STATE_DIR/rollback-pinned" say() { echo "nox-mesh-host-launch: $*" >&2; } +# Which host to run: the version a rollback pinned, or the most recently delivered one, or the one +# placed by hand when nothing has been delivered (novox/hq ADR 0141). +# +# **Asked every time round the loop, not once.** Standing aside for a successor is a clean exit, and +# the next turn has to run what is on disk NOW — resolving this once would restart the same binary +# for ever and the upgrade would never take. +# +# Newest by when it arrived, never by how its name sorts: a version string is whatever the source was +# described as, and those do not sort — "1.10" orders before "1.9". Ordering by name would start an +# older host and call it an upgrade. +pick_host() { + if [ -s "$PINNED" ]; then + pinned="$(tr -d '[:space:]' < "$PINNED" 2>/dev/null || true)" + if [ -n "$pinned" ] && [ -x "$VERSIONS/$pinned/$BINARY" ]; then + echo "$VERSIONS/$pinned/$BINARY" + return 0 + fi + say "the pinned version '$pinned' is not delivered; ignoring the pin" + fi + # A directory with no executable in it is not a version: an interrupted delivery leaves one, and + # running "the newest" would then mean running nothing. + for candidate in $(ls -1t "$VERSIONS" 2>/dev/null || true); do + if [ -x "$VERSIONS/$candidate/$BINARY" ]; then + echo "$VERSIONS/$candidate/$BINARY" + return 0 + fi + done + echo "$FALLBACK" +} + child= stopping= @@ -95,6 +130,14 @@ while :; do fi fi + HOST="$(pick_host)" + if [ ! -x "$HOST" ]; then + say "no host to run: nothing delivered under $VERSIONS and $FALLBACK is not executable." + printf 'no host binary\n' > "$HALTED" + exit 0 + fi + say "running $HOST" + "$HOST" run & child=$! status=0 diff --git a/packaging/nox-mesh-host-rollback b/packaging/nox-mesh-host-rollback index f0608a3..5ae56b1 100755 --- a/packaging/nox-mesh-host-rollback +++ b/packaging/nox-mesh-host-rollback @@ -1,20 +1,29 @@ #!/bin/sh # Put the host back on the last version that worked. # -# novox/hq ADR 0005. This runs when nox-mesh-host will not start, so it shares no code with it -# and calls none of it: a binary that cannot start cannot be its own recovery. POSIX sh, no +# novox/hq ADR 0005 and ADR 0141. This runs when nox-mesh-host will not start, so it shares no code +# with it and calls none of it: a binary that cannot start cannot be its own recovery. POSIX sh, no # bashisms, nothing that has to be installed. # -# It is deliberately dull. Everything it does is one of: read a file, run the package manager, -# ask the service manager to try again. +# It is deliberately dull. Everything it does is one of: read a file, look at a directory, write a +# file. +# +# **It used to reinstall a package.** It read the known-good version and asked one operating system's +# package manager for it, out of that package manager's cache. Two things were wrong with that. No +# machine in this mesh had the host installed as a package, so the recovery could not run on any of +# them; and the host is built per operating system (ADR 0005), so a recovery written in one package +# manager's terms could not run on two of the three. Versions now live side by side in directories +# named for them, so going back is choosing a directory — which is the same on every machine. set -eu STATE_DIR="${MESH_HOST_STATE_DIR:-/var/lib/mesh-host}" -PKG_CACHE="${MESH_HOST_PKG_CACHE:-/var/cache/pacman/pkg}" -PACKAGE="${MESH_HOST_PACKAGE:-nox-mesh-host}" +LIBEXEC="${MESH_HOST_LIBEXEC:-/usr/lib/nox-mesh-host}" +VERSIONS="$LIBEXEC/versions" +BINARY="nox-mesh-host" KNOWN_GOOD="$STATE_DIR/known-good" ATTEMPTED="$STATE_DIR/rollback-attempted" +PINNED="$STATE_DIR/rollback-pinned" say() { echo "nox-mesh-host-rollback: $*" >&2; } @@ -43,23 +52,24 @@ if [ -z "$VERSION" ]; then exit 0 fi -PKG="$(ls "$PKG_CACHE"/"$PACKAGE"-"$VERSION"-*.pkg.tar.* 2>/dev/null | head -n 1 || true)" -if [ -z "$PKG" ]; then - say "known-good is $VERSION and no package for it is in $PKG_CACHE." - say "the cache was cleaned, or that version was never installed from here." +# The version that last worked may be the one that was placed by hand, which is not delivered and has +# no directory. Nothing to choose, and saying so is better than pinning a version that is not there — +# the launcher would ignore the pin and start the newest again, which is the binary that is failing. +if [ ! -x "$VERSIONS/$VERSION/$BINARY" ]; then + say "known-good is $VERSION and no such version is delivered under $VERSIONS." + say "it was retired, or that host was placed by hand and never delivered." say "cannot roll back. this node needs a person." exit 1 fi -say "rolling back to $VERSION ($PKG)" +say "rolling back to $VERSION ($VERSIONS/$VERSION/$BINARY)" printf '%s\n' "$VERSION" > "$ATTEMPTED" -if ! pacman -U --noconfirm "$PKG"; then - say "the package manager refused to install $PKG." - exit 1 -fi +# The pin is what stops the launcher starting the newest again. Written last, so a failure above +# leaves the machine choosing for itself rather than pinned to something this script did not verify. +printf '%s\n' "$VERSION" > "$PINNED" -# Deliberately does NOT start anything. The launcher called this and will exec the host next, -# so starting it here would run two. novox/hq ADR 0005 moved that responsibility; this script -# installs a version and says so, and nothing else. -say "rolled back to $VERSION. the launcher will start it." +# Deliberately does NOT start anything. The launcher called this and will run the host next, so +# starting it here would run two. novox/hq ADR 0005 moved that responsibility; this script chooses a +# version and says so, and nothing else. +say "pinned $VERSION. the launcher will start it." diff --git a/packaging/rollback_test.sh b/packaging/rollback_test.sh index 27e71ba..83190a0 100755 --- a/packaging/rollback_test.sh +++ b/packaging/rollback_test.sh @@ -1,9 +1,15 @@ #!/bin/sh # Tests for nox-mesh-host-rollback. # -# It runs on a machine where the host will not start, which is the one moment nobody can afford -# it to be wrong — and the one moment it is hardest to debug. So it is tested here, against a -# real filesystem, with a stub package manager that records what it was asked to do. +# It runs on a machine where the host will not start, which is the one moment nobody can afford it to +# be wrong — and the one moment it is hardest to debug. So it is tested here, against a real +# filesystem holding real delivered versions. +# +# **These used to stub a package manager.** The script reinstalled the known-good version with +# `pacman -U` out of the package cache, which no machine in this mesh used and which two of the three +# operating systems the host is built for do not have (novox/hq ADR 0141). Going back is now choosing +# a directory, so there is nothing to stub: the thing under test is the filesystem, and a fake would +# only assert that the fake behaves as expected (novox/hq ADR 0017). set -eu cd "$(dirname "$0")" SCRIPT="$PWD/nox-mesh-host-rollback" @@ -12,26 +18,15 @@ PASS=0; FAIL=0 setup() { WORK="$(mktemp -d)" export MESH_HOST_STATE_DIR="$WORK/state" - export MESH_HOST_PKG_CACHE="$WORK/cache" - export MESH_HOST_PACKAGE="nox-mesh-host" - mkdir -p "$MESH_HOST_STATE_DIR" "$MESH_HOST_PKG_CACHE" "$WORK/bin" + export MESH_HOST_LIBEXEC="$WORK/libexec" + mkdir -p "$MESH_HOST_STATE_DIR" "$MESH_HOST_LIBEXEC/versions" +} - # Stubs on PATH. Not mocks of the script's own logic — the boundary is real commands, and - # these record the calls so a test can assert what the script asked the machine to do. - cat > "$WORK/bin/pacman" <<'STUB' -#!/bin/sh -echo "$@" >> "$MESH_HOST_STATE_DIR/pacman.calls" -[ -n "${STUB_PACMAN_FAILS:-}" ] && exit 1 -exit 0 -STUB - cat > "$WORK/bin/systemctl" <<'STUB' -#!/bin/sh -echo "$@" >> "$MESH_HOST_STATE_DIR/systemctl.calls" -exit 0 -STUB - chmod +x "$WORK/bin/pacman" "$WORK/bin/systemctl" - PATH="$WORK/bin:$PATH"; export PATH - unset STUB_PACMAN_FAILS || true +# deliver a version the way the mesh would: a directory named for it, with the binary inside. +deliver() { + mkdir -p "$MESH_HOST_LIBEXEC/versions/$1" + printf '#!/bin/sh\nexit 0\n' > "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host" + chmod +x "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host" } check() { # name, condition-description, actual, expected @@ -41,32 +36,38 @@ check() { # name, condition-description, actual, expected # --- a normal rollback --------------------------------------------------------------------- setup -echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good" -touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst" -"$SCRIPT" >/dev/null 2>&1 -check "installs the known-good version" "pacman is asked to install the cached package" \ - "$(grep -c 'nox-mesh-host-1.4.2' "$MESH_HOST_STATE_DIR/pacman.calls" 2>/dev/null || echo 0)" "1" -# It installs and stops. The launcher execs the host next, and starting it here would run two -# (novox/hq ADR 0005). -check "does not start anything itself" "the launcher owns starting" \ - "$([ -f "$MESH_HOST_STATE_DIR/systemctl.calls" ] && echo started || echo not-started)" "not-started" +deliver 1.4.2 +deliver 1.5.0 +echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good" +RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$? +check "pins the known-good version" "the launcher reads the pin and runs that version instead of the newest" \ + "$(cat "$MESH_HOST_STATE_DIR/rollback-pinned" 2>/dev/null || echo MISSING)" "1.4.2" check "records that it rolled back" "the attempted marker holds the version" \ "$(cat "$MESH_HOST_STATE_DIR/rollback-attempted" 2>/dev/null || echo MISSING)" "1.4.2" +check "succeeds" "a rollback that found its version is not a failure" "$RC" "0" +# It chooses and stops. The launcher runs the host next, and starting it here would run two +# (novox/hq ADR 0005). +check "does not start anything itself" "the launcher owns starting" \ + "$(ls "$MESH_HOST_STATE_DIR" | grep -c started || true)" "0" +# The version it rolled back FROM is left alone: it is the newest, and retiring it is the running +# host's job after a reconcile it completes, never a recovery's. +check "leaves the failing version on disk" "a recovery deletes nothing" \ + "$([ -x "$MESH_HOST_LIBEXEC/versions/1.5.0/nox-mesh-host" ] && echo present || echo gone)" "present" # --- it rolls back only once --------------------------------------------------------------- setup -echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good" -echo "1.4.2" > "$MESH_HOST_STATE_DIR/rollback-attempted" -touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst" -"$SCRIPT" >/dev/null 2>&1 +deliver 1.4.2 +echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good" +echo 1.4.2 > "$MESH_HOST_STATE_DIR/rollback-attempted" +"$SCRIPT" >/dev/null 2>&1 || true check "does not roll back twice" "a second failure is the machine, not the binary" \ - "$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called" + "$([ -e "$MESH_HOST_STATE_DIR/rollback-pinned" ] && echo pinned || echo untouched)" "untouched" # --- nothing to roll back to --------------------------------------------------------------- setup -set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e +RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$? check "no known-good: does nothing" "a host that never reconciled has no version to return to" \ - "$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called" + "$([ -e "$MESH_HOST_STATE_DIR/rollback-attempted" ] && echo attempted || echo untouched)" "untouched" # The exit code is asserted from a real run, not from a literal. An earlier version of this # compared "0" to "0" and could not fail — which hid an injected fault that made the script die # here instead of returning cleanly. @@ -74,23 +75,29 @@ check "no known-good: exits zero" "an installation failure is not a rollback fai setup printf ' \n' > "$MESH_HOST_STATE_DIR/known-good" -"$SCRIPT" >/dev/null 2>&1 -check "blank known-good: refuses to guess" "installing nothing and reporting success is the fault this prevents" \ - "$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called" +"$SCRIPT" >/dev/null 2>&1 || true +check "blank known-good: refuses to guess" "pinning nothing and reporting success is the fault this prevents" \ + "$([ -e "$MESH_HOST_STATE_DIR/rollback-pinned" ] && echo pinned || echo untouched)" "untouched" -# --- the cache was cleaned ------------------------------------------------------------------ +# --- the known-good version is not delivered ------------------------------------------------- +# It was retired, or that host was placed on the machine by hand and never delivered — which is how +# every first host arrives. Pinning it anyway would have the launcher ignore the pin and start the +# newest again, which is the binary that is failing. setup -echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good" -set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e -check "missing package: fails loudly" "cannot roll back, and says so rather than reporting success" "$RC" "1" +deliver 1.5.0 +echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good" +RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$? +check "version not delivered: fails loudly" "cannot roll back, and says so rather than reporting success" "$RC" "1" +check "version not delivered: pins nothing" "a pin the launcher would ignore is worse than none" \ + "$([ -e "$MESH_HOST_STATE_DIR/rollback-pinned" ] && echo pinned || echo untouched)" "untouched" -# --- the package manager refuses ------------------------------------------------------------- +# --- a version directory with no binary in it ------------------------------------------------ +# An interrupted delivery leaves one. Pinning it would start nothing. setup -echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good" -touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst" -STUB_PACMAN_FAILS=1 ; export STUB_PACMAN_FAILS -set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e -check "pacman fails: exits non-zero" "a failed rollback is a failure the launcher must see" "$RC" "1" +mkdir -p "$MESH_HOST_LIBEXEC/versions/1.4.2" +echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good" +RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$? +check "half-delivered version: fails loudly" "a directory is not a version; the binary is" "$RC" "1" printf '\nrollback: %d passed, %d failed\n' "$PASS" "$FAIL" [ "$FAIL" -eq 0 ]