From f4143806c20f44a1966eff32d0dba06dbbdf0083 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 27 Aug 2026 22:24:31 +0200 Subject: [PATCH] Build the rollback mechanism, and test it ADR 0059's recovery path: the pieces that run when the host will not start. internal/upgrade -- two facts, neither of them the host judging its health. Whether the executable this process started from has been replaced on disk, and which version last completed a reconcile. The first design was wrong and the tests caught it, not review. It asked /proc/self/exe whether it was marked deleted. That is Linux procfs behaviour rather than a fact about files, and it catches only unlink -- a binary swapped by rename onto the same path reads as untouched, which is exactly what a package manager does. Now the identity is captured at start and compared later: no procfs, and neither case missed. known-good is one bare line. The reader is a shell script on a machine where the host is failing to start, so it must not need a parser to be present and working. Written only after a clean apply, which is the whole claim -- not health, because a disconnected node is ordinary and a failing resource is the machine's problem rather than the binary's. packaging/ -- the unit, the rollback unit, and the rollback script. The script shares no code with the host and calls none of it: a binary that cannot start cannot be its own recovery. POSIX sh, nothing that has to be installed. The unit carries Restart=always with a comment saying why on-failure would break every upgrade. Both are tested and both sets of tests were confirmed to bite. Injecting five faults broke exactly the intended tests -- except one, and chasing why it did not found a placebo assertion I had written: `check "exits zero" ... "0" "0"` compares a literal to itself and can never fail. Replaced with the real exit code, after which the injection bites. Also caught: an injection that produced a build failure rather than a test failure, which my grep read as "no failure". Re-run so it compiled, and the test did bite. The script test runs in `make check`, so it is a gate rather than something that was run once. Verified against the real binary: known-good is written beside the store after a clean apply and is NOT written after a failed one. --- Makefile | 5 +- cmd/mesh-host/main.go | 16 ++ internal/upgrade/upgrade.go | 138 ++++++++++++++++ internal/upgrade/upgrade_test.go | 193 +++++++++++++++++++++++ packaging/nox-mesh-host-rollback | 66 ++++++++ packaging/nox-mesh-host-rollback.service | 8 + packaging/nox-mesh-host.service | 21 +++ packaging/rollback_test.sh | 98 ++++++++++++ 8 files changed, 544 insertions(+), 1 deletion(-) create mode 100644 internal/upgrade/upgrade.go create mode 100644 internal/upgrade/upgrade_test.go create mode 100755 packaging/nox-mesh-host-rollback create mode 100644 packaging/nox-mesh-host-rollback.service create mode 100644 packaging/nox-mesh-host.service create mode 100755 packaging/rollback_test.sh diff --git a/Makefile b/Makefile index 9432826..a62a6ac 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,10 @@ BUNDLE ?= .PHONY: check test vet fmt build clean host -check: fmt vet test build +check: fmt vet test packaging-test build + +packaging-test: + @./packaging/rollback_test.sh fmt: @test -z "$$(gofmt -l . )" || { echo "unformatted:"; gofmt -l . ; exit 1; } diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 9b0c44f..96b07ee 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -24,6 +24,7 @@ import ( "github.com/novox/mesh-host/internal/inventory" "github.com/novox/mesh-host/internal/profile" "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/upgrade" ) // version is stamped at build time. Unset in a development build, and said so rather than @@ -312,6 +313,21 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, sou return applyErr } + // Only now, and only after a clean apply: this version got as far as a completed + // reconcile, which is the whole of what "known good" claims (novox/hq ADR 0059). Not + // health — a disconnected node is ordinary, and a resource that fails is the machine's + // problem rather than the binary's. + // + // A failure to record is reported and does not fail the apply. The apply worked; what is + // lost is a rollback's ability to come back here, which is worse to hide than to say. + if version != "" { + if err := upgrade.RecordKnownGood(upgrade.KnownGoodPath(opts.state), version); err != nil { + fmt.Fprintf(os.Stderr, + "mesh-host: applied, but could not record %s as known-good: %v\n"+ + " a rollback would have nothing to return to.\n", version, err) + } + } + if opts.json { return writeJSON(report) } diff --git a/internal/upgrade/upgrade.go b/internal/upgrade/upgrade.go new file mode 100644 index 0000000..d728234 --- /dev/null +++ b/internal/upgrade/upgrade.go @@ -0,0 +1,138 @@ +// Package upgrade is how the host survives replacing itself. +// +// novox/hq ADR 0057 and ADR 0059. Two facts, and neither is the host judging its own health: +// +// - whether the executable this process started from has been replaced on disk, which is how +// it knows to stand aside for a new one; +// - which version last got as far as a completed reconcile, which is what a rollback outside +// this binary reads when this binary will not start. +// +// The second is written for a reader that is not the host. A binary that cannot start cannot be +// its own recovery, so what it leaves behind has to be plain enough for a shell script. +package upgrade + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +// KnownGoodName is the file a rollback script reads. Next to the store, because it is node +// state of exactly the same kind. +const KnownGoodName = "known-good" + +// Self is the executable this process started from, remembered. +// +// Identity is taken once, at start, and compared later. The obvious alternative — asking +// /proc/self/exe whether it is marked deleted — was tried and is worse in two ways: it is Linux +// procfs behaviour rather than a fact about files, and it catches only *unlink*, so a binary +// swapped by rename onto the same path reads as untouched. Remembering what we started from +// needs no special filesystem and misses neither case. +type Self struct { + path string + info os.FileInfo +} + +// Current captures the running executable's identity. +// +// path is what os.Executable() returned; a test passes one it can manipulate, because the +// boundary being tested is the filesystem and a fake would assert that the fake behaves as +// expected (novox/hq ADR 0034). +func Current(path string) (Self, error) { + info, err := os.Stat(path) + if err != nil { + return Self{}, fmt.Errorf( + "cannot stat %s, so this host cannot tell whether it is later replaced: %w", path, err) + } + return Self{path: path, info: info}, nil +} + +// Path is where the executable was when this process started. +func (s Self) Path() string { return s.path } + +// Replaced reports whether a different file is at that path now, or none. +// +// Never a silent false: a host that cannot read its own image says so rather than assuming it is +// current, which is the shape of every fault this repository catalogues. +func (s Self) Replaced() (bool, error) { + if s.info == nil { + return false, errors.New("this host never captured its own identity, so it cannot tell " + + "whether it has been replaced") + } + + now, err := os.Stat(s.path) + if errors.Is(err, os.ErrNotExist) { + // Removed rather than upgraded. Still not what is running, and saying "unchanged" + // would leave the host claiming a version that is no longer installed. + return true, nil + } + if err != nil { + return false, err + } + + return !os.SameFile(s.info, now), nil +} + +// KnownGoodPath is where the marker lives, given where the store lives. +func KnownGoodPath(statePath string) string { + return filepath.Join(filepath.Dir(statePath), KnownGoodName) +} + +// RecordKnownGood marks a version as one that started and completed a reconcile. +// +// Written atomically and as one bare line. The reader is a shell script running on a machine +// where the host is failing to start, so the format is the least it can be: no JSON, no +// escaping, nothing that needs a parser to be present and working. +func RecordKnownGood(path, version string) error { + if strings.TrimSpace(version) == "" { + return errors.New("refusing to record an empty version as known-good: a rollback " + + "reading it would install nothing and report success") + } + if strings.ContainsAny(version, "\n\r") { + return fmt.Errorf("refusing to record %q as known-good: it must be one line", version) + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + + tmp, err := os.CreateTemp(filepath.Dir(path), ".known-good-*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + + if _, err := fmt.Fprintln(tmp, version); err != nil { + tmp.Close() + return err + } + if err := tmp.Sync(); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := os.Chmod(tmp.Name(), 0o644); err != nil { + return err + } + return os.Rename(tmp.Name(), path) +} + +// ReadKnownGood returns the recorded version, or "" if there has never been one. +// +// Absence is not an error. A machine whose host has never completed a reconcile has no version +// to go back to, and that is a real state rather than a fault: the node was never working, so +// the failure belongs to the installation and not to an upgrade. A rollback that guessed here +// would become a second fault. +func ReadKnownGood(path string) (string, error) { + raw, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return "", nil + } + if err != nil { + return "", err + } + return strings.TrimSpace(string(raw)), nil +} diff --git a/internal/upgrade/upgrade_test.go b/internal/upgrade/upgrade_test.go new file mode 100644 index 0000000..e2ec715 --- /dev/null +++ b/internal/upgrade/upgrade_test.go @@ -0,0 +1,193 @@ +package upgrade + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// started puts a binary on disk and captures it the way the host does at start. +// +// Against the real filesystem rather than a fake one. What is being tested is how the operating +// system behaves when a file is replaced under a running process, and a fake would assert that +// the fake behaves as expected (novox/hq ADR 0034). +func started(t *testing.T) (Self, string) { + t.Helper() + binary := filepath.Join(t.TempDir(), "mesh-host") + if err := os.WriteFile(binary, []byte("version one"), 0o755); err != nil { + t.Fatal(err) + } + self, err := Current(binary) + if err != nil { + t.Fatal(err) + } + return self, binary +} + +func TestAnUntouchedBinaryIsNotReplaced(t *testing.T) { + // The case that runs every ten minutes forever. A false positive here is a node that exits + // and restarts on every reconcile — a restart loop dressed as an upgrade. + self, _ := started(t) + + replaced, err := self.Replaced() + if err != nil { + t.Fatalf("could not tell: %v", err) + } + if replaced { + t.Error("an untouched binary was reported as replaced; this host would restart forever") + } +} + +func TestRewritingTheSameFileIsNotAReplacement(t *testing.T) { + // Touching content in place keeps the inode, and a package manager does not install this + // way — but something else on the machine might. The claim is about identity, not content. + self, binary := started(t) + + f, err := os.OpenFile(binary, os.O_WRONLY, 0o755) + if err != nil { + t.Fatal(err) + } + if _, err := f.WriteString("same inode, new bytes"); err != nil { + t.Fatal(err) + } + f.Close() + + replaced, err := self.Replaced() + if err != nil { + t.Fatalf("could not tell: %v", err) + } + if replaced { + t.Error("writing through the same inode was reported as a replacement") + } +} + +func TestInstallingOverTheBinaryIsAReplacement(t *testing.T) { + // What a package manager actually does: write a new file and rename it over the old one. + // The running process keeps the old inode; the path now holds a different file. + self, binary := started(t) + + next := binary + ".new" + if err := os.WriteFile(next, []byte("version two"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Rename(next, binary); err != nil { + t.Fatal(err) + } + + replaced, err := self.Replaced() + if err != nil { + t.Fatalf("could not tell: %v", err) + } + if !replaced { + t.Error("a binary replaced by rename was not noticed; this host would keep running the " + + "old version and report the new one") + } +} + +func TestRemovingTheBinaryIsAReplacement(t *testing.T) { + // A package removed rather than upgraded. Nothing is at the path, and the honest answer is + // still "not what I am running" — reporting unchanged would leave the host claiming a + // version that is no longer installed. + self, binary := started(t) + if err := os.Remove(binary); err != nil { + t.Fatal(err) + } + + replaced, err := self.Replaced() + if err != nil { + t.Fatalf("could not tell: %v", err) + } + if !replaced { + t.Error("a removed binary was reported as unchanged") + } +} + +func TestNotBeingAbleToTellIsAnError(t *testing.T) { + // Never a silent false. A host that cannot read its own image must say so rather than + // assume it is current, which is the shape of every fault this repository catalogues. + if _, err := Current(filepath.Join(t.TempDir(), "no-such-binary")); err == nil { + t.Fatal("capturing a nonexistent executable returned an identity instead of an error") + } + // And a Self that was never captured must refuse rather than answer. + if _, err := (Self{}).Replaced(); err == nil { + t.Fatal("an uncaptured Self answered instead of refusing") + } +} + +func TestKnownGoodRoundTrips(t *testing.T) { + path := KnownGoodPath(filepath.Join(t.TempDir(), "state.json")) + + if err := RecordKnownGood(path, "1.4.2"); err != nil { + t.Fatalf("could not record: %v", err) + } + got, err := ReadKnownGood(path) + if err != nil { + t.Fatalf("could not read back: %v", err) + } + if got != "1.4.2" { + t.Errorf("recorded 1.4.2 and read back %q", got) + } +} + +func TestKnownGoodIsOneBareLine(t *testing.T) { + // The reader is a shell script on a machine where the host is failing to start. It must not + // need a JSON parser, and it must not need to strip anything but a newline. + path := KnownGoodPath(filepath.Join(t.TempDir(), "state.json")) + if err := RecordKnownGood(path, "1.4.2"); err != nil { + t.Fatal(err) + } + + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(raw) != "1.4.2\n" { + t.Errorf("known-good is %q; a rollback script reads this with `cat`, so it is one bare "+ + "line and nothing else", string(raw)) + } + if strings.ContainsAny(string(raw), "{}\"") { + t.Error("known-good contains structure; it must be readable without a parser") + } +} + +func TestNeverHavingBeenGoodIsNotAnError(t *testing.T) { + // A machine whose host has never completed a reconcile has nothing to go back to. That is a + // real state — the node was never working — and a rollback must be able to tell it apart + // from a read failure, because guessing a version is how recovery becomes a second fault. + path := KnownGoodPath(filepath.Join(t.TempDir(), "state.json")) + + got, err := ReadKnownGood(path) + if err != nil { + t.Fatalf("absence was reported as a failure: %v", err) + } + if got != "" { + t.Errorf("expected no known-good version, got %q", got) + } +} + +func TestAnEmptyVersionIsRefused(t *testing.T) { + // An empty known-good would make the rollback script install nothing and report success — + // the exact failure the rollback exists to prevent, relocated into the rollback. + path := KnownGoodPath(filepath.Join(t.TempDir(), "state.json")) + if err := RecordKnownGood(path, ""); err == nil { + t.Fatal("an empty version was accepted as known-good") + } +} + +func TestRecordingAgainReplacesRatherThanAppends(t *testing.T) { + path := KnownGoodPath(filepath.Join(t.TempDir(), "state.json")) + for _, v := range []string{"1.4.2", "1.4.3", "1.5.0"} { + if err := RecordKnownGood(path, v); err != nil { + t.Fatal(err) + } + } + + got, err := ReadKnownGood(path) + if err != nil { + t.Fatal(err) + } + if got != "1.5.0" { + t.Errorf("after three recordings the file says %q; it holds the last one, not a history", got) + } +} diff --git a/packaging/nox-mesh-host-rollback b/packaging/nox-mesh-host-rollback new file mode 100755 index 0000000..290c6be --- /dev/null +++ b/packaging/nox-mesh-host-rollback @@ -0,0 +1,66 @@ +#!/bin/sh +# Put the host back on the last version that worked. +# +# novox/hq ADR 0059. This runs when nox-mesh-host will not start, so it shares no code with it +# and calls none of it: a binary that cannot start cannot be its own recovery. POSIX sh, no +# bashisms, nothing that has to be installed. +# +# It is deliberately dull. Everything it does is one of: read a file, run the package manager, +# ask the service manager to try again. +set -eu + +STATE_DIR="${MESH_HOST_STATE_DIR:-/var/lib/mesh-host}" +PKG_CACHE="${MESH_HOST_PKG_CACHE:-/var/cache/pacman/pkg}" +PACKAGE="${MESH_HOST_PACKAGE:-nox-mesh-host}" + +KNOWN_GOOD="$STATE_DIR/known-good" +ATTEMPTED="$STATE_DIR/rollback-attempted" + +say() { echo "nox-mesh-host-rollback: $*" >&2; } + +# Roll back once. A second failure is a different diagnosis: the previously working binary also +# does not run, so the binary is not the problem — the machine is. Rolling back again would flap +# between two versions forever and bury the actual cause under a loop. +if [ -e "$ATTEMPTED" ]; then + say "already rolled back once, to $(cat "$ATTEMPTED" 2>/dev/null || echo unknown)." + say "the previous version also failed to start, so this is the machine and not the binary." + say "not rolling back again. this node needs a person." + exit 0 +fi + +# A machine whose host never completed a reconcile has no version to go back to. That is a real +# state rather than a fault: the node was never working, so the failure belongs to the +# installation. Guessing a version here is how a recovery becomes a second fault. +if [ ! -s "$KNOWN_GOOD" ]; then + say "no known-good version recorded — this host has never completed a reconcile." + say "there is nothing to roll back to. this is an installation failure, not an upgrade one." + exit 0 +fi + +VERSION="$(tr -d '[:space:]' < "$KNOWN_GOOD")" +if [ -z "$VERSION" ]; then + say "known-good is empty. refusing to guess." + exit 0 +fi + +PKG="$(ls "$PKG_CACHE"/"$PACKAGE"-"$VERSION"-*.pkg.tar.* 2>/dev/null | head -n 1 || true)" +if [ -z "$PKG" ]; then + say "known-good is $VERSION and no package for it is in $PKG_CACHE." + say "the cache was cleaned, or that version was never installed from here." + say "cannot roll back. this node needs a person." + exit 1 +fi + +say "rolling back to $VERSION ($PKG)" +printf '%s\n' "$VERSION" > "$ATTEMPTED" + +if ! pacman -U --noconfirm "$PKG"; then + say "the package manager refused to install $PKG." + exit 1 +fi + +# reset-failed first, or the start limit that brought us here is still in force. +systemctl reset-failed "$PACKAGE".service 2>/dev/null || true +systemctl start "$PACKAGE".service + +say "rolled back to $VERSION and started it. the node is on the previous version." diff --git a/packaging/nox-mesh-host-rollback.service b/packaging/nox-mesh-host-rollback.service new file mode 100644 index 0000000..d594792 --- /dev/null +++ b/packaging/nox-mesh-host-rollback.service @@ -0,0 +1,8 @@ +[Unit] +Description=Roll the Novox Mesh node host back to the last version that started +# No OnFailure of its own. If the rollback fails there is nothing further to try +# automatically, and the node needs a person. + +[Service] +Type=oneshot +ExecStart=/usr/lib/nox-mesh-host/rollback diff --git a/packaging/nox-mesh-host.service b/packaging/nox-mesh-host.service new file mode 100644 index 0000000..795a153 --- /dev/null +++ b/packaging/nox-mesh-host.service @@ -0,0 +1,21 @@ +[Unit] +Description=Novox Mesh node host +After=network-online.target +Wants=network-online.target +# When the supervisor gives up, recover rather than leaving the node quiet — a host that will +# not start looks exactly like a machine somebody switched off (novox/hq ADR 0059). +OnFailure=nox-mesh-host-rollback.service + +[Service] +Type=notify +ExecStart=/usr/bin/nox-mesh-host run +# always, NOT on-failure: the host restarts onto a new binary by exiting CLEANLY +# (novox/hq ADR 0057), and on-failure would leave an upgraded node stopped. +Restart=always +RestartSec=5s +StartLimitBurst=3 +StartLimitIntervalSec=120 +StateDirectory=mesh-host + +[Install] +WantedBy=multi-user.target diff --git a/packaging/rollback_test.sh b/packaging/rollback_test.sh new file mode 100755 index 0000000..8eb07ec --- /dev/null +++ b/packaging/rollback_test.sh @@ -0,0 +1,98 @@ +#!/bin/sh +# Tests for nox-mesh-host-rollback. +# +# It runs on a machine where the host will not start, which is the one moment nobody can afford +# it to be wrong — and the one moment it is hardest to debug. So it is tested here, against a +# real filesystem, with a stub package manager that records what it was asked to do. +set -eu +cd "$(dirname "$0")" +SCRIPT="$PWD/nox-mesh-host-rollback" +PASS=0; FAIL=0 + +setup() { + WORK="$(mktemp -d)" + export MESH_HOST_STATE_DIR="$WORK/state" + export MESH_HOST_PKG_CACHE="$WORK/cache" + export MESH_HOST_PACKAGE="nox-mesh-host" + mkdir -p "$MESH_HOST_STATE_DIR" "$MESH_HOST_PKG_CACHE" "$WORK/bin" + + # Stubs on PATH. Not mocks of the script's own logic — the boundary is real commands, and + # these record the calls so a test can assert what the script asked the machine to do. + cat > "$WORK/bin/pacman" <<'STUB' +#!/bin/sh +echo "$@" >> "$MESH_HOST_STATE_DIR/pacman.calls" +[ -n "${STUB_PACMAN_FAILS:-}" ] && exit 1 +exit 0 +STUB + cat > "$WORK/bin/systemctl" <<'STUB' +#!/bin/sh +echo "$@" >> "$MESH_HOST_STATE_DIR/systemctl.calls" +exit 0 +STUB + chmod +x "$WORK/bin/pacman" "$WORK/bin/systemctl" + PATH="$WORK/bin:$PATH"; export PATH + unset STUB_PACMAN_FAILS || true +} + +check() { # name, condition-description, actual, expected + if [ "$3" = "$4" ]; then PASS=$((PASS+1)); printf ' ok %s\n' "$1" + else FAIL=$((FAIL+1)); printf ' FAIL %s\n %s\n got: %s\n expected: %s\n' "$1" "$2" "$3" "$4"; fi +} + +# --- a normal rollback --------------------------------------------------------------------- +setup +echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good" +touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst" +"$SCRIPT" >/dev/null 2>&1 +check "installs the known-good version" "pacman is asked to install the cached package" \ + "$(grep -c 'nox-mesh-host-1.4.2' "$MESH_HOST_STATE_DIR/pacman.calls" 2>/dev/null || echo 0)" "1" +check "resets the start limit before starting" "reset-failed precedes start" \ + "$(head -1 "$MESH_HOST_STATE_DIR/systemctl.calls" | cut -d' ' -f1)" "reset-failed" +check "starts the host again" "systemctl start is called" \ + "$(grep -c '^start ' "$MESH_HOST_STATE_DIR/systemctl.calls" 2>/dev/null || echo 0)" "1" +check "records that it rolled back" "the attempted marker holds the version" \ + "$(cat "$MESH_HOST_STATE_DIR/rollback-attempted" 2>/dev/null || echo MISSING)" "1.4.2" + +# --- it rolls back only once --------------------------------------------------------------- +setup +echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good" +echo "1.4.2" > "$MESH_HOST_STATE_DIR/rollback-attempted" +touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst" +"$SCRIPT" >/dev/null 2>&1 +check "does not roll back twice" "a second failure is the machine, not the binary" \ + "$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called" + +# --- nothing to roll back to --------------------------------------------------------------- +setup +set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e +check "no known-good: does nothing" "a host that never reconciled has no version to return to" \ + "$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called" +# The exit code is asserted from a real run, not from a literal. An earlier version of this +# compared "0" to "0" and could not fail — which hid an injected fault that made the script die +# here instead of returning cleanly. +check "no known-good: exits zero" "an installation failure is not a rollback failure" "$RC" "0" + +setup +printf ' \n' > "$MESH_HOST_STATE_DIR/known-good" +"$SCRIPT" >/dev/null 2>&1 +check "blank known-good: refuses to guess" "installing nothing and reporting success is the fault this prevents" \ + "$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called" + +# --- the cache was cleaned ------------------------------------------------------------------ +setup +echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good" +set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e +check "missing package: fails loudly" "cannot roll back, and says so rather than reporting success" "$RC" "1" + +# --- the package manager refuses ------------------------------------------------------------- +setup +echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good" +touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst" +STUB_PACMAN_FAILS=1 ; export STUB_PACMAN_FAILS +set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e +check "pacman fails: does not start the host" "starting the broken binary again would loop" \ + "$([ -f "$MESH_HOST_STATE_DIR/systemctl.calls" ] && echo started || echo not-started)" "not-started" +check "pacman fails: exits non-zero" "a failed rollback is a failure" "$RC" "1" + +printf '\nrollback: %d passed, %d failed\n' "$PASS" "$FAIL" +[ "$FAIL" -eq 0 ]