jschoubben
f4143806c2
ADR 0059's recovery path: the pieces that run when the host will not start. internal/upgrade -- two facts, neither of them the host judging its health. Whether the executable this process started from has been replaced on disk, and which version last completed a reconcile. The first design was wrong and the tests caught it, not review. It asked /proc/self/exe whether it was marked deleted. That is Linux procfs behaviour rather than a fact about files, and it catches only unlink -- a binary swapped by rename onto the same path reads as untouched, which is exactly what a package manager does. Now the identity is captured at start and compared later: no procfs, and neither case missed. known-good is one bare line. The reader is a shell script on a machine where the host is failing to start, so it must not need a parser to be present and working. Written only after a clean apply, which is the whole claim -- not health, because a disconnected node is ordinary and a failing resource is the machine's problem rather than the binary's. packaging/ -- the unit, the rollback unit, and the rollback script. The script shares no code with the host and calls none of it: a binary that cannot start cannot be its own recovery. POSIX sh, nothing that has to be installed. The unit carries Restart=always with a comment saying why on-failure would break every upgrade. Both are tested and both sets of tests were confirmed to bite. Injecting five faults broke exactly the intended tests -- except one, and chasing why it did not found a placebo assertion I had written: `check "exits zero" ... "0" "0"` compares a literal to itself and can never fail. Replaced with the real exit code, after which the injection bites. Also caught: an injection that produced a build failure rather than a test failure, which my grep read as "no failure". Re-run so it compiled, and the test did bite. The script test runs in `make check`, so it is a gate rather than something that was run once. Verified against the real binary: known-good is written beside the store after a clean apply and is NOT written after a failed one.
99 lines
4.7 KiB
Bash
Executable File
99 lines
4.7 KiB
Bash
Executable File
#!/bin/sh
|
|
# Tests for nox-mesh-host-rollback.
|
|
#
|
|
# It runs on a machine where the host will not start, which is the one moment nobody can afford
|
|
# it to be wrong — and the one moment it is hardest to debug. So it is tested here, against a
|
|
# real filesystem, with a stub package manager that records what it was asked to do.
|
|
set -eu
|
|
cd "$(dirname "$0")"
|
|
SCRIPT="$PWD/nox-mesh-host-rollback"
|
|
PASS=0; FAIL=0
|
|
|
|
setup() {
|
|
WORK="$(mktemp -d)"
|
|
export MESH_HOST_STATE_DIR="$WORK/state"
|
|
export MESH_HOST_PKG_CACHE="$WORK/cache"
|
|
export MESH_HOST_PACKAGE="nox-mesh-host"
|
|
mkdir -p "$MESH_HOST_STATE_DIR" "$MESH_HOST_PKG_CACHE" "$WORK/bin"
|
|
|
|
# Stubs on PATH. Not mocks of the script's own logic — the boundary is real commands, and
|
|
# these record the calls so a test can assert what the script asked the machine to do.
|
|
cat > "$WORK/bin/pacman" <<'STUB'
|
|
#!/bin/sh
|
|
echo "$@" >> "$MESH_HOST_STATE_DIR/pacman.calls"
|
|
[ -n "${STUB_PACMAN_FAILS:-}" ] && exit 1
|
|
exit 0
|
|
STUB
|
|
cat > "$WORK/bin/systemctl" <<'STUB'
|
|
#!/bin/sh
|
|
echo "$@" >> "$MESH_HOST_STATE_DIR/systemctl.calls"
|
|
exit 0
|
|
STUB
|
|
chmod +x "$WORK/bin/pacman" "$WORK/bin/systemctl"
|
|
PATH="$WORK/bin:$PATH"; export PATH
|
|
unset STUB_PACMAN_FAILS || true
|
|
}
|
|
|
|
check() { # name, condition-description, actual, expected
|
|
if [ "$3" = "$4" ]; then PASS=$((PASS+1)); printf ' ok %s\n' "$1"
|
|
else FAIL=$((FAIL+1)); printf ' FAIL %s\n %s\n got: %s\n expected: %s\n' "$1" "$2" "$3" "$4"; fi
|
|
}
|
|
|
|
# --- a normal rollback ---------------------------------------------------------------------
|
|
setup
|
|
echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
|
|
touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst"
|
|
"$SCRIPT" >/dev/null 2>&1
|
|
check "installs the known-good version" "pacman is asked to install the cached package" \
|
|
"$(grep -c 'nox-mesh-host-1.4.2' "$MESH_HOST_STATE_DIR/pacman.calls" 2>/dev/null || echo 0)" "1"
|
|
check "resets the start limit before starting" "reset-failed precedes start" \
|
|
"$(head -1 "$MESH_HOST_STATE_DIR/systemctl.calls" | cut -d' ' -f1)" "reset-failed"
|
|
check "starts the host again" "systemctl start is called" \
|
|
"$(grep -c '^start ' "$MESH_HOST_STATE_DIR/systemctl.calls" 2>/dev/null || echo 0)" "1"
|
|
check "records that it rolled back" "the attempted marker holds the version" \
|
|
"$(cat "$MESH_HOST_STATE_DIR/rollback-attempted" 2>/dev/null || echo MISSING)" "1.4.2"
|
|
|
|
# --- it rolls back only once ---------------------------------------------------------------
|
|
setup
|
|
echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
|
|
echo "1.4.2" > "$MESH_HOST_STATE_DIR/rollback-attempted"
|
|
touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst"
|
|
"$SCRIPT" >/dev/null 2>&1
|
|
check "does not roll back twice" "a second failure is the machine, not the binary" \
|
|
"$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called"
|
|
|
|
# --- nothing to roll back to ---------------------------------------------------------------
|
|
setup
|
|
set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e
|
|
check "no known-good: does nothing" "a host that never reconciled has no version to return to" \
|
|
"$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called"
|
|
# The exit code is asserted from a real run, not from a literal. An earlier version of this
|
|
# compared "0" to "0" and could not fail — which hid an injected fault that made the script die
|
|
# here instead of returning cleanly.
|
|
check "no known-good: exits zero" "an installation failure is not a rollback failure" "$RC" "0"
|
|
|
|
setup
|
|
printf ' \n' > "$MESH_HOST_STATE_DIR/known-good"
|
|
"$SCRIPT" >/dev/null 2>&1
|
|
check "blank known-good: refuses to guess" "installing nothing and reporting success is the fault this prevents" \
|
|
"$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called"
|
|
|
|
# --- the cache was cleaned ------------------------------------------------------------------
|
|
setup
|
|
echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
|
|
set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e
|
|
check "missing package: fails loudly" "cannot roll back, and says so rather than reporting success" "$RC" "1"
|
|
|
|
# --- the package manager refuses -------------------------------------------------------------
|
|
setup
|
|
echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
|
|
touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst"
|
|
STUB_PACMAN_FAILS=1 ; export STUB_PACMAN_FAILS
|
|
set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e
|
|
check "pacman fails: does not start the host" "starting the broken binary again would loop" \
|
|
"$([ -f "$MESH_HOST_STATE_DIR/systemctl.calls" ] && echo started || echo not-started)" "not-started"
|
|
check "pacman fails: exits non-zero" "a failed rollback is a failure" "$RC" "1"
|
|
|
|
printf '\nrollback: %d passed, %d failed\n' "$PASS" "$FAIL"
|
|
[ "$FAIL" -eq 0 ]
|