jschoubben
057f34f924
ADR 0061. Recovery was the most systemd-specific part of the host, and it is the part that must work on a machine where nothing else does -- which made unit-file syntax a poor place for it, because syntax cannot be tested and the one time it runs is the one time nobody can afford it wrong. So StartLimitBurst and OnFailure move into a launcher script that init starts instead of the host. The unit drops to start-at-boot and restart-on-exit, which OpenRC, runit, s6 and an Android init.rc can all express. Everything 0059 decided is kept: two watchdogs, roll back once, recovery is local, the rollback shares no code with the host. The counter is the whole mechanism, so it is what the tests are mostly about. Three real problems came out of writing them: A counter file holding "1 2" became "12" -- `tr -d [:space:]` concatenates rather than rejecting -- which is past the limit, so a HEALTHY node rolled itself back. Now it reads the first field and insists on a plain integer. The corrupt-counter test used "not-a-number", which shell arithmetic happens to evaluate to 0, so it passed with the guard removed and proved nothing. Replaced with values that discriminate: "5x" errors under set -e and kills the launcher, and "0x10" is read as HEX 16 -- past the limit, so again a healthy node rolls back. And the test harness itself was wrong. With `set -e` and a bare launcher call, removing a guard killed the script at the first corrupt case and silently skipped everything after -- reporting a full pass over tests that never ran. Every launcher call now records its failure instead of aborting. Same class as the placebo assertion found last time, and the reason to keep injecting faults rather than trusting green. Both scripts run in `make check`. 27 launcher tests, 9 rollback tests, all confirmed to bite.
97 lines
4.5 KiB
Bash
Executable File
97 lines
4.5 KiB
Bash
Executable File
#!/bin/sh
|
|
# Tests for nox-mesh-host-rollback.
|
|
#
|
|
# It runs on a machine where the host will not start, which is the one moment nobody can afford
|
|
# it to be wrong — and the one moment it is hardest to debug. So it is tested here, against a
|
|
# real filesystem, with a stub package manager that records what it was asked to do.
|
|
set -eu
|
|
cd "$(dirname "$0")"
|
|
SCRIPT="$PWD/nox-mesh-host-rollback"
|
|
PASS=0; FAIL=0
|
|
|
|
setup() {
|
|
WORK="$(mktemp -d)"
|
|
export MESH_HOST_STATE_DIR="$WORK/state"
|
|
export MESH_HOST_PKG_CACHE="$WORK/cache"
|
|
export MESH_HOST_PACKAGE="nox-mesh-host"
|
|
mkdir -p "$MESH_HOST_STATE_DIR" "$MESH_HOST_PKG_CACHE" "$WORK/bin"
|
|
|
|
# Stubs on PATH. Not mocks of the script's own logic — the boundary is real commands, and
|
|
# these record the calls so a test can assert what the script asked the machine to do.
|
|
cat > "$WORK/bin/pacman" <<'STUB'
|
|
#!/bin/sh
|
|
echo "$@" >> "$MESH_HOST_STATE_DIR/pacman.calls"
|
|
[ -n "${STUB_PACMAN_FAILS:-}" ] && exit 1
|
|
exit 0
|
|
STUB
|
|
cat > "$WORK/bin/systemctl" <<'STUB'
|
|
#!/bin/sh
|
|
echo "$@" >> "$MESH_HOST_STATE_DIR/systemctl.calls"
|
|
exit 0
|
|
STUB
|
|
chmod +x "$WORK/bin/pacman" "$WORK/bin/systemctl"
|
|
PATH="$WORK/bin:$PATH"; export PATH
|
|
unset STUB_PACMAN_FAILS || true
|
|
}
|
|
|
|
check() { # name, condition-description, actual, expected
|
|
if [ "$3" = "$4" ]; then PASS=$((PASS+1)); printf ' ok %s\n' "$1"
|
|
else FAIL=$((FAIL+1)); printf ' FAIL %s\n %s\n got: %s\n expected: %s\n' "$1" "$2" "$3" "$4"; fi
|
|
}
|
|
|
|
# --- a normal rollback ---------------------------------------------------------------------
|
|
setup
|
|
echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
|
|
touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst"
|
|
"$SCRIPT" >/dev/null 2>&1
|
|
check "installs the known-good version" "pacman is asked to install the cached package" \
|
|
"$(grep -c 'nox-mesh-host-1.4.2' "$MESH_HOST_STATE_DIR/pacman.calls" 2>/dev/null || echo 0)" "1"
|
|
# It installs and stops. The launcher execs the host next, and starting it here would run two
|
|
# (novox/hq ADR 0061).
|
|
check "does not start anything itself" "the launcher owns starting" \
|
|
"$([ -f "$MESH_HOST_STATE_DIR/systemctl.calls" ] && echo started || echo not-started)" "not-started"
|
|
check "records that it rolled back" "the attempted marker holds the version" \
|
|
"$(cat "$MESH_HOST_STATE_DIR/rollback-attempted" 2>/dev/null || echo MISSING)" "1.4.2"
|
|
|
|
# --- it rolls back only once ---------------------------------------------------------------
|
|
setup
|
|
echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
|
|
echo "1.4.2" > "$MESH_HOST_STATE_DIR/rollback-attempted"
|
|
touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst"
|
|
"$SCRIPT" >/dev/null 2>&1
|
|
check "does not roll back twice" "a second failure is the machine, not the binary" \
|
|
"$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called"
|
|
|
|
# --- nothing to roll back to ---------------------------------------------------------------
|
|
setup
|
|
set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e
|
|
check "no known-good: does nothing" "a host that never reconciled has no version to return to" \
|
|
"$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called"
|
|
# The exit code is asserted from a real run, not from a literal. An earlier version of this
|
|
# compared "0" to "0" and could not fail — which hid an injected fault that made the script die
|
|
# here instead of returning cleanly.
|
|
check "no known-good: exits zero" "an installation failure is not a rollback failure" "$RC" "0"
|
|
|
|
setup
|
|
printf ' \n' > "$MESH_HOST_STATE_DIR/known-good"
|
|
"$SCRIPT" >/dev/null 2>&1
|
|
check "blank known-good: refuses to guess" "installing nothing and reporting success is the fault this prevents" \
|
|
"$([ -f "$MESH_HOST_STATE_DIR/pacman.calls" ] && echo called || echo not-called)" "not-called"
|
|
|
|
# --- the cache was cleaned ------------------------------------------------------------------
|
|
setup
|
|
echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
|
|
set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e
|
|
check "missing package: fails loudly" "cannot roll back, and says so rather than reporting success" "$RC" "1"
|
|
|
|
# --- the package manager refuses -------------------------------------------------------------
|
|
setup
|
|
echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
|
|
touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst"
|
|
STUB_PACMAN_FAILS=1 ; export STUB_PACMAN_FAILS
|
|
set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e
|
|
check "pacman fails: exits non-zero" "a failed rollback is a failure the launcher must see" "$RC" "1"
|
|
|
|
printf '\nrollback: %d passed, %d failed\n' "$PASS" "$FAIL"
|
|
[ "$FAIL" -eq 0 ]
|