jschoubben
fbf0fb7d63
The supervision was already right: a clean exit means the host stood aside, and the launcher's next turn runs what is on disk. Two things made it dead code — nothing told the running host a successor was waiting, and the rollback resolved its known-good version through pacman, which no machine here uses and which two of three operating systems do not have. Keeping a version rather than a path was the clue. Versions now live in directories named for them: - the launcher picks the newest delivered one every time round the loop, or the one a rollback pinned, or the host placed by hand when nothing is delivered; - the running host stands aside between reconciles, never inside one, by exiting cleanly — and returns nil so the launcher does not count it as a crash; - a completed reconcile retires what is older than the predecessor, keeping the predecessor because that is what a rollback starts, and never the running one; - rollback pins the predecessor instead of reinstalling a package: no package manager, no cache anyone may clean, same script on every operating system; - the report says which host version produced it, so 'behind' is answerable. Newest is when it arrived, never how the name sorts: '1.10' orders before '1.9', and ordering by name would start an older host and call it an upgrade. novox/hq ADR 0141. The delivery half — a module carrying the next host — follows; until then nothing delivers a version and every machine takes the fallback, which is what it does today.
104 lines
5.3 KiB
Bash
Executable File
104 lines
5.3 KiB
Bash
Executable File
#!/bin/sh
|
|
# Tests for nox-mesh-host-rollback.
|
|
#
|
|
# It runs on a machine where the host will not start, which is the one moment nobody can afford it to
|
|
# be wrong — and the one moment it is hardest to debug. So it is tested here, against a real
|
|
# filesystem holding real delivered versions.
|
|
#
|
|
# **These used to stub a package manager.** The script reinstalled the known-good version with
|
|
# `pacman -U` out of the package cache, which no machine in this mesh used and which two of the three
|
|
# operating systems the host is built for do not have (novox/hq ADR 0141). Going back is now choosing
|
|
# a directory, so there is nothing to stub: the thing under test is the filesystem, and a fake would
|
|
# only assert that the fake behaves as expected (novox/hq ADR 0017).
|
|
set -eu
|
|
cd "$(dirname "$0")"
|
|
SCRIPT="$PWD/nox-mesh-host-rollback"
|
|
PASS=0; FAIL=0
|
|
|
|
setup() {
|
|
WORK="$(mktemp -d)"
|
|
export MESH_HOST_STATE_DIR="$WORK/state"
|
|
export MESH_HOST_LIBEXEC="$WORK/libexec"
|
|
mkdir -p "$MESH_HOST_STATE_DIR" "$MESH_HOST_LIBEXEC/versions"
|
|
}
|
|
|
|
# deliver a version the way the mesh would: a directory named for it, with the binary inside.
|
|
deliver() {
|
|
mkdir -p "$MESH_HOST_LIBEXEC/versions/$1"
|
|
printf '#!/bin/sh\nexit 0\n' > "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host"
|
|
chmod +x "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host"
|
|
}
|
|
|
|
check() { # name, condition-description, actual, expected
|
|
if [ "$3" = "$4" ]; then PASS=$((PASS+1)); printf ' ok %s\n' "$1"
|
|
else FAIL=$((FAIL+1)); printf ' FAIL %s\n %s\n got: %s\n expected: %s\n' "$1" "$2" "$3" "$4"; fi
|
|
}
|
|
|
|
# --- a normal rollback ---------------------------------------------------------------------
|
|
setup
|
|
deliver 1.4.2
|
|
deliver 1.5.0
|
|
echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good"
|
|
RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$?
|
|
check "pins the known-good version" "the launcher reads the pin and runs that version instead of the newest" \
|
|
"$(cat "$MESH_HOST_STATE_DIR/rollback-pinned" 2>/dev/null || echo MISSING)" "1.4.2"
|
|
check "records that it rolled back" "the attempted marker holds the version" \
|
|
"$(cat "$MESH_HOST_STATE_DIR/rollback-attempted" 2>/dev/null || echo MISSING)" "1.4.2"
|
|
check "succeeds" "a rollback that found its version is not a failure" "$RC" "0"
|
|
# It chooses and stops. The launcher runs the host next, and starting it here would run two
|
|
# (novox/hq ADR 0005).
|
|
check "does not start anything itself" "the launcher owns starting" \
|
|
"$(ls "$MESH_HOST_STATE_DIR" | grep -c started || true)" "0"
|
|
# The version it rolled back FROM is left alone: it is the newest, and retiring it is the running
|
|
# host's job after a reconcile it completes, never a recovery's.
|
|
check "leaves the failing version on disk" "a recovery deletes nothing" \
|
|
"$([ -x "$MESH_HOST_LIBEXEC/versions/1.5.0/nox-mesh-host" ] && echo present || echo gone)" "present"
|
|
|
|
# --- it rolls back only once ---------------------------------------------------------------
|
|
setup
|
|
deliver 1.4.2
|
|
echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good"
|
|
echo 1.4.2 > "$MESH_HOST_STATE_DIR/rollback-attempted"
|
|
"$SCRIPT" >/dev/null 2>&1 || true
|
|
check "does not roll back twice" "a second failure is the machine, not the binary" \
|
|
"$([ -e "$MESH_HOST_STATE_DIR/rollback-pinned" ] && echo pinned || echo untouched)" "untouched"
|
|
|
|
# --- nothing to roll back to ---------------------------------------------------------------
|
|
setup
|
|
RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$?
|
|
check "no known-good: does nothing" "a host that never reconciled has no version to return to" \
|
|
"$([ -e "$MESH_HOST_STATE_DIR/rollback-attempted" ] && echo attempted || echo untouched)" "untouched"
|
|
# The exit code is asserted from a real run, not from a literal. An earlier version of this
|
|
# compared "0" to "0" and could not fail — which hid an injected fault that made the script die
|
|
# here instead of returning cleanly.
|
|
check "no known-good: exits zero" "an installation failure is not a rollback failure" "$RC" "0"
|
|
|
|
setup
|
|
printf ' \n' > "$MESH_HOST_STATE_DIR/known-good"
|
|
"$SCRIPT" >/dev/null 2>&1 || true
|
|
check "blank known-good: refuses to guess" "pinning nothing and reporting success is the fault this prevents" \
|
|
"$([ -e "$MESH_HOST_STATE_DIR/rollback-pinned" ] && echo pinned || echo untouched)" "untouched"
|
|
|
|
# --- the known-good version is not delivered -------------------------------------------------
|
|
# It was retired, or that host was placed on the machine by hand and never delivered — which is how
|
|
# every first host arrives. Pinning it anyway would have the launcher ignore the pin and start the
|
|
# newest again, which is the binary that is failing.
|
|
setup
|
|
deliver 1.5.0
|
|
echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good"
|
|
RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$?
|
|
check "version not delivered: fails loudly" "cannot roll back, and says so rather than reporting success" "$RC" "1"
|
|
check "version not delivered: pins nothing" "a pin the launcher would ignore is worse than none" \
|
|
"$([ -e "$MESH_HOST_STATE_DIR/rollback-pinned" ] && echo pinned || echo untouched)" "untouched"
|
|
|
|
# --- a version directory with no binary in it ------------------------------------------------
|
|
# An interrupted delivery leaves one. Pinning it would start nothing.
|
|
setup
|
|
mkdir -p "$MESH_HOST_LIBEXEC/versions/1.4.2"
|
|
echo 1.4.2 > "$MESH_HOST_STATE_DIR/known-good"
|
|
RC=0; "$SCRIPT" >/dev/null 2>&1 || RC=$?
|
|
check "half-delivered version: fails loudly" "a directory is not a version; the binary is" "$RC" "1"
|
|
|
|
printf '\nrollback: %d passed, %d failed\n' "$PASS" "$FAIL"
|
|
[ "$FAIL" -eq 0 ]
|