jochen
0c405b70cc
The launcher trusted a counter only a by-hand reconcile ever cleared and a known-good nothing in the daemon wrote, so no machine could roll its host back; the controller and the node tools were replaced in place with nothing kept. - The launcher runs a delivered host that is not known-good on trial: one that crashes, stops for nothing, or does not report within ten minutes goes back to known-good, once per version, recorded in rolled-back. The host proves itself when the mesh takes a report under its own build, says every standing verdict on its reports, never stands aside for a rolled-back version, and restarts its service once when its launcher was replaced on disk. - The engine keeps the controller's and the node tools' previous build beside the new one and judges the new one: the lease taken by the controller it started (read-only direct get of mesh-controller_lease/holder), or this machine's runtime answering $SRV.PING.node-tools.<node>, within sixty seconds of time it could ask. Not healthy: the previous restored, once, said. Proved: the previous deleted. A build declared not-reversible is never rolled back. - Retire never removes a version newer than the running one.
346 lines
17 KiB
Bash
Executable File
346 lines
17 KiB
Bash
Executable File
#!/bin/sh
|
|
# Tests for nox-mesh-host-launch.
|
|
#
|
|
# The counter is the whole mechanism and it is the part to get wrong: never cleared and a node
|
|
# rolls back on a healthy boot; cleared too eagerly and it never rolls back at all. So the
|
|
# counter is what most of these assert.
|
|
set -eu
|
|
cd "$(dirname "$0")"
|
|
LAUNCH="$PWD/nox-mesh-host-launch"
|
|
PASS=0; FAIL=0
|
|
|
|
setup() {
|
|
WORK="$(mktemp -d)"
|
|
export MESH_HOST_STATE_DIR="$WORK/state"
|
|
export MESH_HOST_LIBEXEC="$WORK/libexec"
|
|
export MESH_HOST_BIN="$WORK/bin/nox-mesh-host"
|
|
export MESH_HOST_START_LIMIT=3
|
|
export MESH_HOST_BACKOFF=0
|
|
export MESH_HOST_RUN_ONCE=1
|
|
mkdir -p "$MESH_HOST_STATE_DIR" "$MESH_HOST_LIBEXEC" "$WORK/bin"
|
|
|
|
# A host that records being started. It exits immediately, which is what the launcher's
|
|
# exec makes indistinguishable from a host that ran for a week — the launcher is gone by
|
|
# then either way.
|
|
cat > "$MESH_HOST_BIN" <<'STUB'
|
|
#!/bin/sh
|
|
echo "$@" >> "$MESH_HOST_STATE_DIR/host.starts"
|
|
exit "${STUB_HOST_EXIT:-1}"
|
|
STUB
|
|
chmod +x "$MESH_HOST_BIN"
|
|
export MESH_HOST_TRIAL_BOUND=600 MESH_HOST_TRIAL_TICK=1 MESH_HOST_STOP_GRACE=1
|
|
}
|
|
|
|
# `|| true` on every launcher call above: a launcher that exits non-zero is something to
|
|
# ASSERT, not something to abort on. With `set -e` and a bare call, removing a guard from the
|
|
# launcher killed this script at the first corrupt-counter case and silently skipped the rest —
|
|
# reporting a full pass over tests that never ran.
|
|
check() { if [ "$3" = "$4" ]; then PASS=$((PASS+1)); printf ' ok %s\n' "$1"
|
|
else FAIL=$((FAIL+1)); printf ' FAIL %s\n %s\n got: %s\n expected: %s\n' "$1" "$2" "$3" "$4"; fi; }
|
|
|
|
count() { cat "$MESH_HOST_STATE_DIR/start-attempts" 2>/dev/null || echo MISSING; }
|
|
started() { [ -f "$MESH_HOST_STATE_DIR/host.starts" ] && echo yes || echo no; }
|
|
rolled() { [ -s "$MESH_HOST_STATE_DIR/rolled-back" ] && echo yes || echo no; }
|
|
|
|
# --- the ordinary start -----------------------------------------------------------------------
|
|
setup
|
|
"$LAUNCH" >/dev/null 2>&1 || true
|
|
check "starts the host" "the common case, every boot" "$(started)" "yes"
|
|
check "counts the attempt" "the counter is what decides a rollback later" "$(count)" "1"
|
|
check "does not roll back" "a first start is not a failure" "$(rolled)" "no"
|
|
|
|
# --- failures below the limit -----------------------------------------------------------------
|
|
setup
|
|
i=1; while [ $i -le 3 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done
|
|
check "three starts do not trigger a rollback" "the limit is exceeded, not reached" "$(rolled)" "no"
|
|
check "counts them all" "" "$(count)" "3"
|
|
|
|
# --- the host clears the counter on success ----------------------------------------------------
|
|
setup
|
|
i=1; while [ $i -le 2 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done
|
|
printf '0\n' > "$MESH_HOST_STATE_DIR/start-attempts" # what the host does on a completed reconcile
|
|
i=1; while [ $i -le 3 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done
|
|
check "a cleared counter prevents a rollback" "a node up for months must not roll back on a healthy boot" \
|
|
"$(rolled)" "no"
|
|
|
|
# --- halted stays halted --------------------------------------------------------------------------
|
|
setup
|
|
echo "rolled back and still failing" > "$MESH_HOST_STATE_DIR/halted"
|
|
"$LAUNCH" >/dev/null 2>&1 || true
|
|
check "a halted node does not start the host" "nothing further is tried automatically" "$(started)" "no"
|
|
set +e; "$LAUNCH" >/dev/null 2>&1; RC=$?; set -e
|
|
check "a halted node exits zero" "a supervisor loop that is slow and visible beats a crash loop" "$RC" "0"
|
|
|
|
# --- a corrupt counter ------------------------------------------------------------------------------
|
|
#
|
|
# The values here are chosen because they DISCRIMINATE. An earlier version used
|
|
# "not-a-number", which shell arithmetic happens to evaluate to 0 — so the test passed with the
|
|
# guard removed and proved nothing. These two do not:
|
|
#
|
|
# 5x shell arithmetic errors, and under `set -e` the launcher dies without starting the host
|
|
# 0x10 is read as HEX 16 — past the limit, so a healthy node would roll back for no reason
|
|
for corrupt in "5x" "0x10" "1 2" ""; do
|
|
setup
|
|
echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
|
|
printf '%s\n' "$corrupt" > "$MESH_HOST_STATE_DIR/start-attempts"
|
|
"$LAUNCH" >/dev/null 2>&1 || true
|
|
# The exact number is not the property — "1 2" legitimately recovers a leading 1, while
|
|
# "5x" is rejected to 0. What must hold for every one of them is that the launcher
|
|
# survives its own state and does not read it as "past the limit".
|
|
check "corrupt counter [$corrupt]: starts the host" "the launcher must not die on its own state" \
|
|
"$(started)" "yes"
|
|
check "corrupt counter [$corrupt]: does not roll back" "a healthy node must not roll back on a bad counter" \
|
|
"$(rolled)" "no"
|
|
check "corrupt counter [$corrupt]: counter is a sane integer" "it is written back for the next start to read" \
|
|
"$(count | grep -cE '^[0-9]+$')" "1"
|
|
done
|
|
|
|
# --- the loop, and shutting down ------------------------------------------------------------
|
|
#
|
|
# These need the launcher to actually run as a supervisor rather than one iteration, so they do
|
|
# not set MESH_HOST_RUN_ONCE.
|
|
|
|
# A host that exits 0 has upgraded itself and stood aside (novox/hq ADR 0005). The launcher must
|
|
# start it again — and must NOT count it, because it did not fail.
|
|
setup
|
|
unset MESH_HOST_RUN_ONCE
|
|
cat > "$MESH_HOST_BIN" <<'STUB'
|
|
#!/bin/sh
|
|
echo start >> "$MESH_HOST_STATE_DIR/host.starts"
|
|
# Exit 0 three times, then hang so the launcher stops looping and can be killed.
|
|
if [ "$(wc -l < "$MESH_HOST_STATE_DIR/host.starts")" -lt 3 ]; then exit 0; fi
|
|
sleep 30
|
|
STUB
|
|
chmod +x "$MESH_HOST_BIN"
|
|
"$LAUNCH" >/dev/null 2>&1 &
|
|
LP=$!
|
|
sleep 1
|
|
check "a clean exit restarts the host" "that is how it stands aside for a new binary" \
|
|
"$([ "$(wc -l < "$MESH_HOST_STATE_DIR/host.starts" 2>/dev/null || echo 0)" -ge 3 ] && echo looped || echo stopped)" "looped"
|
|
# No counter file at all: nothing has failed, so nothing has been counted.
|
|
check "a clean exit is not counted as a failure" "it finished, it did not fail" "$(count)" "MISSING"
|
|
|
|
# Shutting down: the signal must reach the host, and the launcher must wait for it rather than
|
|
# exiting and leaving the host to be killed mid-apply.
|
|
kill -TERM "$LP" 2>/dev/null
|
|
sleep 1
|
|
check "SIGTERM stops the launcher" "a supervisor that ignores shutdown hangs the machine" \
|
|
"$(kill -0 "$LP" 2>/dev/null && echo running || echo stopped)" "stopped"
|
|
check "and does not leave the host running" "the child must go down with it" \
|
|
"$(pgrep -f "$MESH_HOST_BIN" >/dev/null 2>&1 && echo orphaned || echo reaped)" "reaped"
|
|
|
|
# A crash IS counted, and the launcher keeps going.
|
|
setup
|
|
unset MESH_HOST_RUN_ONCE
|
|
export MESH_HOST_BACKOFF=0
|
|
cat > "$MESH_HOST_BIN" <<'STUB'
|
|
#!/bin/sh
|
|
echo start >> "$MESH_HOST_STATE_DIR/host.starts"
|
|
if [ "$(wc -l < "$MESH_HOST_STATE_DIR/host.starts")" -lt 2 ]; then exit 3; fi
|
|
sleep 30
|
|
STUB
|
|
chmod +x "$MESH_HOST_BIN"
|
|
"$LAUNCH" >/dev/null 2>&1 &
|
|
LP=$!
|
|
sleep 1
|
|
check "a crash is counted" "unlike a clean exit, which is not" "$(count)" "1"
|
|
kill -TERM "$LP" 2>/dev/null; sleep 1; pkill -f "$MESH_HOST_BIN" 2>/dev/null || true
|
|
|
|
# --- which version it runs (novox/hq ADR 0141) ------------------------------------------------
|
|
#
|
|
# Versions live side by side in directories named for them. The launcher picks one every time round
|
|
# the loop, never once: standing aside for a successor is a clean exit, and the next turn has to run
|
|
# what is on disk NOW — resolved once, the same binary would restart for ever and no upgrade would
|
|
# ever take.
|
|
|
|
# deliver a version as the mesh would, recording which one ran so a test can assert the choice.
|
|
deliver() {
|
|
mkdir -p "$MESH_HOST_LIBEXEC/versions/$1"
|
|
cat > "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host" <<STUB
|
|
#!/bin/sh
|
|
echo "$1" >> "\$MESH_HOST_STATE_DIR/which.ran"
|
|
${3:-}
|
|
exit "\${STUB_HOST_EXIT:-1}"
|
|
STUB
|
|
chmod +x "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host"
|
|
# When it arrived is what "newest" means, so it is set rather than left to the clock.
|
|
touch -d "$2" "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host" "$MESH_HOST_LIBEXEC/versions/$1"
|
|
}
|
|
which_ran() { cat "$MESH_HOST_STATE_DIR/which.ran" 2>/dev/null || echo NONE; }
|
|
|
|
# Newest is when it arrived, not how its name sorts: "1.10" orders before "1.9" by name, so ordering
|
|
# by name would run an older host and call it an upgrade.
|
|
setup
|
|
deliver 1.10 "2 hours ago"
|
|
deliver 1.9 "1 hour ago"
|
|
"$LAUNCH" >/dev/null 2>&1 || true
|
|
check "runs the newest delivered version" "newest is when it arrived, not how the name sorts" \
|
|
"$(which_ran)" "1.9"
|
|
|
|
# A pin from a rollback beats the newest, or the launcher would start the failing binary again and
|
|
# the rollback would flap.
|
|
setup
|
|
deliver 1.9 "2 hours ago"
|
|
deliver 2.0 "1 hour ago"
|
|
echo 1.9 > "$MESH_HOST_STATE_DIR/rollback-pinned"
|
|
"$LAUNCH" >/dev/null 2>&1 || true
|
|
check "a pinned version beats the newest" "otherwise a rollback starts the binary it just rejected" \
|
|
"$(which_ran)" "1.9"
|
|
|
|
# A pin naming a version that is not there is ignored rather than fatal: the machine choosing for
|
|
# itself is better than a machine that starts nothing.
|
|
setup
|
|
deliver 2.0 "1 hour ago"
|
|
echo 1.9 > "$MESH_HOST_STATE_DIR/rollback-pinned"
|
|
"$LAUNCH" >/dev/null 2>&1 || true
|
|
check "an undeliverable pin is ignored" "a machine that starts nothing is worse than one that chooses" \
|
|
"$(which_ran)" "2.0"
|
|
|
|
# An interrupted delivery leaves a directory with no binary in it. Treating it as the newest would
|
|
# mean running nothing.
|
|
setup
|
|
deliver 1.9 "2 hours ago"
|
|
mkdir -p "$MESH_HOST_LIBEXEC/versions/2.0-half"
|
|
touch -d "1 minute ago" "$MESH_HOST_LIBEXEC/versions/2.0-half"
|
|
"$LAUNCH" >/dev/null 2>&1 || true
|
|
check "skips a version with no binary" "a directory is not a version; the binary is" \
|
|
"$(which_ran)" "1.9"
|
|
|
|
# Nothing delivered: the host placed by hand, which is how the first one always arrives. Without this
|
|
# the change would strand every machine in the mesh on the day it ships.
|
|
setup
|
|
"$LAUNCH" >/dev/null 2>&1 || true
|
|
check "falls back to the host placed by hand" "every first host arrives this way" "$(started)" "yes"
|
|
|
|
# --- the launcher witnesses the host's successor (novox/hq to-be 45 §8) --------------------------
|
|
#
|
|
# A delivered host that is not the known-good one runs on trial: it must write itself into known-good
|
|
# (which the host does when the mesh has taken a report it made under its own build) within the bound.
|
|
# One that crashes, stops for nothing, or never reports goes back to known-good, once, recorded.
|
|
|
|
ran_count() { grep -xF "$1" "$MESH_HOST_STATE_DIR/which.ran" 2>/dev/null | wc -l | tr -d ' '; }
|
|
last_ran() { tail -n 1 "$MESH_HOST_STATE_DIR/which.ran" 2>/dev/null || echo NONE; }
|
|
record_of() { cut -f"$2" "$MESH_HOST_STATE_DIR/rolled-back" 2>/dev/null | sed -n "${1}p"; }
|
|
records() { grep . "$MESH_HOST_STATE_DIR/rolled-back" 2>/dev/null | wc -l | tr -d ' '; }
|
|
|
|
# A new host that crashes at once: three tries, then the known-good one, recorded once.
|
|
setup
|
|
deliver 1.0 "2 hours ago"
|
|
deliver 2.0 "1 hour ago"
|
|
echo 1.0 > "$MESH_HOST_STATE_DIR/known-good"
|
|
i=1; while [ $i -le 4 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done
|
|
check "a crashing new host is tried three times" "the limit is the evidence" "$(ran_count 2.0)" "3"
|
|
check "then the known-good one runs" "the witness restores the build before" "$(last_ran)" "1.0"
|
|
check "the rollback is recorded once" "one line per verdict" "$(records)" "1"
|
|
check "naming what failed" "from" "$(record_of 1 1)" "2.0"
|
|
check "and what runs instead" "to" "$(record_of 1 2)" "1.0"
|
|
check "as a rollback" "outcome" "$(record_of 1 4)" "rolled-back"
|
|
check "saying why" "why" "$(record_of 1 5 | grep -c 'failed 3 times')" "1"
|
|
check "the counter starts again for the version gone back to" "or it halts at once" \
|
|
"$([ "$(count)" -lt 3 ] && echo below-limit || echo "at-limit($(count))")" "below-limit"
|
|
# Not retried: the rolled-back version is still the newest delivered, and is never started again.
|
|
i=1; while [ $i -le 2 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done
|
|
check "a rolled-back version is never started again" "one rollback per version" "$(ran_count 2.0)" "3"
|
|
check "and it is not recorded twice" "" "$(records)" "1"
|
|
# What it went back to failing too is the machine, not a binary: halted, and said.
|
|
"$LAUNCH" >/dev/null 2>&1 || true
|
|
check "the version gone back to failing too halts" "rolling back again would flap" \
|
|
"$([ -f "$MESH_HOST_STATE_DIR/halted" ] && echo halted || echo running)" "halted"
|
|
check "and the halt is recorded" "" "$(record_of 2 4)" "halted"
|
|
check "with no rollback to a third version" "" "$(ran_count 2.0)" "3"
|
|
|
|
# A new host that exits cleanly at once, standing aside for nothing: counted, then rolled back.
|
|
setup
|
|
deliver 1.0 "2 hours ago"
|
|
deliver 2.0 "1 hour ago" "exit 0"
|
|
echo 1.0 > "$MESH_HOST_STATE_DIR/known-good"
|
|
i=1; while [ $i -le 4 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done
|
|
check "a new host that stops for nothing is rolled back" "a clean exit with nothing newer is not standing aside" \
|
|
"$(record_of 1 1) -> $(record_of 1 2)" "2.0 -> 1.0"
|
|
check "after three tries" "" "$(ran_count 2.0)" "3"
|
|
|
|
# A new host that runs and never reports: stopped at the bound, rolled back.
|
|
setup
|
|
export MESH_HOST_TRIAL_BOUND=2
|
|
deliver 1.0 "2 hours ago"
|
|
deliver 2.0 "1 hour ago" "sleep 30"
|
|
echo 1.0 > "$MESH_HOST_STATE_DIR/known-good"
|
|
"$LAUNCH" >/dev/null 2>&1 || true
|
|
check "a host that never reports is rolled back at the bound" "started and did nothing" \
|
|
"$(record_of 1 1) -> $(record_of 1 2)" "2.0 -> 1.0"
|
|
check "saying it did not report" "" "$(record_of 1 5 | grep -c 'did not report within 2s')" "1"
|
|
check "and the known-good one runs" "" "$(last_ran)" "1.0"
|
|
check "the silent host is not left running" "the witness stops it" \
|
|
"$(pgrep -f "$MESH_HOST_LIBEXEC/versions/2.0" >/dev/null 2>&1 && echo running || echo stopped)" "stopped"
|
|
|
|
# A new host that reports in bound is proved: no rollback, and the trial ends.
|
|
setup
|
|
export MESH_HOST_TRIAL_BOUND=3
|
|
deliver 1.0 "2 hours ago"
|
|
deliver 2.0 "1 hour ago" "echo 2.0 > \"\$MESH_HOST_STATE_DIR/known-good\"; sleep 4"
|
|
echo 1.0 > "$MESH_HOST_STATE_DIR/known-good"
|
|
"$LAUNCH" >/dev/null 2>&1 || true
|
|
check "a host that reports in bound is not rolled back" "a healthy update undoes nothing" "$(rolled)" "no"
|
|
check "it ran past its bound" "the witness let it be once it reported" "$(ran_count 2.0)" "1"
|
|
"$LAUNCH" >/dev/null 2>&1 || true
|
|
check "and is the one run after" "known-good now" "$(last_ran)" "2.0"
|
|
check "and no longer on trial" "the trial file goes" \
|
|
"$([ -e "$MESH_HOST_STATE_DIR/trial" ] && echo on-trial || echo proved)" "proved"
|
|
|
|
# A launcher restarted after the bound passed rolls back without starting the host again.
|
|
setup
|
|
deliver 1.0 "2 hours ago"
|
|
deliver 2.0 "1 hour ago"
|
|
echo 1.0 > "$MESH_HOST_STATE_DIR/known-good"
|
|
printf '2.0 %s\n' "$(( $(date +%s) - 4000 ))" > "$MESH_HOST_STATE_DIR/trial"
|
|
"$LAUNCH" >/dev/null 2>&1 || true
|
|
check "a trial past its bound is ended at the next start" "a host that keeps restarting cannot outrun its bound" \
|
|
"$(ran_count 2.0) $(record_of 1 1)" "0 2.0"
|
|
|
|
# A newer host delivered after a rollback runs, on trial; the one rolled back stays refused.
|
|
setup
|
|
deliver 1.0 "3 hours ago"
|
|
deliver 2.0 "2 hours ago"
|
|
echo 1.0 > "$MESH_HOST_STATE_DIR/known-good"
|
|
printf '2.0\t1.0\t%s\trolled-back\tit failed 3 times in a row\n' "$(date +%s)" > "$MESH_HOST_STATE_DIR/rolled-back"
|
|
echo 1.0 > "$MESH_HOST_STATE_DIR/rollback-pinned"
|
|
touch -d "1 hour ago" "$MESH_HOST_STATE_DIR/rollback-pinned"
|
|
deliver 3.0 "1 minute ago"
|
|
"$LAUNCH" >/dev/null 2>&1 || true
|
|
check "a newer delivery after a rollback runs" "a rolled-back version blocks only itself" "$(last_ran)" "3.0"
|
|
check "and the pin goes" "" "$([ -e "$MESH_HOST_STATE_DIR/rollback-pinned" ] && echo pinned || echo free)" "free"
|
|
check "on trial" "" "$(cut -d' ' -f1 "$MESH_HOST_STATE_DIR/trial" 2>/dev/null)" "3.0"
|
|
|
|
# No known-good delivered: nothing to go back to, never halted, tried again slowly.
|
|
setup
|
|
deliver 2.0 "1 hour ago"
|
|
i=1; while [ $i -le 5 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done
|
|
check "with nothing to go back to there is no rollback" "an installation failure, not an upgrade's" "$(rolled)" "no"
|
|
check "and no halt" "" "$([ -f "$MESH_HOST_STATE_DIR/halted" ] && echo halted || echo running)" "running"
|
|
|
|
# A delivered launcher runs at the host's next clean exit, not at the next boot.
|
|
setup
|
|
unset MESH_HOST_RUN_ONCE
|
|
cp "$LAUNCH" "$WORK/launch"
|
|
cat > "$MESH_HOST_BIN" <<STUB
|
|
#!/bin/sh
|
|
echo start >> "\$MESH_HOST_STATE_DIR/host.starts"
|
|
if [ "\$(wc -l < "\$MESH_HOST_STATE_DIR/host.starts")" -eq 1 ]; then
|
|
# The mesh delivers a new launcher, and this host stands aside.
|
|
sed '2i echo renewed >> "\$MESH_HOST_STATE_DIR/launcher.renewed"' "$WORK/launch" > "$WORK/launch.new"
|
|
chmod +x "$WORK/launch.new"; mv "$WORK/launch.new" "$WORK/launch"
|
|
exit 0
|
|
fi
|
|
sleep 30
|
|
STUB
|
|
chmod +x "$MESH_HOST_BIN"
|
|
"$WORK/launch" >/dev/null 2>&1 &
|
|
LP=$!
|
|
sleep 1
|
|
check "a replaced launcher runs itself at the next clean exit" "or a launcher fix waits for a reboot" \
|
|
"$(cat "$MESH_HOST_STATE_DIR/launcher.renewed" 2>/dev/null || echo NOT-RENEWED)" "renewed"
|
|
kill -TERM "$LP" 2>/dev/null; sleep 1; pkill -f "$MESH_HOST_BIN" 2>/dev/null || true
|
|
|
|
printf '\nlaunch: %d passed, %d failed\n' "$PASS" "$FAIL"
|
|
[ "$FAIL" -eq 0 ]
|