Roll a core build back by a witness that is not the new build (hq to-be 45 Phase 4)
The launcher trusted a counter only a by-hand reconcile ever cleared and a known-good nothing in the daemon wrote, so no machine could roll its host back; the controller and the node tools were replaced in place with nothing kept. - The launcher runs a delivered host that is not known-good on trial: one that crashes, stops for nothing, or does not report within ten minutes goes back to known-good, once per version, recorded in rolled-back. The host proves itself when the mesh takes a report under its own build, says every standing verdict on its reports, never stands aside for a rolled-back version, and restarts its service once when its launcher was replaced on disk. - The engine keeps the controller's and the node tools' previous build beside the new one and judges the new one: the lease taken by the controller it started (read-only direct get of mesh-controller_lease/holder), or this machine's runtime answering $SRV.PING.node-tools.<node>, within sixty seconds of time it could ask. Not healthy: the previous restored, once, said. Proved: the previous deleted. A build declared not-reversible is never rolled back. - Retire never removes a version newer than the running one.
This commit is contained in:
jochen
+133
-44
@@ -27,15 +27,8 @@ setup() {
|
||||
echo "$@" >> "$MESH_HOST_STATE_DIR/host.starts"
|
||||
exit "${STUB_HOST_EXIT:-1}"
|
||||
STUB
|
||||
cat > "$MESH_HOST_LIBEXEC/rollback" <<'STUB'
|
||||
#!/bin/sh
|
||||
echo rolled-back >> "$MESH_HOST_STATE_DIR/rollback.calls"
|
||||
[ -n "${STUB_ROLLBACK_FAILS:-}" ] && exit 1
|
||||
echo "$(cat "$MESH_HOST_STATE_DIR/known-good" 2>/dev/null)" > "$MESH_HOST_STATE_DIR/rollback-attempted"
|
||||
exit 0
|
||||
STUB
|
||||
chmod +x "$MESH_HOST_BIN" "$MESH_HOST_LIBEXEC/rollback"
|
||||
unset STUB_ROLLBACK_FAILS || true
|
||||
chmod +x "$MESH_HOST_BIN"
|
||||
export MESH_HOST_TRIAL_BOUND=600 MESH_HOST_TRIAL_TICK=1 MESH_HOST_STOP_GRACE=1
|
||||
}
|
||||
|
||||
# `|| true` on every launcher call above: a launcher that exits non-zero is something to
|
||||
@@ -47,7 +40,7 @@ check() { if [ "$3" = "$4" ]; then PASS=$((PASS+1)); printf ' ok %s\n' "$1"
|
||||
|
||||
count() { cat "$MESH_HOST_STATE_DIR/start-attempts" 2>/dev/null || echo MISSING; }
|
||||
started() { [ -f "$MESH_HOST_STATE_DIR/host.starts" ] && echo yes || echo no; }
|
||||
rolled() { [ -f "$MESH_HOST_STATE_DIR/rollback.calls" ] && echo yes || echo no; }
|
||||
rolled() { [ -s "$MESH_HOST_STATE_DIR/rolled-back" ] && echo yes || echo no; }
|
||||
|
||||
# --- the ordinary start -----------------------------------------------------------------------
|
||||
setup
|
||||
@@ -62,18 +55,6 @@ i=1; while [ $i -le 3 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done
|
||||
check "three starts do not trigger a rollback" "the limit is exceeded, not reached" "$(rolled)" "no"
|
||||
check "counts them all" "" "$(count)" "3"
|
||||
|
||||
# --- past the limit ---------------------------------------------------------------------------
|
||||
setup
|
||||
echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
|
||||
i=1; while [ $i -le 4 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done
|
||||
check "the fourth start rolls back" "three failures is a binary that does not work" "$(rolled)" "yes"
|
||||
check "and still starts the host" "the rolled-back version has to be run" "$(started)" "yes"
|
||||
# Below the limit, not exactly zero. The rollback resets it and the rolled-back version then
|
||||
# fails once here, so 1 is right — the property is that it did NOT inherit a count already at
|
||||
# the limit, which would halt the new version on its first attempt.
|
||||
check "resets the counter after rolling back" "the new version deserves its own attempts, or it halts at once" \
|
||||
"$([ "$(count)" -lt 3 ] && echo below-limit || echo "at-limit($(count))")" "below-limit"
|
||||
|
||||
# --- the host clears the counter on success ----------------------------------------------------
|
||||
setup
|
||||
i=1; while [ $i -le 2 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done
|
||||
@@ -82,15 +63,6 @@ i=1; while [ $i -le 3 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done
|
||||
check "a cleared counter prevents a rollback" "a node up for months must not roll back on a healthy boot" \
|
||||
"$(rolled)" "no"
|
||||
|
||||
# --- rolled back once already -------------------------------------------------------------------
|
||||
setup
|
||||
echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
|
||||
echo "1.4.2" > "$MESH_HOST_STATE_DIR/rollback-attempted"
|
||||
i=1; while [ $i -le 4 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done
|
||||
check "does not roll back twice" "the previous version failing too means the machine, not the binary" \
|
||||
"$(rolled)" "no"
|
||||
check "halts instead" "" "$([ -f "$MESH_HOST_STATE_DIR/halted" ] && echo halted || echo running)" "halted"
|
||||
|
||||
# --- halted stays halted --------------------------------------------------------------------------
|
||||
setup
|
||||
echo "rolled back and still failing" > "$MESH_HOST_STATE_DIR/halted"
|
||||
@@ -99,19 +71,6 @@ check "a halted node does not start the host" "nothing further is tried automati
|
||||
set +e; "$LAUNCH" >/dev/null 2>&1; RC=$?; set -e
|
||||
check "a halted node exits zero" "a supervisor loop that is slow and visible beats a crash loop" "$RC" "0"
|
||||
|
||||
# --- the rollback itself fails ----------------------------------------------------------------------
|
||||
setup
|
||||
echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
|
||||
STUB_ROLLBACK_FAILS=1; export STUB_ROLLBACK_FAILS
|
||||
i=1; while [ $i -le 4 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done
|
||||
check "a failed rollback halts" "restarting into the same failure would loop forever" \
|
||||
"$([ -f "$MESH_HOST_STATE_DIR/halted" ] && echo halted || echo running)" "halted"
|
||||
# Counted, not "was it ever started": the first three attempts DID start it, correctly, and
|
||||
# only the fourth must not. An earlier version of this asserted the host was never started and
|
||||
# failed for that reason rather than for a fault.
|
||||
check "and does not start it on the halting attempt" "three starts, not four" \
|
||||
"$(wc -l < "$MESH_HOST_STATE_DIR/host.starts" 2>/dev/null || echo 0)" "3"
|
||||
|
||||
# --- a corrupt counter ------------------------------------------------------------------------------
|
||||
#
|
||||
# The values here are chosen because they DISCRIMINATE. An earlier version used
|
||||
@@ -200,6 +159,7 @@ deliver() {
|
||||
cat > "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host" <<STUB
|
||||
#!/bin/sh
|
||||
echo "$1" >> "\$MESH_HOST_STATE_DIR/which.ran"
|
||||
${3:-}
|
||||
exit "\${STUB_HOST_EXIT:-1}"
|
||||
STUB
|
||||
chmod +x "$MESH_HOST_LIBEXEC/versions/$1/nox-mesh-host"
|
||||
@@ -252,5 +212,134 @@ setup
|
||||
"$LAUNCH" >/dev/null 2>&1 || true
|
||||
check "falls back to the host placed by hand" "every first host arrives this way" "$(started)" "yes"
|
||||
|
||||
# --- the launcher witnesses the host's successor (novox/hq to-be 45 §8) --------------------------
|
||||
#
|
||||
# A delivered host that is not the known-good one runs on trial: it must write itself into known-good
|
||||
# (which the host does when the mesh has taken a report it made under its own build) within the bound.
|
||||
# One that crashes, stops for nothing, or never reports goes back to known-good, once, recorded.
|
||||
|
||||
ran_count() { grep -xF "$1" "$MESH_HOST_STATE_DIR/which.ran" 2>/dev/null | wc -l | tr -d ' '; }
|
||||
last_ran() { tail -n 1 "$MESH_HOST_STATE_DIR/which.ran" 2>/dev/null || echo NONE; }
|
||||
record_of() { cut -f"$2" "$MESH_HOST_STATE_DIR/rolled-back" 2>/dev/null | sed -n "${1}p"; }
|
||||
records() { grep . "$MESH_HOST_STATE_DIR/rolled-back" 2>/dev/null | wc -l | tr -d ' '; }
|
||||
|
||||
# A new host that crashes at once: three tries, then the known-good one, recorded once.
|
||||
setup
|
||||
deliver 1.0 "2 hours ago"
|
||||
deliver 2.0 "1 hour ago"
|
||||
echo 1.0 > "$MESH_HOST_STATE_DIR/known-good"
|
||||
i=1; while [ $i -le 4 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done
|
||||
check "a crashing new host is tried three times" "the limit is the evidence" "$(ran_count 2.0)" "3"
|
||||
check "then the known-good one runs" "the witness restores the build before" "$(last_ran)" "1.0"
|
||||
check "the rollback is recorded once" "one line per verdict" "$(records)" "1"
|
||||
check "naming what failed" "from" "$(record_of 1 1)" "2.0"
|
||||
check "and what runs instead" "to" "$(record_of 1 2)" "1.0"
|
||||
check "as a rollback" "outcome" "$(record_of 1 4)" "rolled-back"
|
||||
check "saying why" "why" "$(record_of 1 5 | grep -c 'failed 3 times')" "1"
|
||||
check "the counter starts again for the version gone back to" "or it halts at once" \
|
||||
"$([ "$(count)" -lt 3 ] && echo below-limit || echo "at-limit($(count))")" "below-limit"
|
||||
# Not retried: the rolled-back version is still the newest delivered, and is never started again.
|
||||
i=1; while [ $i -le 2 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done
|
||||
check "a rolled-back version is never started again" "one rollback per version" "$(ran_count 2.0)" "3"
|
||||
check "and it is not recorded twice" "" "$(records)" "1"
|
||||
# What it went back to failing too is the machine, not a binary: halted, and said.
|
||||
"$LAUNCH" >/dev/null 2>&1 || true
|
||||
check "the version gone back to failing too halts" "rolling back again would flap" \
|
||||
"$([ -f "$MESH_HOST_STATE_DIR/halted" ] && echo halted || echo running)" "halted"
|
||||
check "and the halt is recorded" "" "$(record_of 2 4)" "halted"
|
||||
check "with no rollback to a third version" "" "$(ran_count 2.0)" "3"
|
||||
|
||||
# A new host that exits cleanly at once, standing aside for nothing: counted, then rolled back.
|
||||
setup
|
||||
deliver 1.0 "2 hours ago"
|
||||
deliver 2.0 "1 hour ago" "exit 0"
|
||||
echo 1.0 > "$MESH_HOST_STATE_DIR/known-good"
|
||||
i=1; while [ $i -le 4 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done
|
||||
check "a new host that stops for nothing is rolled back" "a clean exit with nothing newer is not standing aside" \
|
||||
"$(record_of 1 1) -> $(record_of 1 2)" "2.0 -> 1.0"
|
||||
check "after three tries" "" "$(ran_count 2.0)" "3"
|
||||
|
||||
# A new host that runs and never reports: stopped at the bound, rolled back.
|
||||
setup
|
||||
export MESH_HOST_TRIAL_BOUND=2
|
||||
deliver 1.0 "2 hours ago"
|
||||
deliver 2.0 "1 hour ago" "sleep 30"
|
||||
echo 1.0 > "$MESH_HOST_STATE_DIR/known-good"
|
||||
"$LAUNCH" >/dev/null 2>&1 || true
|
||||
check "a host that never reports is rolled back at the bound" "started and did nothing" \
|
||||
"$(record_of 1 1) -> $(record_of 1 2)" "2.0 -> 1.0"
|
||||
check "saying it did not report" "" "$(record_of 1 5 | grep -c 'did not report within 2s')" "1"
|
||||
check "and the known-good one runs" "" "$(last_ran)" "1.0"
|
||||
check "the silent host is not left running" "the witness stops it" \
|
||||
"$(pgrep -f "$MESH_HOST_LIBEXEC/versions/2.0" >/dev/null 2>&1 && echo running || echo stopped)" "stopped"
|
||||
|
||||
# A new host that reports in bound is proved: no rollback, and the trial ends.
|
||||
setup
|
||||
export MESH_HOST_TRIAL_BOUND=3
|
||||
deliver 1.0 "2 hours ago"
|
||||
deliver 2.0 "1 hour ago" "echo 2.0 > \"\$MESH_HOST_STATE_DIR/known-good\"; sleep 4"
|
||||
echo 1.0 > "$MESH_HOST_STATE_DIR/known-good"
|
||||
"$LAUNCH" >/dev/null 2>&1 || true
|
||||
check "a host that reports in bound is not rolled back" "a healthy update undoes nothing" "$(rolled)" "no"
|
||||
check "it ran past its bound" "the witness let it be once it reported" "$(ran_count 2.0)" "1"
|
||||
"$LAUNCH" >/dev/null 2>&1 || true
|
||||
check "and is the one run after" "known-good now" "$(last_ran)" "2.0"
|
||||
check "and no longer on trial" "the trial file goes" \
|
||||
"$([ -e "$MESH_HOST_STATE_DIR/trial" ] && echo on-trial || echo proved)" "proved"
|
||||
|
||||
# A launcher restarted after the bound passed rolls back without starting the host again.
|
||||
setup
|
||||
deliver 1.0 "2 hours ago"
|
||||
deliver 2.0 "1 hour ago"
|
||||
echo 1.0 > "$MESH_HOST_STATE_DIR/known-good"
|
||||
printf '2.0 %s\n' "$(( $(date +%s) - 4000 ))" > "$MESH_HOST_STATE_DIR/trial"
|
||||
"$LAUNCH" >/dev/null 2>&1 || true
|
||||
check "a trial past its bound is ended at the next start" "a host that keeps restarting cannot outrun its bound" \
|
||||
"$(ran_count 2.0) $(record_of 1 1)" "0 2.0"
|
||||
|
||||
# A newer host delivered after a rollback runs, on trial; the one rolled back stays refused.
|
||||
setup
|
||||
deliver 1.0 "3 hours ago"
|
||||
deliver 2.0 "2 hours ago"
|
||||
echo 1.0 > "$MESH_HOST_STATE_DIR/known-good"
|
||||
printf '2.0\t1.0\t%s\trolled-back\tit failed 3 times in a row\n' "$(date +%s)" > "$MESH_HOST_STATE_DIR/rolled-back"
|
||||
echo 1.0 > "$MESH_HOST_STATE_DIR/rollback-pinned"
|
||||
touch -d "1 hour ago" "$MESH_HOST_STATE_DIR/rollback-pinned"
|
||||
deliver 3.0 "1 minute ago"
|
||||
"$LAUNCH" >/dev/null 2>&1 || true
|
||||
check "a newer delivery after a rollback runs" "a rolled-back version blocks only itself" "$(last_ran)" "3.0"
|
||||
check "and the pin goes" "" "$([ -e "$MESH_HOST_STATE_DIR/rollback-pinned" ] && echo pinned || echo free)" "free"
|
||||
check "on trial" "" "$(cut -d' ' -f1 "$MESH_HOST_STATE_DIR/trial" 2>/dev/null)" "3.0"
|
||||
|
||||
# No known-good delivered: nothing to go back to, never halted, tried again slowly.
|
||||
setup
|
||||
deliver 2.0 "1 hour ago"
|
||||
i=1; while [ $i -le 5 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done
|
||||
check "with nothing to go back to there is no rollback" "an installation failure, not an upgrade's" "$(rolled)" "no"
|
||||
check "and no halt" "" "$([ -f "$MESH_HOST_STATE_DIR/halted" ] && echo halted || echo running)" "running"
|
||||
|
||||
# A delivered launcher runs at the host's next clean exit, not at the next boot.
|
||||
setup
|
||||
unset MESH_HOST_RUN_ONCE
|
||||
cp "$LAUNCH" "$WORK/launch"
|
||||
cat > "$MESH_HOST_BIN" <<STUB
|
||||
#!/bin/sh
|
||||
echo start >> "\$MESH_HOST_STATE_DIR/host.starts"
|
||||
if [ "\$(wc -l < "\$MESH_HOST_STATE_DIR/host.starts")" -eq 1 ]; then
|
||||
# The mesh delivers a new launcher, and this host stands aside.
|
||||
sed '2i echo renewed >> "\$MESH_HOST_STATE_DIR/launcher.renewed"' "$WORK/launch" > "$WORK/launch.new"
|
||||
chmod +x "$WORK/launch.new"; mv "$WORK/launch.new" "$WORK/launch"
|
||||
exit 0
|
||||
fi
|
||||
sleep 30
|
||||
STUB
|
||||
chmod +x "$MESH_HOST_BIN"
|
||||
"$WORK/launch" >/dev/null 2>&1 &
|
||||
LP=$!
|
||||
sleep 1
|
||||
check "a replaced launcher runs itself at the next clean exit" "or a launcher fix waits for a reboot" \
|
||||
"$(cat "$MESH_HOST_STATE_DIR/launcher.renewed" 2>/dev/null || echo NOT-RENEWED)" "renewed"
|
||||
kill -TERM "$LP" 2>/dev/null; sleep 1; pkill -f "$MESH_HOST_BIN" 2>/dev/null || true
|
||||
|
||||
printf '\nlaunch: %d passed, %d failed\n' "$PASS" "$FAIL"
|
||||
[ "$FAIL" -eq 0 ]
|
||||
|
||||
+199
-39
@@ -6,10 +6,22 @@
|
||||
# up, when to roll back: all of it is policy, and policy in a unit file can only be read and
|
||||
# hoped for.
|
||||
#
|
||||
# **It is the witness of the host's own successor** (novox/hq to-be 45 §8, ADR 0227 rule 8). A
|
||||
# delivered host that is not the known-good one runs on trial: it must report its declaration
|
||||
# under its own build — which it marks by writing itself into known-good — within a bound. One
|
||||
# that crashes repeatedly, exits without standing aside for anything, or does not report in
|
||||
# bound is stopped, recorded in `rolled-back` with why, and the known-good one is run. The host
|
||||
# says every record on its reports, so the mesh raises it as a condition. One rollback per
|
||||
# version: a version in `rolled-back` is never started again; a newer delivery is.
|
||||
#
|
||||
# It does NOT exec the host. Exec would replace this process, and then only the init could
|
||||
# restart anything — which is the arrangement this exists to remove. The cost of staying is
|
||||
# signal handling, below.
|
||||
#
|
||||
# Everything a rollback does is one of: read a file, look at a directory, write a file. It shares
|
||||
# no code with the host and calls none of it: a binary that cannot start cannot be its own
|
||||
# recovery.
|
||||
#
|
||||
# POSIX sh. `set -e` is deliberately absent: this script's whole job is to inspect exit codes,
|
||||
# and -e would make it exit on the first one it is meant to handle.
|
||||
set -u
|
||||
@@ -23,16 +35,46 @@ VERSIONS="$LIBEXEC/versions"
|
||||
BINARY="nox-mesh-host"
|
||||
LIMIT="${MESH_HOST_START_LIMIT:-3}"
|
||||
BACKOFF="${MESH_HOST_BACKOFF:-5}"
|
||||
# How long a host on trial has to report, in seconds: to-be 45 §8's ten minutes from the apply.
|
||||
BOUND="${MESH_HOST_TRIAL_BOUND:-600}"
|
||||
TICK="${MESH_HOST_TRIAL_TICK:-5}"
|
||||
GRACE="${MESH_HOST_STOP_GRACE:-20}"
|
||||
ONCE="${MESH_HOST_RUN_ONCE:-}" # tests run one iteration; nothing else sets this
|
||||
|
||||
ATTEMPTS="$STATE_DIR/start-attempts"
|
||||
HALTED="$STATE_DIR/halted"
|
||||
PINNED="$STATE_DIR/rollback-pinned"
|
||||
KNOWN_GOOD="$STATE_DIR/known-good"
|
||||
# One line per verdict: from, to, when (seconds since the epoch), outcome, why — tab-separated, read
|
||||
# by the host (internal/upgrade) and said on its reports.
|
||||
ROLLED="$STATE_DIR/rolled-back"
|
||||
TRIAL="$STATE_DIR/trial"
|
||||
EXPIRED="$STATE_DIR/trial-expired"
|
||||
|
||||
say() { echo "nox-mesh-host-launch: $*" >&2; }
|
||||
|
||||
# Which host to run: the version a rollback pinned, or the most recently delivered one, or the one
|
||||
# placed by hand when nothing has been delivered (novox/hq ADR 0141).
|
||||
now() { date +%s; }
|
||||
|
||||
known_good() { tr -d '[:space:]' < "$KNOWN_GOOD" 2>/dev/null || true; }
|
||||
|
||||
delivered() { [ -n "$1" ] && [ -x "$VERSIONS/$1/$BINARY" ]; }
|
||||
|
||||
rolled_back() { [ -s "$ROLLED" ] && cut -f1 "$ROLLED" 2>/dev/null | grep -qxF "$1"; }
|
||||
|
||||
# The version a host path is: the directory it was delivered in, or nothing for the host placed by hand.
|
||||
version_of() {
|
||||
case "$1" in
|
||||
"$VERSIONS"/*/"$BINARY") v="${1#"$VERSIONS"/}"; echo "${v%/"$BINARY"}" ;;
|
||||
*) echo "" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
record() { # from to outcome why
|
||||
printf '%s\t%s\t%s\t%s\t%s\n' "$1" "$2" "$(now)" "$3" "$4" >> "$ROLLED"
|
||||
}
|
||||
|
||||
# Which host to run: the newest delivered one that was never rolled back — or, while a rollback's pin
|
||||
# stands, the version it pinned — or the one placed by hand when nothing has been delivered.
|
||||
#
|
||||
# **Asked every time round the loop, not once.** Standing aside for a successor is a clean exit, and
|
||||
# the next turn has to run what is on disk NOW — resolving this once would restart the same binary
|
||||
@@ -41,27 +83,76 @@ say() { echo "nox-mesh-host-launch: $*" >&2; }
|
||||
# Newest by when it arrived, never by how its name sorts: a version string is whatever the source was
|
||||
# described as, and those do not sort — "1.10" orders before "1.9". Ordering by name would start an
|
||||
# older host and call it an upgrade.
|
||||
#
|
||||
# **A pin stands until something newer arrives.** A version delivered after the pin was written, and
|
||||
# never rolled back, is a new build the mesh asks for: the pin goes and it runs, on trial.
|
||||
pick_host() {
|
||||
if [ -s "$PINNED" ]; then
|
||||
pinned="$(tr -d '[:space:]' < "$PINNED" 2>/dev/null || true)"
|
||||
if [ -n "$pinned" ] && [ -x "$VERSIONS/$pinned/$BINARY" ]; then
|
||||
echo "$VERSIONS/$pinned/$BINARY"
|
||||
return 0
|
||||
fi
|
||||
say "the pinned version '$pinned' is not delivered; ignoring the pin"
|
||||
fi
|
||||
newest=
|
||||
# A directory with no executable in it is not a version: an interrupted delivery leaves one, and
|
||||
# running "the newest" would then mean running nothing.
|
||||
for candidate in $(ls -1t "$VERSIONS" 2>/dev/null || true); do
|
||||
if [ -x "$VERSIONS/$candidate/$BINARY" ]; then
|
||||
echo "$VERSIONS/$candidate/$BINARY"
|
||||
return 0
|
||||
fi
|
||||
delivered "$candidate" || continue
|
||||
rolled_back "$candidate" && continue
|
||||
newest="$candidate"
|
||||
break
|
||||
done
|
||||
if [ -s "$PINNED" ]; then
|
||||
pinned="$(tr -d '[:space:]' < "$PINNED" 2>/dev/null || true)"
|
||||
if [ -n "$newest" ] && [ "$newest" != "$pinned" ] && [ "$VERSIONS/$newest" -nt "$PINNED" ]; then
|
||||
say "host $newest arrived after the rollback to $pinned; running it"
|
||||
rm -f "$PINNED"
|
||||
elif delivered "$pinned"; then
|
||||
echo "$VERSIONS/$pinned/$BINARY"
|
||||
return 0
|
||||
else
|
||||
say "the pinned version '$pinned' is not delivered; ignoring the pin"
|
||||
fi
|
||||
fi
|
||||
if [ -n "$newest" ]; then
|
||||
echo "$VERSIONS/$newest/$BINARY"
|
||||
return 0
|
||||
fi
|
||||
echo "$FALLBACK"
|
||||
}
|
||||
|
||||
# Go back from `from` to the known-good version, once, and say so. False when there is nowhere to go.
|
||||
roll_back() { # from why
|
||||
kg="$(known_good)"
|
||||
if [ -z "$1" ] || [ "$1" = "$kg" ] || ! delivered "$kg"; then
|
||||
return 1
|
||||
fi
|
||||
record "$1" "$kg" rolled-back "$2"
|
||||
# The pin is what stops the launcher starting a version newer than known-good that is not the one
|
||||
# rolled back; the record is what stops it starting this one again.
|
||||
printf '%s\n' "$kg" > "$PINNED"
|
||||
rm -f "$TRIAL"
|
||||
say "rolled back from host $1 to $kg: $2"
|
||||
return 0
|
||||
}
|
||||
|
||||
# Watch a host on trial: done when it writes itself into known-good, stopped when the bound passes.
|
||||
# A subshell beside the host, so the launcher's own wait is untouched.
|
||||
trial_watch() { # pid version deadline
|
||||
while kill -0 "$1" 2>/dev/null; do
|
||||
[ "$(known_good)" = "$2" ] && return 0
|
||||
if [ "$(now)" -ge "$3" ]; then
|
||||
printf '%s\n' "$2" > "$EXPIRED"
|
||||
say "host $2 did not report within ${BOUND}s of starting; stopping it"
|
||||
kill -TERM "$1" 2>/dev/null
|
||||
waited=0
|
||||
while kill -0 "$1" 2>/dev/null && [ "$waited" -lt "$GRACE" ]; do
|
||||
sleep 1
|
||||
waited=$((waited + 1))
|
||||
done
|
||||
kill -KILL "$1" 2>/dev/null
|
||||
return 0
|
||||
fi
|
||||
sleep "$TICK"
|
||||
done
|
||||
}
|
||||
|
||||
child=
|
||||
watcher=
|
||||
stopping=
|
||||
|
||||
# The machine is shutting down. Pass it on and wait for the host to finish — a supervisor that
|
||||
@@ -77,6 +168,9 @@ on_term() {
|
||||
trap on_term TERM INT
|
||||
|
||||
mkdir -p "$STATE_DIR"
|
||||
# What this launcher is, so a delivered successor of it is run at the next clean exit rather than at
|
||||
# the next boot.
|
||||
SELF="$(cksum < "$0" 2>/dev/null || true)"
|
||||
|
||||
while :; do
|
||||
if [ -n "$stopping" ]; then
|
||||
@@ -89,8 +183,8 @@ while :; do
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Consecutive failed starts, not starts. Cleared by the host itself when it completes a
|
||||
# reconcile, which is the only evidence either this or known-good has.
|
||||
# Consecutive failed starts, not starts. Cleared by the host itself when it reports, which is
|
||||
# the only evidence either this or known-good has.
|
||||
#
|
||||
# Read the FIRST FIELD, then insist it is a plain integer.
|
||||
#
|
||||
@@ -105,49 +199,102 @@ while :; do
|
||||
'' | *[!0-9]*) count=0 ;;
|
||||
esac
|
||||
|
||||
HOST="$(pick_host)"
|
||||
version="$(version_of "$HOST")"
|
||||
|
||||
if [ "$count" -ge "$LIMIT" ]; then
|
||||
if [ -e "$STATE_DIR/rollback-attempted" ]; then
|
||||
say "the host failed $count times after a rollback. the previous version does not"
|
||||
say "start either, so this is the machine and not the binary."
|
||||
if roll_back "$version" "it failed $count times in a row"; then
|
||||
# Fresh count for the version now run: it deserves its own attempts, and without this
|
||||
# it inherits a count already over the limit and halts at once. The variable too, not
|
||||
# only the file: resetting one and not the other made the next failure count from the
|
||||
# OLD value — so the rolled-back version got one attempt instead of three.
|
||||
count=0
|
||||
printf '%s\n' "$count" > "$ATTEMPTS"
|
||||
continue
|
||||
fi
|
||||
kg="$(known_good)"
|
||||
if [ -n "$kg" ] && { [ "$version" = "$kg" ] || [ -z "$version" ]; } && [ -s "$ROLLED" ]; then
|
||||
# Already gone back, and what it went back to fails as well: this is the machine and
|
||||
# not a binary. Rolling back again would flap between two versions for ever.
|
||||
say "the host failed $count times after a rollback. the version it went back to does"
|
||||
say "not start either, so this is the machine and not the binary."
|
||||
record "${version:-placed-by-hand}" "" halted "the version rolled back to failed $count times as well"
|
||||
printf 'rolled back and still failing\n' > "$HALTED"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
say "the host failed $count times. rolling back."
|
||||
if "$LIBEXEC/rollback"; then
|
||||
# Fresh count for the version just installed: it deserves its own attempts, and
|
||||
# without this it inherits a count already over the limit and halts at once.
|
||||
#
|
||||
# The variable too, not only the file. Resetting one and not the other made the
|
||||
# next failure count from the OLD value — so the rolled-back version got one
|
||||
# attempt instead of three.
|
||||
count=0
|
||||
printf '%s\n' "$count" > "$ATTEMPTS"
|
||||
else
|
||||
say "rollback failed. halting rather than restarting into the same failure."
|
||||
printf 'rollback failed\n' > "$HALTED"
|
||||
exit 0
|
||||
fi
|
||||
# Nothing to go back to: no host has ever reported here, or the one that did was placed by
|
||||
# hand and is not delivered. An installation failure rather than an upgrade's — said, and
|
||||
# tried again slowly, never guessed at.
|
||||
say "the host failed $count times and there is no delivered known-good version to go back to."
|
||||
count=0
|
||||
printf '%s\n' "$count" > "$ATTEMPTS"
|
||||
fi
|
||||
|
||||
HOST="$(pick_host)"
|
||||
if [ ! -x "$HOST" ]; then
|
||||
say "no host to run: nothing delivered under $VERSIONS and $FALLBACK is not executable."
|
||||
printf 'no host binary\n' > "$HALTED"
|
||||
exit 0
|
||||
fi
|
||||
say "running $HOST"
|
||||
|
||||
# **On trial**: a delivered version that is not the known-good one, while a known-good one is
|
||||
# delivered to go back to. The trial starts when this version was first started, and survives
|
||||
# this launcher restarting.
|
||||
trial=
|
||||
deadline=
|
||||
kg="$(known_good)"
|
||||
if [ -n "$version" ] && [ "$version" != "$kg" ] && delivered "$kg"; then
|
||||
trial=yes
|
||||
started=
|
||||
if [ -s "$TRIAL" ]; then
|
||||
read -r on since _ < "$TRIAL" 2>/dev/null || on=
|
||||
[ "${on:-}" = "$version" ] && started="${since:-}"
|
||||
fi
|
||||
case "$started" in
|
||||
'' | *[!0-9]*) started="$(now)"; printf '%s %s\n' "$version" "$started" > "$TRIAL" ;;
|
||||
esac
|
||||
deadline=$((started + BOUND))
|
||||
if [ "$(now)" -ge "$deadline" ]; then
|
||||
roll_back "$version" "it did not report within ${BOUND}s of starting" && continue
|
||||
fi
|
||||
else
|
||||
rm -f "$TRIAL"
|
||||
fi
|
||||
|
||||
rm -f "$EXPIRED"
|
||||
say "running $HOST${trial:+ (on trial until it reports)}"
|
||||
"$HOST" run &
|
||||
child=$!
|
||||
watcher=
|
||||
if [ -n "$trial" ]; then
|
||||
trial_watch "$child" "$version" "$deadline" &
|
||||
watcher=$!
|
||||
fi
|
||||
status=0
|
||||
wait "$child" || status=$?
|
||||
if [ -n "$stopping" ]; then
|
||||
# The signal interrupted the wait, not the host: it was told, and is finishing what it was
|
||||
# doing. Waited for, so the service manager does not kill it half way through an apply.
|
||||
wait "$child" 2>/dev/null
|
||||
fi
|
||||
child=
|
||||
if [ -n "$watcher" ]; then
|
||||
kill "$watcher" 2>/dev/null
|
||||
wait "$watcher" 2>/dev/null
|
||||
watcher=
|
||||
fi
|
||||
|
||||
if [ -n "$stopping" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ -n "$trial" ] && [ -s "$EXPIRED" ] && [ "$(cat "$EXPIRED" 2>/dev/null)" = "$version" ]; then
|
||||
rm -f "$EXPIRED"
|
||||
roll_back "$version" "it did not report within ${BOUND}s of starting"
|
||||
count=0
|
||||
printf '%s\n' "$count" > "$ATTEMPTS"
|
||||
continue
|
||||
fi
|
||||
|
||||
# A signal the host did not survive, and we are not shutting down: treat it as a crash.
|
||||
case "$status" in
|
||||
0)
|
||||
@@ -158,8 +305,21 @@ while :; do
|
||||
# incremented here rather than before the start: counting attempts meant a host
|
||||
# that upgraded itself three times rolled itself back, having worked perfectly
|
||||
# every time.
|
||||
say "the host exited cleanly; starting it again"
|
||||
continue
|
||||
#
|
||||
# **Unless it stood aside for nothing.** A host on trial that exits cleanly while the
|
||||
# same host is still the one to run has not stood aside for a successor: it has
|
||||
# stopped, and a host that stops at once would otherwise loop here for ever unseen.
|
||||
if [ -n "$trial" ] && [ "$(pick_host)" = "$HOST" ] && [ "$(known_good)" != "$version" ]; then
|
||||
say "host $version exited cleanly on trial with nothing newer to stand aside for"
|
||||
else
|
||||
# A delivered successor of this launcher runs from here on, not from the next boot.
|
||||
if [ -n "$SELF" ] && [ "$(cksum < "$0" 2>/dev/null || true)" != "$SELF" ]; then
|
||||
say "the launcher was replaced; running the new one"
|
||||
exec "$0"
|
||||
fi
|
||||
say "the host exited cleanly; starting it again"
|
||||
continue
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
#!/bin/sh
|
||||
# Put the host back on the last version that worked.
|
||||
#
|
||||
# **Superseded by the launcher itself** (novox/hq to-be 45 §8): a launcher from then on rolls back on
|
||||
# its own — per version, recorded in `rolled-back` and said on the host's reports — and does not call
|
||||
# this. Kept, unchanged, for the launchers before it still running on a machine until it restarts
|
||||
# them; the host restarts its service once it sees its launcher was replaced.
|
||||
#
|
||||
# novox/hq ADR 0005 and ADR 0141. This runs when nox-mesh-host will not start, so it shares no code
|
||||
# with it and calls none of it: a binary that cannot start cannot be its own recovery. POSIX sh, no
|
||||
# bashisms, nothing that has to be installed.
|
||||
|
||||
Reference in New Issue
Block a user