Makefile

@@ -12,6 +12,7 @@ check: fmt vet test packaging-test build
 
 packaging-test:
 	@./packaging/rollback_test.sh
+	@./packaging/launch_test.sh
 
 fmt:
 	@test -z "$$(gofmt -l . )" || { echo "unformatted:"; gofmt -l . ; exit 1; }

packaging/launch_test.sh

@@ -0,0 +1,135 @@
+#!/bin/sh
+# Tests for nox-mesh-host-launch.
+#
+# The counter is the whole mechanism and it is the part to get wrong: never cleared and a node
+# rolls back on a healthy boot; cleared too eagerly and it never rolls back at all. So the
+# counter is what most of these assert.
+set -eu
+cd "$(dirname "$0")"
+LAUNCH="$PWD/nox-mesh-host-launch"
+PASS=0; FAIL=0
+
+setup() {
+	WORK="$(mktemp -d)"
+	export MESH_HOST_STATE_DIR="$WORK/state"
+	export MESH_HOST_LIBEXEC="$WORK/libexec"
+	export MESH_HOST_BIN="$WORK/bin/nox-mesh-host"
+	export MESH_HOST_START_LIMIT=3
+	mkdir -p "$MESH_HOST_STATE_DIR" "$MESH_HOST_LIBEXEC" "$WORK/bin"
+
+	# A host that records being started. It exits immediately, which is what the launcher's
+	# exec makes indistinguishable from a host that ran for a week — the launcher is gone by
+	# then either way.
+	cat > "$MESH_HOST_BIN" <<'STUB'
+#!/bin/sh
+echo "$@" >> "$MESH_HOST_STATE_DIR/host.starts"
+exit 0
+STUB
+	cat > "$MESH_HOST_LIBEXEC/rollback" <<'STUB'
+#!/bin/sh
+echo rolled-back >> "$MESH_HOST_STATE_DIR/rollback.calls"
+[ -n "${STUB_ROLLBACK_FAILS:-}" ] && exit 1
+echo "$(cat "$MESH_HOST_STATE_DIR/known-good" 2>/dev/null)" > "$MESH_HOST_STATE_DIR/rollback-attempted"
+exit 0
+STUB
+	chmod +x "$MESH_HOST_BIN" "$MESH_HOST_LIBEXEC/rollback"
+	unset STUB_ROLLBACK_FAILS || true
+}
+
+# `|| true` on every launcher call above: a launcher that exits non-zero is something to
+# ASSERT, not something to abort on. With `set -e` and a bare call, removing a guard from the
+# launcher killed this script at the first corrupt-counter case and silently skipped the rest —
+# reporting a full pass over tests that never ran.
+check() { if [ "$3" = "$4" ]; then PASS=$((PASS+1)); printf '  ok   %s\n' "$1"
+	else FAIL=$((FAIL+1)); printf '  FAIL %s\n       %s\n       got:      %s\n       expected: %s\n' "$1" "$2" "$3" "$4"; fi; }
+
+count() { cat "$MESH_HOST_STATE_DIR/start-attempts" 2>/dev/null || echo MISSING; }
+started() { [ -f "$MESH_HOST_STATE_DIR/host.starts" ] && echo yes || echo no; }
+rolled() { [ -f "$MESH_HOST_STATE_DIR/rollback.calls" ] && echo yes || echo no; }
+
+# --- the ordinary start -----------------------------------------------------------------------
+setup
+"$LAUNCH" >/dev/null 2>&1 || true
+check "starts the host" "the common case, every boot" "$(started)" "yes"
+check "counts the attempt" "the counter is what decides a rollback later" "$(count)" "1"
+check "does not roll back" "a first start is not a failure" "$(rolled)" "no"
+
+# --- failures below the limit -----------------------------------------------------------------
+setup
+i=1; while [ $i -le 3 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done
+check "three starts do not trigger a rollback" "the limit is exceeded, not reached" "$(rolled)" "no"
+check "counts them all" "" "$(count)" "3"
+
+# --- past the limit ---------------------------------------------------------------------------
+setup
+echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
+i=1; while [ $i -le 4 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done
+check "the fourth start rolls back" "three failures is a binary that does not work" "$(rolled)" "yes"
+check "and still starts the host" "the rolled-back version has to be run" "$(started)" "yes"
+check "resets the counter after rolling back" "the new version deserves its own attempts, or it halts at once" \
+	"$(count)" "0"
+
+# --- the host clears the counter on success ----------------------------------------------------
+setup
+i=1; while [ $i -le 2 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done
+printf '0\n' > "$MESH_HOST_STATE_DIR/start-attempts"   # what the host does on a completed reconcile
+i=1; while [ $i -le 3 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done
+check "a cleared counter prevents a rollback" "a node up for months must not roll back on a healthy boot" \
+	"$(rolled)" "no"
+
+# --- rolled back once already -------------------------------------------------------------------
+setup
+echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
+echo "1.4.2" > "$MESH_HOST_STATE_DIR/rollback-attempted"
+i=1; while [ $i -le 4 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done
+check "does not roll back twice" "the previous version failing too means the machine, not the binary" \
+	"$(rolled)" "no"
+check "halts instead" "" "$([ -f "$MESH_HOST_STATE_DIR/halted" ] && echo halted || echo running)" "halted"
+
+# --- halted stays halted --------------------------------------------------------------------------
+setup
+echo "rolled back and still failing" > "$MESH_HOST_STATE_DIR/halted"
+"$LAUNCH" >/dev/null 2>&1 || true
+check "a halted node does not start the host" "nothing further is tried automatically" "$(started)" "no"
+set +e; "$LAUNCH" >/dev/null 2>&1; RC=$?; set -e
+check "a halted node exits zero" "a supervisor loop that is slow and visible beats a crash loop" "$RC" "0"
+
+# --- the rollback itself fails ----------------------------------------------------------------------
+setup
+echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
+STUB_ROLLBACK_FAILS=1; export STUB_ROLLBACK_FAILS
+i=1; while [ $i -le 4 ]; do "$LAUNCH" >/dev/null 2>&1 || true; i=$((i+1)); done
+check "a failed rollback halts" "restarting into the same failure would loop forever" \
+	"$([ -f "$MESH_HOST_STATE_DIR/halted" ] && echo halted || echo running)" "halted"
+# Counted, not "was it ever started": the first three attempts DID start it, correctly, and
+# only the fourth must not. An earlier version of this asserted the host was never started and
+# failed for that reason rather than for a fault.
+check "and does not start it on the halting attempt" "three starts, not four" \
+	"$(wc -l < "$MESH_HOST_STATE_DIR/host.starts" 2>/dev/null || echo 0)" "3"
+
+# --- a corrupt counter ------------------------------------------------------------------------------
+#
+# The values here are chosen because they DISCRIMINATE. An earlier version used
+# "not-a-number", which shell arithmetic happens to evaluate to 0 — so the test passed with the
+# guard removed and proved nothing. These two do not:
+#
+#   5x     shell arithmetic errors, and under `set -e` the launcher dies without starting the host
+#   0x10   is read as HEX 16 — past the limit, so a healthy node would roll back for no reason
+for corrupt in "5x" "0x10" "1 2" ""; do
+	setup
+	echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
+	printf '%s\n' "$corrupt" > "$MESH_HOST_STATE_DIR/start-attempts"
+	"$LAUNCH" >/dev/null 2>&1 || true
+	# The exact number is not the property — "1 2" legitimately recovers a leading 1, while
+	# "5x" is rejected to 0. What must hold for every one of them is that the launcher
+	# survives its own state and does not read it as "past the limit".
+	check "corrupt counter [$corrupt]: starts the host" "the launcher must not die on its own state" \
+		"$(started)" "yes"
+	check "corrupt counter [$corrupt]: does not roll back" "a healthy node must not roll back on a bad counter" \
+		"$(rolled)" "no"
+	check "corrupt counter [$corrupt]: counter is a sane integer" "it is written back for the next start to read" \
+		"$(count | grep -cE '^[0-9]+$')" "1"
+done
+
+printf '\nlaunch: %d passed, %d failed\n' "$PASS" "$FAIL"
+[ "$FAIL" -eq 0 ]

packaging/nox-mesh-host-launch

@@ -0,0 +1,72 @@
+#!/bin/sh
+# Start the host, and decide what to do when it will not start.
+#
+# novox/hq ADR 0061. The init is asked for two things — start this at boot, start it again if it
+# exits — and everything else is here, because this is the one piece that has to work on a
+# machine where the host does not. Unit-file syntax cannot be tested; this can.
+#
+# POSIX sh, no bashisms, nothing that has to be installed.
+set -eu
+
+STATE_DIR="${MESH_HOST_STATE_DIR:-/var/lib/mesh-host}"
+LIBEXEC="${MESH_HOST_LIBEXEC:-/usr/lib/nox-mesh-host}"
+HOST="${MESH_HOST_BIN:-/usr/bin/nox-mesh-host}"
+LIMIT="${MESH_HOST_START_LIMIT:-3}"
+
+ATTEMPTS="$STATE_DIR/start-attempts"
+HALTED="$STATE_DIR/halted"
+
+say() { echo "nox-mesh-host-launch: $*" >&2; }
+
+mkdir -p "$STATE_DIR"
+
+# Halted: rolled back once and the previous version failed too, so the binary is not the problem.
+# Nothing further is tried automatically. Exit zero — a supervisor restarting this forever is a
+# slow visible loop rather than a crash loop, and the node stays down until a person looks.
+if [ -e "$HALTED" ]; then
+	say "halted: $(cat "$HALTED" 2>/dev/null || echo 'reason not recorded')"
+	say "not starting the host. this node needs a person."
+	exit 0
+fi
+
+# Read the FIRST FIELD, then insist it is a plain integer.
+#
+# Stripping whitespace instead concatenates, and that is not a hypothetical: a counter file
+# holding "1 2" became "12", which is past the limit, so a healthy node rolled itself back. An
+# unreadable counter must fail towards "start normally", never towards "give up".
+count=0
+if [ -s "$ATTEMPTS" ]; then
+	read -r count _ < "$ATTEMPTS" 2>/dev/null || count=0
+fi
+case "${count:-}" in
+	'' | *[!0-9]*) count=0 ;;
+esac
+
+count=$((count + 1))
+printf '%s\n' "$count" > "$ATTEMPTS"
+
+if [ "$count" -gt "$LIMIT" ]; then
+	# The host has failed to get through a reconcile $LIMIT times running. The counter is
+	# cleared by the host itself on success, so reaching here means none of those starts
+	# worked — not that the machine has been up a long time.
+	if [ -e "$STATE_DIR/rollback-attempted" ]; then
+		say "the host failed $count times after a rollback. the previous version does not"
+		say "start either, so this is the machine and not the binary."
+		printf 'rolled back and still failing\n' > "$HALTED"
+		exit 0
+	fi
+
+	say "the host failed $count times. rolling back."
+	if "$LIBEXEC/rollback"; then
+		# Fresh count for the version we just installed: it deserves its own attempts, and
+		# without this it inherits a count already over the limit and halts immediately.
+		printf '0\n' > "$ATTEMPTS"
+	else
+		say "rollback failed. halting rather than restarting into the same failure."
+		printf 'rollback failed\n' > "$HALTED"
+		exit 0
+	fi
+fi
+
+# exec, so the host is what the supervisor watches and signals reach it directly.
+exec "$HOST" run

packaging/nox-mesh-host-rollback

@@ -59,8 +59,7 @@ if ! pacman -U --noconfirm "$PKG"; then
 	exit 1
 fi
 
-# reset-failed first, or the start limit that brought us here is still in force.
+# Deliberately does NOT start anything. The launcher called this and will exec the host next,
-systemctl reset-failed "$PACKAGE".service 2>/dev/null || true
+# so starting it here would run two. novox/hq ADR 0061 moved that responsibility; this script
-systemctl start "$PACKAGE".service
+# installs a version and says so, and nothing else.
-
+say "rolled back to $VERSION. the launcher will start it."
-say "rolled back to $VERSION and started it. the node is on the previous version."

packaging/nox-mesh-host-rollback.service

@@ -1,8 +0,0 @@
-[Unit]
-Description=Roll the Novox Mesh node host back to the last version that started
-# No OnFailure of its own. If the rollback fails there is nothing further to try
-# automatically, and the node needs a person.
-
-[Service]
-Type=oneshot
-ExecStart=/usr/lib/nox-mesh-host/rollback

packaging/nox-mesh-host.service

@@ -2,19 +2,16 @@
 Description=Novox Mesh node host
 After=network-online.target
 Wants=network-online.target
-# When the supervisor gives up, recover rather than leaving the node quiet — a host that will
-# not start looks exactly like a machine somebody switched off (novox/hq ADR 0059).
-OnFailure=nox-mesh-host-rollback.service
 
+# Two lines of policy and no more (novox/hq ADR 0061). Counting failed starts and rolling back
+# lives in the launcher, where it can be tested — so this file is transcription for any other
+# init rather than design.
 [Service]
-Type=notify
+ExecStart=/usr/lib/nox-mesh-host/launch
-ExecStart=/usr/bin/nox-mesh-host run
 # always, NOT on-failure: the host restarts onto a new binary by exiting CLEANLY
-# (novox/hq ADR 0057), and on-failure would leave an upgraded node stopped.
+# (novox/hq ADR 0057), and on-failure would leave every upgraded node stopped.
 Restart=always
 RestartSec=5s
-StartLimitBurst=3
-StartLimitIntervalSec=120
 StateDirectory=mesh-host
 
 [Install]

packaging/rollback_test.sh

@@ -46,10 +46,10 @@ touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst"
 "$SCRIPT" >/dev/null 2>&1
 check "installs the known-good version" "pacman is asked to install the cached package" \
 	"$(grep -c 'nox-mesh-host-1.4.2' "$MESH_HOST_STATE_DIR/pacman.calls" 2>/dev/null || echo 0)" "1"
-check "resets the start limit before starting" "reset-failed precedes start" \
+# It installs and stops. The launcher execs the host next, and starting it here would run two
-	"$(head -1 "$MESH_HOST_STATE_DIR/systemctl.calls" | cut -d' ' -f1)" "reset-failed"
+# (novox/hq ADR 0061).
-check "starts the host again" "systemctl start is called" \
+check "does not start anything itself" "the launcher owns starting" \
-	"$(grep -c '^start ' "$MESH_HOST_STATE_DIR/systemctl.calls" 2>/dev/null || echo 0)" "1"
+	"$([ -f "$MESH_HOST_STATE_DIR/systemctl.calls" ] && echo started || echo not-started)" "not-started"
 check "records that it rolled back" "the attempted marker holds the version" \
 	"$(cat "$MESH_HOST_STATE_DIR/rollback-attempted" 2>/dev/null || echo MISSING)" "1.4.2"
 
@@ -90,9 +90,7 @@ echo "1.4.2" > "$MESH_HOST_STATE_DIR/known-good"
 touch "$MESH_HOST_PKG_CACHE/nox-mesh-host-1.4.2-1-x86_64.pkg.tar.zst"
 STUB_PACMAN_FAILS=1 ; export STUB_PACMAN_FAILS
 set +e; "$SCRIPT" >/dev/null 2>&1; RC=$?; set -e
-check "pacman fails: does not start the host" "starting the broken binary again would loop" \
+check "pacman fails: exits non-zero" "a failed rollback is a failure the launcher must see" "$RC" "1"
-	"$([ -f "$MESH_HOST_STATE_DIR/systemctl.calls" ] && echo started || echo not-started)" "not-started"
-check "pacman fails: exits non-zero" "a failed rollback is a failure" "$RC" "1"
 
 printf '\nrollback: %d passed, %d failed\n' "$PASS" "$FAIL"
 [ "$FAIL" -eq 0 ]