mesh-host/packaging/nox-mesh-host-launch

#!/bin/sh
# Supervise the host: start it, watch it, and decide what to do when it stops.
#
# novox/hq ADR 0005. The init is asked for ONE thing — run this at boot — and everything else
# lives here, in a script that can be tested. Whether to restart, how long to wait, when to give
# up, when to roll back: all of it is policy, and policy in a unit file can only be read and
# hoped for.
#
# It does NOT exec the host. Exec would replace this process, and then only the init could
# restart anything — which is the arrangement this exists to remove. The cost of staying is
# signal handling, below.
#
# POSIX sh. `set -e` is deliberately absent: this script's whole job is to inspect exit codes,
# and -e would make it exit on the first one it is meant to handle.
set -u

STATE_DIR="${MESH_HOST_STATE_DIR:-/var/lib/mesh-host}"
LIBEXEC="${MESH_HOST_LIBEXEC:-/usr/lib/nox-mesh-host}"
# The host that was placed by hand, used only when nothing has been delivered. The first host on a
# machine always arrives this way; every one after it is delivered (novox/hq ADR 0141).
FALLBACK="${MESH_HOST_BIN:-/usr/bin/nox-mesh-host}"
VERSIONS="$LIBEXEC/versions"
BINARY="nox-mesh-host"
LIMIT="${MESH_HOST_START_LIMIT:-3}"
BACKOFF="${MESH_HOST_BACKOFF:-5}"
ONCE="${MESH_HOST_RUN_ONCE:-}"   # tests run one iteration; nothing else sets this

ATTEMPTS="$STATE_DIR/start-attempts"
HALTED="$STATE_DIR/halted"
PINNED="$STATE_DIR/rollback-pinned"

say() { echo "nox-mesh-host-launch: $*" >&2; }

# Which host to run: the version a rollback pinned, or the most recently delivered one, or the one
# placed by hand when nothing has been delivered (novox/hq ADR 0141).
#
# **Asked every time round the loop, not once.** Standing aside for a successor is a clean exit, and
# the next turn has to run what is on disk NOW — resolving this once would restart the same binary
# for ever and the upgrade would never take.
#
# Newest by when it arrived, never by how its name sorts: a version string is whatever the source was
# described as, and those do not sort — "1.10" orders before "1.9". Ordering by name would start an
# older host and call it an upgrade.
pick_host() {
	if [ -s "$PINNED" ]; then
		pinned="$(tr -d '[:space:]' < "$PINNED" 2>/dev/null || true)"
		if [ -n "$pinned" ] && [ -x "$VERSIONS/$pinned/$BINARY" ]; then
			echo "$VERSIONS/$pinned/$BINARY"
			return 0
		fi
		say "the pinned version '$pinned' is not delivered; ignoring the pin"
	fi
	# A directory with no executable in it is not a version: an interrupted delivery leaves one, and
	# running "the newest" would then mean running nothing.
	for candidate in $(ls -1t "$VERSIONS" 2>/dev/null || true); do
		if [ -x "$VERSIONS/$candidate/$BINARY" ]; then
			echo "$VERSIONS/$candidate/$BINARY"
			return 0
		fi
	done
	echo "$FALLBACK"
}

child=
stopping=

# The machine is shutting down. Pass it on and wait for the host to finish — a supervisor that
# exits while its child is still running leaves the host to be killed rather than to stop, and
# an apply interrupted that way is exactly the half-configured machine this project is about.
on_term() {
	stopping=yes
	if [ -n "$child" ]; then
		say "stopping: passing the signal to the host"
		kill -TERM "$child" 2>/dev/null
	fi
}
trap on_term TERM INT

mkdir -p "$STATE_DIR"

while :; do
	if [ -n "$stopping" ]; then
		exit 0
	fi

	if [ -e "$HALTED" ]; then
		say "halted: $(cat "$HALTED" 2>/dev/null || echo 'reason not recorded')"
		say "not starting the host. this node needs a person."
		exit 0
	fi

	# Consecutive failed starts, not starts. Cleared by the host itself when it completes a
	# reconcile, which is the only evidence either this or known-good has.
	#
	# Read the FIRST FIELD, then insist it is a plain integer.
	#
	# Stripping whitespace instead concatenates, and that is not hypothetical: a counter
	# holding "1 2" became "12", past the limit, so a healthy node rolled itself back. An
	# unreadable counter must fail towards "start normally", never towards "give up".
	count=0
	if [ -s "$ATTEMPTS" ]; then
		read -r count _ < "$ATTEMPTS" 2>/dev/null || count=0
	fi
	case "${count:-}" in
		'' | *[!0-9]*) count=0 ;;
	esac

	if [ "$count" -ge "$LIMIT" ]; then
		if [ -e "$STATE_DIR/rollback-attempted" ]; then
			say "the host failed $count times after a rollback. the previous version does not"
			say "start either, so this is the machine and not the binary."
			printf 'rolled back and still failing\n' > "$HALTED"
			exit 0
		fi

		say "the host failed $count times. rolling back."
		if "$LIBEXEC/rollback"; then
			# Fresh count for the version just installed: it deserves its own attempts, and
			# without this it inherits a count already over the limit and halts at once.
			#
			# The variable too, not only the file. Resetting one and not the other made the
			# next failure count from the OLD value — so the rolled-back version got one
			# attempt instead of three.
			count=0
			printf '%s\n' "$count" > "$ATTEMPTS"
		else
			say "rollback failed. halting rather than restarting into the same failure."
			printf 'rollback failed\n' > "$HALTED"
			exit 0
		fi
	fi

	HOST="$(pick_host)"
	if [ ! -x "$HOST" ]; then
		say "no host to run: nothing delivered under $VERSIONS and $FALLBACK is not executable."
		printf 'no host binary\n' > "$HALTED"
		exit 0
	fi
	say "running $HOST"

	"$HOST" run &
	child=$!
	status=0
	wait "$child" || status=$?
	child=

	if [ -n "$stopping" ]; then
		exit 0
	fi

	# A signal the host did not survive, and we are not shutting down: treat it as a crash.
	case "$status" in
		0)
			# Exited cleanly. That is how the host stands aside for a new binary after an
			# upgrade (novox/hq ADR 0005) — so loop and run whatever is now on disk.
			#
			# Deliberately NOT counted, and this is the whole reason the counter is
			# incremented here rather than before the start: counting attempts meant a host
			# that upgraded itself three times rolled itself back, having worked perfectly
			# every time.
			say "the host exited cleanly; starting it again"
			continue
			;;
	esac

	count=$((count + 1))
	printf '%s\n' "$count" > "$ATTEMPTS"

	say "the host exited $status ($count consecutive); restarting in ${BACKOFF}s"
	[ -n "$ONCE" ] && exit "$status"
	sleep "$BACKOFF"
done