jschoubben
b8a766f234
novox/hq ADR 0141 and 04-ISSUES/142. A version was being delivered to a machine and nothing started it: the launcher on these machines predates the versions mechanism and runs the fixed binary path, so the delivery was correct and inert. Delivered as a FILE resource, not as part of an archive, and the difference is the whole reason this is safe. A file is written atomically — temp file in the same directory, then rename — so the running launcher keeps the inode it was started from and the next start picks up the new one. An archive writes in place with truncate, which would cut the file a running shell is reading halfway through. The manifest therefore carries a second copy of the script, and a test refuses any difference between it and packaging/nox-mesh-host-launch. Proven by drifting one and watching it fail. Two copies of a script is a bad thing to accept, and the alternative was writing over a running supervisor. Together the two resources complete the loop: the version lands, the running host stands aside because it sees one delivered, and the launcher that starts next is the one that looks in versions/ and picks the newest by arrival.
33 lines
7.2 KiB
JSON
33 lines
7.2 KiB
JSON
{
|
|
"module": "mesh-host",
|
|
"version": "1",
|
|
"slug": "host",
|
|
"build": {
|
|
"artifacts": [
|
|
{
|
|
"name": "host-arch",
|
|
"kind": "bundle",
|
|
"language": "go",
|
|
"system": "arch",
|
|
"from": "cmd/mesh-host",
|
|
"binary": "nox-mesh-host"
|
|
}
|
|
]
|
|
},
|
|
"resources": [
|
|
{
|
|
"id": "launcher",
|
|
"type": "file",
|
|
"path": "/usr/lib/nox-mesh-host/launch",
|
|
"mode": "0755",
|
|
"content": "#!/bin/sh\n# Supervise the host: start it, watch it, and decide what to do when it stops.\n#\n# novox/hq ADR 0005. The init is asked for ONE thing \u2014 run this at boot \u2014 and everything else\n# lives here, in a script that can be tested. Whether to restart, how long to wait, when to give\n# up, when to roll back: all of it is policy, and policy in a unit file can only be read and\n# hoped for.\n#\n# It does NOT exec the host. Exec would replace this process, and then only the init could\n# restart anything \u2014 which is the arrangement this exists to remove. The cost of staying is\n# signal handling, below.\n#\n# POSIX sh. `set -e` is deliberately absent: this script's whole job is to inspect exit codes,\n# and -e would make it exit on the first one it is meant to handle.\nset -u\n\nSTATE_DIR=\"${MESH_HOST_STATE_DIR:-/var/lib/mesh-host}\"\nLIBEXEC=\"${MESH_HOST_LIBEXEC:-/usr/lib/nox-mesh-host}\"\n# The host that was placed by hand, used only when nothing has been delivered. The first host on a\n# machine always arrives this way; every one after it is delivered (novox/hq ADR 0141).\nFALLBACK=\"${MESH_HOST_BIN:-/usr/bin/nox-mesh-host}\"\nVERSIONS=\"$LIBEXEC/versions\"\nBINARY=\"nox-mesh-host\"\nLIMIT=\"${MESH_HOST_START_LIMIT:-3}\"\nBACKOFF=\"${MESH_HOST_BACKOFF:-5}\"\nONCE=\"${MESH_HOST_RUN_ONCE:-}\" # tests run one iteration; nothing else sets this\n\nATTEMPTS=\"$STATE_DIR/start-attempts\"\nHALTED=\"$STATE_DIR/halted\"\nPINNED=\"$STATE_DIR/rollback-pinned\"\n\nsay() { echo \"nox-mesh-host-launch: $*\" >&2; }\n\n# Which host to run: the version a rollback pinned, or the most recently delivered one, or the one\n# placed by hand when nothing has been delivered (novox/hq ADR 0141).\n#\n# **Asked every time round the loop, not once.** Standing aside for a successor is a clean exit, and\n# the next turn has to run what is on disk NOW \u2014 resolving this once would restart the same binary\n# for ever and the upgrade would never take.\n#\n# Newest by when it arrived, never by how its name sorts: a version string is whatever the source was\n# described as, and those do not sort \u2014 \"1.10\" orders before \"1.9\". Ordering by name would start an\n# older host and call it an upgrade.\npick_host() {\n\tif [ -s \"$PINNED\" ]; then\n\t\tpinned=\"$(tr -d '[:space:]' < \"$PINNED\" 2>/dev/null || true)\"\n\t\tif [ -n \"$pinned\" ] && [ -x \"$VERSIONS/$pinned/$BINARY\" ]; then\n\t\t\techo \"$VERSIONS/$pinned/$BINARY\"\n\t\t\treturn 0\n\t\tfi\n\t\tsay \"the pinned version '$pinned' is not delivered; ignoring the pin\"\n\tfi\n\t# A directory with no executable in it is not a version: an interrupted delivery leaves one, and\n\t# running \"the newest\" would then mean running nothing.\n\tfor candidate in $(ls -1t \"$VERSIONS\" 2>/dev/null || true); do\n\t\tif [ -x \"$VERSIONS/$candidate/$BINARY\" ]; then\n\t\t\techo \"$VERSIONS/$candidate/$BINARY\"\n\t\t\treturn 0\n\t\tfi\n\tdone\n\techo \"$FALLBACK\"\n}\n\nchild=\nstopping=\n\n# The machine is shutting down. Pass it on and wait for the host to finish \u2014 a supervisor that\n# exits while its child is still running leaves the host to be killed rather than to stop, and\n# an apply interrupted that way is exactly the half-configured machine this project is about.\non_term() {\n\tstopping=yes\n\tif [ -n \"$child\" ]; then\n\t\tsay \"stopping: passing the signal to the host\"\n\t\tkill -TERM \"$child\" 2>/dev/null\n\tfi\n}\ntrap on_term TERM INT\n\nmkdir -p \"$STATE_DIR\"\n\nwhile :; do\n\tif [ -n \"$stopping\" ]; then\n\t\texit 0\n\tfi\n\n\tif [ -e \"$HALTED\" ]; then\n\t\tsay \"halted: $(cat \"$HALTED\" 2>/dev/null || echo 'reason not recorded')\"\n\t\tsay \"not starting the host. this node needs a person.\"\n\t\texit 0\n\tfi\n\n\t# Consecutive failed starts, not starts. Cleared by the host itself when it completes a\n\t# reconcile, which is the only evidence either this or known-good has.\n\t#\n\t# Read the FIRST FIELD, then insist it is a plain integer.\n\t#\n\t# Stripping whitespace instead concatenates, and that is not hypothetical: a counter\n\t# holding \"1 2\" became \"12\", past the limit, so a healthy node rolled itself back. An\n\t# unreadable counter must fail towards \"start normally\", never towards \"give up\".\n\tcount=0\n\tif [ -s \"$ATTEMPTS\" ]; then\n\t\tread -r count _ < \"$ATTEMPTS\" 2>/dev/null || count=0\n\tfi\n\tcase \"${count:-}\" in\n\t\t'' | *[!0-9]*) count=0 ;;\n\tesac\n\n\tif [ \"$count\" -ge \"$LIMIT\" ]; then\n\t\tif [ -e \"$STATE_DIR/rollback-attempted\" ]; then\n\t\t\tsay \"the host failed $count times after a rollback. the previous version does not\"\n\t\t\tsay \"start either, so this is the machine and not the binary.\"\n\t\t\tprintf 'rolled back and still failing\\n' > \"$HALTED\"\n\t\t\texit 0\n\t\tfi\n\n\t\tsay \"the host failed $count times. rolling back.\"\n\t\tif \"$LIBEXEC/rollback\"; then\n\t\t\t# Fresh count for the version just installed: it deserves its own attempts, and\n\t\t\t# without this it inherits a count already over the limit and halts at once.\n\t\t\t#\n\t\t\t# The variable too, not only the file. Resetting one and not the other made the\n\t\t\t# next failure count from the OLD value \u2014 so the rolled-back version got one\n\t\t\t# attempt instead of three.\n\t\t\tcount=0\n\t\t\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\t\telse\n\t\t\tsay \"rollback failed. halting rather than restarting into the same failure.\"\n\t\t\tprintf 'rollback failed\\n' > \"$HALTED\"\n\t\t\texit 0\n\t\tfi\n\tfi\n\n\tHOST=\"$(pick_host)\"\n\tif [ ! -x \"$HOST\" ]; then\n\t\tsay \"no host to run: nothing delivered under $VERSIONS and $FALLBACK is not executable.\"\n\t\tprintf 'no host binary\\n' > \"$HALTED\"\n\t\texit 0\n\tfi\n\tsay \"running $HOST\"\n\n\t\"$HOST\" run &\n\tchild=$!\n\tstatus=0\n\twait \"$child\" || status=$?\n\tchild=\n\n\tif [ -n \"$stopping\" ]; then\n\t\texit 0\n\tfi\n\n\t# A signal the host did not survive, and we are not shutting down: treat it as a crash.\n\tcase \"$status\" in\n\t\t0)\n\t\t\t# Exited cleanly. That is how the host stands aside for a new binary after an\n\t\t\t# upgrade (novox/hq ADR 0005) \u2014 so loop and run whatever is now on disk.\n\t\t\t#\n\t\t\t# Deliberately NOT counted, and this is the whole reason the counter is\n\t\t\t# incremented here rather than before the start: counting attempts meant a host\n\t\t\t# that upgraded itself three times rolled itself back, having worked perfectly\n\t\t\t# every time.\n\t\t\tsay \"the host exited cleanly; starting it again\"\n\t\t\tcontinue\n\t\t\t;;\n\tesac\n\n\tcount=$((count + 1))\n\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\n\tsay \"the host exited $status ($count consecutive); restarting in ${BACKOFF}s\"\n\t[ -n \"$ONCE\" ] && exit \"$status\"\n\tsleep \"$BACKOFF\"\ndone\n"
|
|
},
|
|
{
|
|
"id": "next",
|
|
"type": "archive",
|
|
"artifact": "host-arch",
|
|
"path": "/usr/lib/nox-mesh-host/versions/${version}"
|
|
}
|
|
]
|
|
}
|