node-tools: start no bundle again once the runtime itself is stopping (issue 1126)

The service manager stops node-tools by signalling its whole group, so every
bundle gets SIGTERM with it; the launcher took each exit for a fault and started
the bundle again, for the 15 s the runtime took to stop. On ace's reboot of
2026-10-28 those restarted bundles dialled the system bus once polkit and the bus
had stopped, the bus and polkit came back during the shutdown, nothing stopped
them a second time, and the reboot waited 29 minutes on its job timeout.

ShutDown, called first on SIGTERM, keeps any bundle from starting again, by a
restart or by a call; a running one still answers.
This commit is contained in:
2026-10-28 15:30:12 +01:00
parent 6cc3398880
commit f016c67a72
3 changed files with 101 additions and 3 deletions
+21 -3
View File
@@ -15,6 +15,7 @@ import (
"regexp"
"strings"
"sync"
"sync/atomic"
"syscall"
"time"
@@ -112,6 +113,17 @@ var (
RestartSettle = time.Minute
)
// shuttingDown is set once the runtime itself is stopping (ShutDown): from then on no bundle is started again.
var shuttingDown atomic.Bool
// ShutDown says the runtime itself is stopping, before it stops its bundles one by one. The service manager stops
// the runtime by signalling its whole group, so every bundle receives SIGTERM with it, and a bundle's exit must not
// be taken for a fault: started again, it would run on through the machine's shutdown with nothing left to stop it
// (issue 1126: on ace's reboot of 2026-10-28 the restarted bundles dialled the system bus again and started it
// during the shutdown, which then waited 29 minutes). After ShutDown no bundle is started again, by a restart or a
// call.
func ShutDown() { shuttingDown.Store(true) }
// StopWait bounds how long Launched.Stop waits for its bundle's exit to be handled; a bundle that
// ignores SIGTERM longer than this is left to exit on its own (novox/hq issue 476).
var StopWait = 5 * time.Second
@@ -444,7 +456,10 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
pause = pauseFirst
}
mu.Unlock()
if !wasStopped && wasServing {
if wasServing && !wasStopped && shuttingDown.Load() {
logf("[mesh-tools] %s; not started again: the runtime is stopping", why)
}
if !wasStopped && wasServing && !shuttingDown.Load() {
// Every bundle stays up (ADR 0198): code that runs long — a handler, a provisioner —
// is not waiting for a call to bring it back, and a tool bundle back early costs nothing.
logf("[mesh-tools] %s; started again in %s", why, wait.Round(100*time.Millisecond))
@@ -470,6 +485,9 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
return nil, fmt.Errorf("%s's bundle is stopped; it answers no call", module)
}
if c == nil {
if shuttingDown.Load() {
return nil, fmt.Errorf("%s's bundle is not started again: the runtime is stopping", module)
}
fresh, err := start()
if err != nil {
return nil, err
@@ -497,7 +515,7 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
go func() {
time.Sleep(after)
mu.Lock()
if stopped || current != nil {
if stopped || current != nil || shuttingDown.Load() {
mu.Unlock()
return
}
@@ -506,7 +524,7 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
mu.Lock()
defer mu.Unlock()
if err != nil {
if !stopped {
if !stopped && !shuttingDown.Load() {
wait := pause
if pause *= 2; pause > pauseMost {
pause = pauseMost