node-tools: start no bundle again once the runtime itself is stopping (issue 1126)
The service manager stops node-tools by signalling its whole group, so every bundle gets SIGTERM with it; the launcher took each exit for a fault and started the bundle again, for the 15 s the runtime took to stop. On ace's reboot of 2026-10-28 those restarted bundles dialled the system bus once polkit and the bus had stopped, the bus and polkit came back during the shutdown, nothing stopped them a second time, and the reboot waited 29 minutes on its job timeout. ShutDown, called first on SIGTERM, keeps any bundle from starting again, by a restart or by a call; a running one still answers.
This commit is contained in:
@@ -15,6 +15,7 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -112,6 +113,17 @@ var (
|
||||
RestartSettle = time.Minute
|
||||
)
|
||||
|
||||
// shuttingDown is set once the runtime itself is stopping (ShutDown): from then on no bundle is started again.
|
||||
var shuttingDown atomic.Bool
|
||||
|
||||
// ShutDown says the runtime itself is stopping, before it stops its bundles one by one. The service manager stops
|
||||
// the runtime by signalling its whole group, so every bundle receives SIGTERM with it, and a bundle's exit must not
|
||||
// be taken for a fault: started again, it would run on through the machine's shutdown with nothing left to stop it
|
||||
// (issue 1126: on ace's reboot of 2026-10-28 the restarted bundles dialled the system bus again and started it
|
||||
// during the shutdown, which then waited 29 minutes). After ShutDown no bundle is started again, by a restart or a
|
||||
// call.
|
||||
func ShutDown() { shuttingDown.Store(true) }
|
||||
|
||||
// StopWait bounds how long Launched.Stop waits for its bundle's exit to be handled; a bundle that
|
||||
// ignores SIGTERM longer than this is left to exit on its own (novox/hq issue 476).
|
||||
var StopWait = 5 * time.Second
|
||||
@@ -444,7 +456,10 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
pause = pauseFirst
|
||||
}
|
||||
mu.Unlock()
|
||||
if !wasStopped && wasServing {
|
||||
if wasServing && !wasStopped && shuttingDown.Load() {
|
||||
logf("[mesh-tools] %s; not started again: the runtime is stopping", why)
|
||||
}
|
||||
if !wasStopped && wasServing && !shuttingDown.Load() {
|
||||
// Every bundle stays up (ADR 0198): code that runs long — a handler, a provisioner —
|
||||
// is not waiting for a call to bring it back, and a tool bundle back early costs nothing.
|
||||
logf("[mesh-tools] %s; started again in %s", why, wait.Round(100*time.Millisecond))
|
||||
@@ -470,6 +485,9 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
return nil, fmt.Errorf("%s's bundle is stopped; it answers no call", module)
|
||||
}
|
||||
if c == nil {
|
||||
if shuttingDown.Load() {
|
||||
return nil, fmt.Errorf("%s's bundle is not started again: the runtime is stopping", module)
|
||||
}
|
||||
fresh, err := start()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -497,7 +515,7 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
go func() {
|
||||
time.Sleep(after)
|
||||
mu.Lock()
|
||||
if stopped || current != nil {
|
||||
if stopped || current != nil || shuttingDown.Load() {
|
||||
mu.Unlock()
|
||||
return
|
||||
}
|
||||
@@ -506,7 +524,7 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if err != nil {
|
||||
if !stopped {
|
||||
if !stopped && !shuttingDown.Load() {
|
||||
wait := pause
|
||||
if pause *= 2; pause > pauseMost {
|
||||
pause = pauseMost
|
||||
|
||||
Reference in New Issue
Block a user