From f016c67a72c465c3f799b6f05a0c27a07858262c Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 28 Oct 2026 15:30:12 +0100 Subject: [PATCH 1/2] node-tools: start no bundle again once the runtime itself is stopping (issue 1126) The service manager stops node-tools by signalling its whole group, so every bundle gets SIGTERM with it; the launcher took each exit for a fault and started the bundle again, for the 15 s the runtime took to stop. On ace's reboot of 2026-10-28 those restarted bundles dialled the system bus once polkit and the bus had stopped, the bus and polkit came back during the shutdown, nothing stopped them a second time, and the reboot waited 29 minutes on its job timeout. ShutDown, called first on SIGTERM, keeps any bundle from starting again, by a restart or by a call; a running one still answers. --- node-tools/cmd/node-tools/main.go | 4 + node-tools/internal/launch/error_code_test.go | 76 +++++++++++++++++++ node-tools/internal/launch/launch.go | 24 +++++- 3 files changed, 101 insertions(+), 3 deletions(-) diff --git a/node-tools/cmd/node-tools/main.go b/node-tools/cmd/node-tools/main.go index 816b270..f053f93 100644 --- a/node-tools/cmd/node-tools/main.go +++ b/node-tools/cmd/node-tools/main.go @@ -26,6 +26,7 @@ import ( "github.com/novox/mesh-tools/node-tools/internal/alive" "github.com/novox/mesh-tools/node-tools/internal/bus" "github.com/novox/mesh-tools/node-tools/internal/console" + "github.com/novox/mesh-tools/node-tools/internal/launch" "github.com/novox/mesh-tools/node-tools/internal/provide" "github.com/novox/mesh-tools/node-tools/internal/runtime" ) @@ -91,6 +92,9 @@ func main() { signals := make(chan os.Signal, 1) signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT) <-signals + // First of all: the service manager signalled every bundle with this process, and none is started again + // (issue 1126). + launch.ShutDown() stopSaying() stopProviding() stop() diff --git a/node-tools/internal/launch/error_code_test.go b/node-tools/internal/launch/error_code_test.go index ace06ec..7002f46 100644 --- a/node-tools/internal/launch/error_code_test.go +++ b/node-tools/internal/launch/error_code_test.go @@ -3,7 +3,12 @@ package launch import ( "errors" "fmt" + "os" + "path/filepath" + "strings" + "syscall" "testing" + "time" ) type lostWrite struct{} @@ -28,3 +33,74 @@ func TestALostWriteIsAnsweredWithItsOwnCodeAndNothingElseIs(t *testing.T) { t.Errorf("a plain refusal is answered %d", got) } } + +// A bundle that answers the handshake and an empty tool list, and notes each start and its pid in $OUT. +const notingBundle = `#!/bin/sh +printf 'started %s\n' "$$" >> "$OUT" +while IFS= read -r line; do + id=$(printf '%s' "$line" | sed -n 's/.*"id":\([0-9]*\)[,}].*/\1/p') + case "$line" in + *'"method":"initialize"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{}}\n' "$id" ;; + *'"method":"tools/list"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{"tools":[]}}\n' "$id" ;; + esac +done +` + +// The service manager stops the runtime by signalling its whole group, every bundle with it (issue 1126, ace's +// reboot of 2026-10-28: every bundle "exited (SIGTERM); started again in 500ms" for the 15 s the runtime took +// to stop, the power bundle among them dialling the system bus again, which restarted the bus during the +// shutdown and held it 29 minutes). A bundle that exits while it serves is started again; once the runtime +// itself is stopping, none is, by a restart or by a call. +func TestABundleIsStartedAgainUnlessTheRuntimeIsStopping(t *testing.T) { + saved := RestartFirst + RestartFirst = 50 * time.Millisecond + t.Cleanup(func() { RestartFirst = saved; shuttingDown.Store(false) }) + for _, row := range []struct { + name string + stopping bool + starts int + }{ + {"a bundle that dies while the runtime serves", false, 2}, + {"a bundle signalled with the stopping runtime", true, 1}, + } { + t.Run(row.name, func(t *testing.T) { + shuttingDown.Store(false) + dir := t.TempDir() + entry, out := filepath.Join(dir, "bundle"), filepath.Join(dir, "out") + if err := os.WriteFile(entry, []byte(notingBundle), 0o755); err != nil { + t.Fatal(err) + } + l, err := Start("noting", entry, append(os.Environ(), "OUT="+out), &subscribing{}, t.Logf) + if err != nil { + t.Fatal(err) + } + t.Cleanup(l.Stop) + said, _ := os.ReadFile(out) + var pid int + if _, err := fmt.Sscanf(string(said), "started %d", &pid); err != nil { + t.Fatalf("the bundle noted %q", said) + } + if row.stopping { + ShutDown() + } + if err := syscall.Kill(pid, syscall.SIGTERM); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(2 * time.Second) + for { + said, _ = os.ReadFile(out) + n := strings.Count(string(said), "started") + if n > row.starts { + t.Fatalf("started %d times, want %d", n, row.starts) + } + if time.Now().After(deadline) { + if n != row.starts { + t.Fatalf("started %d times, want %d", n, row.starts) + } + break + } + time.Sleep(20 * time.Millisecond) + } + }) + } +} diff --git a/node-tools/internal/launch/launch.go b/node-tools/internal/launch/launch.go index 6492a39..8167b2c 100644 --- a/node-tools/internal/launch/launch.go +++ b/node-tools/internal/launch/launch.go @@ -15,6 +15,7 @@ import ( "regexp" "strings" "sync" + "sync/atomic" "syscall" "time" @@ -112,6 +113,17 @@ var ( RestartSettle = time.Minute ) +// shuttingDown is set once the runtime itself is stopping (ShutDown): from then on no bundle is started again. +var shuttingDown atomic.Bool + +// ShutDown says the runtime itself is stopping, before it stops its bundles one by one. The service manager stops +// the runtime by signalling its whole group, so every bundle receives SIGTERM with it, and a bundle's exit must not +// be taken for a fault: started again, it would run on through the machine's shutdown with nothing left to stop it +// (issue 1126: on ace's reboot of 2026-10-28 the restarted bundles dialled the system bus again and started it +// during the shutdown, which then waited 29 minutes). After ShutDown no bundle is started again, by a restart or a +// call. +func ShutDown() { shuttingDown.Store(true) } + // StopWait bounds how long Launched.Stop waits for its bundle's exit to be handled; a bundle that // ignores SIGTERM longer than this is left to exit on its own (novox/hq issue 476). var StopWait = 5 * time.Second @@ -444,7 +456,10 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an pause = pauseFirst } mu.Unlock() - if !wasStopped && wasServing { + if wasServing && !wasStopped && shuttingDown.Load() { + logf("[mesh-tools] %s; not started again: the runtime is stopping", why) + } + if !wasStopped && wasServing && !shuttingDown.Load() { // Every bundle stays up (ADR 0198): code that runs long — a handler, a provisioner — // is not waiting for a call to bring it back, and a tool bundle back early costs nothing. logf("[mesh-tools] %s; started again in %s", why, wait.Round(100*time.Millisecond)) @@ -470,6 +485,9 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an return nil, fmt.Errorf("%s's bundle is stopped; it answers no call", module) } if c == nil { + if shuttingDown.Load() { + return nil, fmt.Errorf("%s's bundle is not started again: the runtime is stopping", module) + } fresh, err := start() if err != nil { return nil, err @@ -497,7 +515,7 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an go func() { time.Sleep(after) mu.Lock() - if stopped || current != nil { + if stopped || current != nil || shuttingDown.Load() { mu.Unlock() return } @@ -506,7 +524,7 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an mu.Lock() defer mu.Unlock() if err != nil { - if !stopped { + if !stopped && !shuttingDown.Load() { wait := pause if pause *= 2; pause > pauseMost { pause = pauseMost From a9b02bc67604fffd906146c12260d2bed90928d4 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 28 Oct 2026 15:40:39 +0100 Subject: [PATCH 2/2] node-tools: keep the stopping-runtime test under a second, and start no first bundle once stopping The test watched two seconds per row; a 10 ms restart pause and a 250 ms window prove the same. launch.Start itself now refuses while the runtime stops, so a reload's first start is held too (issue 1126). --- node-tools/internal/launch/error_code_test.go | 16 +++++++++++++--- node-tools/internal/launch/launch.go | 3 +++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/node-tools/internal/launch/error_code_test.go b/node-tools/internal/launch/error_code_test.go index 7002f46..3b3e1b2 100644 --- a/node-tools/internal/launch/error_code_test.go +++ b/node-tools/internal/launch/error_code_test.go @@ -53,7 +53,7 @@ done // itself is stopping, none is, by a restart or by a call. func TestABundleIsStartedAgainUnlessTheRuntimeIsStopping(t *testing.T) { saved := RestartFirst - RestartFirst = 50 * time.Millisecond + RestartFirst = 10 * time.Millisecond t.Cleanup(func() { RestartFirst = saved; shuttingDown.Store(false) }) for _, row := range []struct { name string @@ -86,20 +86,30 @@ func TestABundleIsStartedAgainUnlessTheRuntimeIsStopping(t *testing.T) { if err := syscall.Kill(pid, syscall.SIGTERM); err != nil { t.Fatal(err) } - deadline := time.Now().Add(2 * time.Second) + // The restart row ends at its second start; the stopping row watches a window many restart pauses long. + deadline := time.Now().Add(250 * time.Millisecond) for { said, _ = os.ReadFile(out) n := strings.Count(string(said), "started") if n > row.starts { t.Fatalf("started %d times, want %d", n, row.starts) } + if n == row.starts && !row.stopping { + break + } if time.Now().After(deadline) { if n != row.starts { t.Fatalf("started %d times, want %d", n, row.starts) } break } - time.Sleep(20 * time.Millisecond) + time.Sleep(5 * time.Millisecond) + } + if row.stopping { + // And a bundle first loaded now (a reload) is not started either. + if _, err := Start("late", entry, append(os.Environ(), "OUT="+out), &subscribing{}, t.Logf); err == nil { + t.Error("a bundle was started after the runtime began stopping") + } } }) } diff --git a/node-tools/internal/launch/launch.go b/node-tools/internal/launch/launch.go index 8167b2c..fecb1c2 100644 --- a/node-tools/internal/launch/launch.go +++ b/node-tools/internal/launch/launch.go @@ -194,6 +194,9 @@ var colour = regexp.MustCompile("\x1b\\[[0-9;]*m") // Start launches one bundle and learns its tools. It fails when the child cannot be started or does // not complete the handshake. A child that exits later is started again on its next call. func Start(module, entry string, env []string, mesh Bus, logf func(string, ...any)) (*Launched, error) { + if shuttingDown.Load() { + return nil, fmt.Errorf("%s's bundle is not started: the runtime is stopping", module) + } var mu sync.Mutex var current *child // The child that last subscribed is the one events are handed to: a restarted child subscribes