diff --git a/node-tools/cmd/node-tools/main.go b/node-tools/cmd/node-tools/main.go index 816b270..f053f93 100644 --- a/node-tools/cmd/node-tools/main.go +++ b/node-tools/cmd/node-tools/main.go @@ -26,6 +26,7 @@ import ( "github.com/novox/mesh-tools/node-tools/internal/alive" "github.com/novox/mesh-tools/node-tools/internal/bus" "github.com/novox/mesh-tools/node-tools/internal/console" + "github.com/novox/mesh-tools/node-tools/internal/launch" "github.com/novox/mesh-tools/node-tools/internal/provide" "github.com/novox/mesh-tools/node-tools/internal/runtime" ) @@ -91,6 +92,9 @@ func main() { signals := make(chan os.Signal, 1) signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT) <-signals + // First of all: the service manager signalled every bundle with this process, and none is started again + // (issue 1126). + launch.ShutDown() stopSaying() stopProviding() stop() diff --git a/node-tools/internal/launch/error_code_test.go b/node-tools/internal/launch/error_code_test.go index ace06ec..7002f46 100644 --- a/node-tools/internal/launch/error_code_test.go +++ b/node-tools/internal/launch/error_code_test.go @@ -3,7 +3,12 @@ package launch import ( "errors" "fmt" + "os" + "path/filepath" + "strings" + "syscall" "testing" + "time" ) type lostWrite struct{} @@ -28,3 +33,74 @@ func TestALostWriteIsAnsweredWithItsOwnCodeAndNothingElseIs(t *testing.T) { t.Errorf("a plain refusal is answered %d", got) } } + +// A bundle that answers the handshake and an empty tool list, and notes each start and its pid in $OUT. +const notingBundle = `#!/bin/sh +printf 'started %s\n' "$$" >> "$OUT" +while IFS= read -r line; do + id=$(printf '%s' "$line" | sed -n 's/.*"id":\([0-9]*\)[,}].*/\1/p') + case "$line" in + *'"method":"initialize"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{}}\n' "$id" ;; + *'"method":"tools/list"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{"tools":[]}}\n' "$id" ;; + esac +done +` + +// The service manager stops the runtime by signalling its whole group, every bundle with it (issue 1126, ace's +// reboot of 2026-10-28: every bundle "exited (SIGTERM); started again in 500ms" for the 15 s the runtime took +// to stop, the power bundle among them dialling the system bus again, which restarted the bus during the +// shutdown and held it 29 minutes). A bundle that exits while it serves is started again; once the runtime +// itself is stopping, none is, by a restart or by a call. +func TestABundleIsStartedAgainUnlessTheRuntimeIsStopping(t *testing.T) { + saved := RestartFirst + RestartFirst = 50 * time.Millisecond + t.Cleanup(func() { RestartFirst = saved; shuttingDown.Store(false) }) + for _, row := range []struct { + name string + stopping bool + starts int + }{ + {"a bundle that dies while the runtime serves", false, 2}, + {"a bundle signalled with the stopping runtime", true, 1}, + } { + t.Run(row.name, func(t *testing.T) { + shuttingDown.Store(false) + dir := t.TempDir() + entry, out := filepath.Join(dir, "bundle"), filepath.Join(dir, "out") + if err := os.WriteFile(entry, []byte(notingBundle), 0o755); err != nil { + t.Fatal(err) + } + l, err := Start("noting", entry, append(os.Environ(), "OUT="+out), &subscribing{}, t.Logf) + if err != nil { + t.Fatal(err) + } + t.Cleanup(l.Stop) + said, _ := os.ReadFile(out) + var pid int + if _, err := fmt.Sscanf(string(said), "started %d", &pid); err != nil { + t.Fatalf("the bundle noted %q", said) + } + if row.stopping { + ShutDown() + } + if err := syscall.Kill(pid, syscall.SIGTERM); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(2 * time.Second) + for { + said, _ = os.ReadFile(out) + n := strings.Count(string(said), "started") + if n > row.starts { + t.Fatalf("started %d times, want %d", n, row.starts) + } + if time.Now().After(deadline) { + if n != row.starts { + t.Fatalf("started %d times, want %d", n, row.starts) + } + break + } + time.Sleep(20 * time.Millisecond) + } + }) + } +} diff --git a/node-tools/internal/launch/launch.go b/node-tools/internal/launch/launch.go index 6492a39..8167b2c 100644 --- a/node-tools/internal/launch/launch.go +++ b/node-tools/internal/launch/launch.go @@ -15,6 +15,7 @@ import ( "regexp" "strings" "sync" + "sync/atomic" "syscall" "time" @@ -112,6 +113,17 @@ var ( RestartSettle = time.Minute ) +// shuttingDown is set once the runtime itself is stopping (ShutDown): from then on no bundle is started again. +var shuttingDown atomic.Bool + +// ShutDown says the runtime itself is stopping, before it stops its bundles one by one. The service manager stops +// the runtime by signalling its whole group, so every bundle receives SIGTERM with it, and a bundle's exit must not +// be taken for a fault: started again, it would run on through the machine's shutdown with nothing left to stop it +// (issue 1126: on ace's reboot of 2026-10-28 the restarted bundles dialled the system bus again and started it +// during the shutdown, which then waited 29 minutes). After ShutDown no bundle is started again, by a restart or a +// call. +func ShutDown() { shuttingDown.Store(true) } + // StopWait bounds how long Launched.Stop waits for its bundle's exit to be handled; a bundle that // ignores SIGTERM longer than this is left to exit on its own (novox/hq issue 476). var StopWait = 5 * time.Second @@ -444,7 +456,10 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an pause = pauseFirst } mu.Unlock() - if !wasStopped && wasServing { + if wasServing && !wasStopped && shuttingDown.Load() { + logf("[mesh-tools] %s; not started again: the runtime is stopping", why) + } + if !wasStopped && wasServing && !shuttingDown.Load() { // Every bundle stays up (ADR 0198): code that runs long — a handler, a provisioner — // is not waiting for a call to bring it back, and a tool bundle back early costs nothing. logf("[mesh-tools] %s; started again in %s", why, wait.Round(100*time.Millisecond)) @@ -470,6 +485,9 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an return nil, fmt.Errorf("%s's bundle is stopped; it answers no call", module) } if c == nil { + if shuttingDown.Load() { + return nil, fmt.Errorf("%s's bundle is not started again: the runtime is stopping", module) + } fresh, err := start() if err != nil { return nil, err @@ -497,7 +515,7 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an go func() { time.Sleep(after) mu.Lock() - if stopped || current != nil { + if stopped || current != nil || shuttingDown.Load() { mu.Unlock() return } @@ -506,7 +524,7 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an mu.Lock() defer mu.Unlock() if err != nil { - if !stopped { + if !stopped && !shuttingDown.Load() { wait := pause if pause *= 2; pause > pauseMost { pause = pauseMost