Merge pull request 'node-tools starts no bundle again once it is stopping (group fix/shutdown-bound, issue 1126)' (#46) from fix/shutdown-bound into main

node-tools/cmd/node-tools/main.go

@@ -26,6 +26,7 @@ import (
 	"github.com/novox/mesh-tools/node-tools/internal/alive"
 	"github.com/novox/mesh-tools/node-tools/internal/bus"
 	"github.com/novox/mesh-tools/node-tools/internal/console"
+	"github.com/novox/mesh-tools/node-tools/internal/launch"
 	"github.com/novox/mesh-tools/node-tools/internal/provide"
 	"github.com/novox/mesh-tools/node-tools/internal/runtime"
 )
@@ -91,6 +92,9 @@ func main() {
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
 	<-signals
+	// First of all: the service manager signalled every bundle with this process, and none is started again
+	// (issue 1126).
+	launch.ShutDown()
 	stopSaying()
 	stopProviding()
 	stop()

node-tools/internal/launch/error_code_test.go

@@ -3,7 +3,12 @@ package launch
 import (
 	"errors"
 	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
 	"testing"
+	"time"
 )
 
 type lostWrite struct{}
@@ -28,3 +33,84 @@ func TestALostWriteIsAnsweredWithItsOwnCodeAndNothingElseIs(t *testing.T) {
 		t.Errorf("a plain refusal is answered %d", got)
 	}
 }
+
+// A bundle that answers the handshake and an empty tool list, and notes each start and its pid in $OUT.
+const notingBundle = `#!/bin/sh
+printf 'started %s\n' "$$" >> "$OUT"
+while IFS= read -r line; do
+  id=$(printf '%s' "$line" | sed -n 's/.*"id":\([0-9]*\)[,}].*/\1/p')
+  case "$line" in
+    *'"method":"initialize"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{}}\n' "$id" ;;
+    *'"method":"tools/list"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{"tools":[]}}\n' "$id" ;;
+  esac
+done
+`
+
+// The service manager stops the runtime by signalling its whole group, every bundle with it (issue 1126, ace's
+// reboot of 2026-10-28: every bundle "exited (SIGTERM); started again in 500ms" for the 15 s the runtime took
+// to stop, the power bundle among them dialling the system bus again, which restarted the bus during the
+// shutdown and held it 29 minutes). A bundle that exits while it serves is started again; once the runtime
+// itself is stopping, none is, by a restart or by a call.
+func TestABundleIsStartedAgainUnlessTheRuntimeIsStopping(t *testing.T) {
+	saved := RestartFirst
+	RestartFirst = 10 * time.Millisecond
+	t.Cleanup(func() { RestartFirst = saved; shuttingDown.Store(false) })
+	for _, row := range []struct {
+		name     string
+		stopping bool
+		starts   int
+	}{
+		{"a bundle that dies while the runtime serves", false, 2},
+		{"a bundle signalled with the stopping runtime", true, 1},
+	} {
+		t.Run(row.name, func(t *testing.T) {
+			shuttingDown.Store(false)
+			dir := t.TempDir()
+			entry, out := filepath.Join(dir, "bundle"), filepath.Join(dir, "out")
+			if err := os.WriteFile(entry, []byte(notingBundle), 0o755); err != nil {
+				t.Fatal(err)
+			}
+			l, err := Start("noting", entry, append(os.Environ(), "OUT="+out), &subscribing{}, t.Logf)
+			if err != nil {
+				t.Fatal(err)
+			}
+			t.Cleanup(l.Stop)
+			said, _ := os.ReadFile(out)
+			var pid int
+			if _, err := fmt.Sscanf(string(said), "started %d", &pid); err != nil {
+				t.Fatalf("the bundle noted %q", said)
+			}
+			if row.stopping {
+				ShutDown()
+			}
+			if err := syscall.Kill(pid, syscall.SIGTERM); err != nil {
+				t.Fatal(err)
+			}
+			// The restart row ends at its second start; the stopping row watches a window many restart pauses long.
+			deadline := time.Now().Add(250 * time.Millisecond)
+			for {
+				said, _ = os.ReadFile(out)
+				n := strings.Count(string(said), "started")
+				if n > row.starts {
+					t.Fatalf("started %d times, want %d", n, row.starts)
+				}
+				if n == row.starts && !row.stopping {
+					break
+				}
+				if time.Now().After(deadline) {
+					if n != row.starts {
+						t.Fatalf("started %d times, want %d", n, row.starts)
+					}
+					break
+				}
+				time.Sleep(5 * time.Millisecond)
+			}
+			if row.stopping {
+				// And a bundle first loaded now (a reload) is not started either.
+				if _, err := Start("late", entry, append(os.Environ(), "OUT="+out), &subscribing{}, t.Logf); err == nil {
+					t.Error("a bundle was started after the runtime began stopping")
+				}
+			}
+		})
+	}
+}

node-tools/internal/launch/launch.go

@@ -15,6 +15,7 @@ import (
 	"regexp"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"syscall"
 	"time"
 
@@ -112,6 +113,17 @@ var (
 	RestartSettle = time.Minute
 )
 
+// shuttingDown is set once the runtime itself is stopping (ShutDown): from then on no bundle is started again.
+var shuttingDown atomic.Bool
+
+// ShutDown says the runtime itself is stopping, before it stops its bundles one by one. The service manager stops
+// the runtime by signalling its whole group, so every bundle receives SIGTERM with it, and a bundle's exit must not
+// be taken for a fault: started again, it would run on through the machine's shutdown with nothing left to stop it
+// (issue 1126: on ace's reboot of 2026-10-28 the restarted bundles dialled the system bus again and started it
+// during the shutdown, which then waited 29 minutes). After ShutDown no bundle is started again, by a restart or a
+// call.
+func ShutDown() { shuttingDown.Store(true) }
+
 // StopWait bounds how long Launched.Stop waits for its bundle's exit to be handled; a bundle that
 // ignores SIGTERM longer than this is left to exit on its own (novox/hq issue 476).
 var StopWait = 5 * time.Second
@@ -182,6 +194,9 @@ var colour = regexp.MustCompile("\x1b\\[[0-9;]*m")
 // Start launches one bundle and learns its tools. It fails when the child cannot be started or does
 // not complete the handshake. A child that exits later is started again on its next call.
 func Start(module, entry string, env []string, mesh Bus, logf func(string, ...any)) (*Launched, error) {
+	if shuttingDown.Load() {
+		return nil, fmt.Errorf("%s's bundle is not started: the runtime is stopping", module)
+	}
 	var mu sync.Mutex
 	var current *child
 	// The child that last subscribed is the one events are handed to: a restarted child subscribes
@@ -444,7 +459,10 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
 				pause = pauseFirst
 			}
 			mu.Unlock()
-			if !wasStopped && wasServing {
+			if wasServing && !wasStopped && shuttingDown.Load() {
+				logf("[mesh-tools] %s; not started again: the runtime is stopping", why)
+			}
+			if !wasStopped && wasServing && !shuttingDown.Load() {
 				// Every bundle stays up (ADR 0198): code that runs long — a handler, a provisioner —
 				// is not waiting for a call to bring it back, and a tool bundle back early costs nothing.
 				logf("[mesh-tools] %s; started again in %s", why, wait.Round(100*time.Millisecond))
@@ -470,6 +488,9 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
 			return nil, fmt.Errorf("%s's bundle is stopped; it answers no call", module)
 		}
 		if c == nil {
+			if shuttingDown.Load() {
+				return nil, fmt.Errorf("%s's bundle is not started again: the runtime is stopping", module)
+			}
 			fresh, err := start()
 			if err != nil {
 				return nil, err
@@ -497,7 +518,7 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
 		go func() {
 			time.Sleep(after)
 			mu.Lock()
-			if stopped || current != nil {
+			if stopped || current != nil || shuttingDown.Load() {
 				mu.Unlock()
 				return
 			}
@@ -506,7 +527,7 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
 			mu.Lock()
 			defer mu.Unlock()
 			if err != nil {
-				if !stopped {
+				if !stopped && !shuttingDown.Load() {
 					wait := pause
 					if pause *= 2; pause > pauseMost {
 						pause = pauseMost