Merge pull request 'node-tools starts no bundle again once it is stopping (group fix/shutdown-bound, issue 1126)' (#46) from fix/shutdown-bound into main
This commit was merged in pull request #46.
This commit is contained in:
@@ -26,6 +26,7 @@ import (
|
|||||||
"github.com/novox/mesh-tools/node-tools/internal/alive"
|
"github.com/novox/mesh-tools/node-tools/internal/alive"
|
||||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||||
"github.com/novox/mesh-tools/node-tools/internal/console"
|
"github.com/novox/mesh-tools/node-tools/internal/console"
|
||||||
|
"github.com/novox/mesh-tools/node-tools/internal/launch"
|
||||||
"github.com/novox/mesh-tools/node-tools/internal/provide"
|
"github.com/novox/mesh-tools/node-tools/internal/provide"
|
||||||
"github.com/novox/mesh-tools/node-tools/internal/runtime"
|
"github.com/novox/mesh-tools/node-tools/internal/runtime"
|
||||||
)
|
)
|
||||||
@@ -91,6 +92,9 @@ func main() {
|
|||||||
signals := make(chan os.Signal, 1)
|
signals := make(chan os.Signal, 1)
|
||||||
signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
|
signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
|
||||||
<-signals
|
<-signals
|
||||||
|
// First of all: the service manager signalled every bundle with this process, and none is started again
|
||||||
|
// (issue 1126).
|
||||||
|
launch.ShutDown()
|
||||||
stopSaying()
|
stopSaying()
|
||||||
stopProviding()
|
stopProviding()
|
||||||
stop()
|
stop()
|
||||||
|
|||||||
@@ -3,7 +3,12 @@ package launch
|
|||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
type lostWrite struct{}
|
type lostWrite struct{}
|
||||||
@@ -28,3 +33,84 @@ func TestALostWriteIsAnsweredWithItsOwnCodeAndNothingElseIs(t *testing.T) {
|
|||||||
t.Errorf("a plain refusal is answered %d", got)
|
t.Errorf("a plain refusal is answered %d", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A bundle that answers the handshake and an empty tool list, and notes each start and its pid in $OUT.
|
||||||
|
const notingBundle = `#!/bin/sh
|
||||||
|
printf 'started %s\n' "$$" >> "$OUT"
|
||||||
|
while IFS= read -r line; do
|
||||||
|
id=$(printf '%s' "$line" | sed -n 's/.*"id":\([0-9]*\)[,}].*/\1/p')
|
||||||
|
case "$line" in
|
||||||
|
*'"method":"initialize"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{}}\n' "$id" ;;
|
||||||
|
*'"method":"tools/list"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{"tools":[]}}\n' "$id" ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
`
|
||||||
|
|
||||||
|
// The service manager stops the runtime by signalling its whole group, every bundle with it (issue 1126, ace's
|
||||||
|
// reboot of 2026-10-28: every bundle "exited (SIGTERM); started again in 500ms" for the 15 s the runtime took
|
||||||
|
// to stop, the power bundle among them dialling the system bus again, which restarted the bus during the
|
||||||
|
// shutdown and held it 29 minutes). A bundle that exits while it serves is started again; once the runtime
|
||||||
|
// itself is stopping, none is, by a restart or by a call.
|
||||||
|
func TestABundleIsStartedAgainUnlessTheRuntimeIsStopping(t *testing.T) {
|
||||||
|
saved := RestartFirst
|
||||||
|
RestartFirst = 10 * time.Millisecond
|
||||||
|
t.Cleanup(func() { RestartFirst = saved; shuttingDown.Store(false) })
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
stopping bool
|
||||||
|
starts int
|
||||||
|
}{
|
||||||
|
{"a bundle that dies while the runtime serves", false, 2},
|
||||||
|
{"a bundle signalled with the stopping runtime", true, 1},
|
||||||
|
} {
|
||||||
|
t.Run(row.name, func(t *testing.T) {
|
||||||
|
shuttingDown.Store(false)
|
||||||
|
dir := t.TempDir()
|
||||||
|
entry, out := filepath.Join(dir, "bundle"), filepath.Join(dir, "out")
|
||||||
|
if err := os.WriteFile(entry, []byte(notingBundle), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
l, err := Start("noting", entry, append(os.Environ(), "OUT="+out), &subscribing{}, t.Logf)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(l.Stop)
|
||||||
|
said, _ := os.ReadFile(out)
|
||||||
|
var pid int
|
||||||
|
if _, err := fmt.Sscanf(string(said), "started %d", &pid); err != nil {
|
||||||
|
t.Fatalf("the bundle noted %q", said)
|
||||||
|
}
|
||||||
|
if row.stopping {
|
||||||
|
ShutDown()
|
||||||
|
}
|
||||||
|
if err := syscall.Kill(pid, syscall.SIGTERM); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// The restart row ends at its second start; the stopping row watches a window many restart pauses long.
|
||||||
|
deadline := time.Now().Add(250 * time.Millisecond)
|
||||||
|
for {
|
||||||
|
said, _ = os.ReadFile(out)
|
||||||
|
n := strings.Count(string(said), "started")
|
||||||
|
if n > row.starts {
|
||||||
|
t.Fatalf("started %d times, want %d", n, row.starts)
|
||||||
|
}
|
||||||
|
if n == row.starts && !row.stopping {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
if n != row.starts {
|
||||||
|
t.Fatalf("started %d times, want %d", n, row.starts)
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
time.Sleep(5 * time.Millisecond)
|
||||||
|
}
|
||||||
|
if row.stopping {
|
||||||
|
// And a bundle first loaded now (a reload) is not started either.
|
||||||
|
if _, err := Start("late", entry, append(os.Environ(), "OUT="+out), &subscribing{}, t.Logf); err == nil {
|
||||||
|
t.Error("a bundle was started after the runtime began stopping")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -112,6 +113,17 @@ var (
|
|||||||
RestartSettle = time.Minute
|
RestartSettle = time.Minute
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// shuttingDown is set once the runtime itself is stopping (ShutDown): from then on no bundle is started again.
|
||||||
|
var shuttingDown atomic.Bool
|
||||||
|
|
||||||
|
// ShutDown says the runtime itself is stopping, before it stops its bundles one by one. The service manager stops
|
||||||
|
// the runtime by signalling its whole group, so every bundle receives SIGTERM with it, and a bundle's exit must not
|
||||||
|
// be taken for a fault: started again, it would run on through the machine's shutdown with nothing left to stop it
|
||||||
|
// (issue 1126: on ace's reboot of 2026-10-28 the restarted bundles dialled the system bus again and started it
|
||||||
|
// during the shutdown, which then waited 29 minutes). After ShutDown no bundle is started again, by a restart or a
|
||||||
|
// call.
|
||||||
|
func ShutDown() { shuttingDown.Store(true) }
|
||||||
|
|
||||||
// StopWait bounds how long Launched.Stop waits for its bundle's exit to be handled; a bundle that
|
// StopWait bounds how long Launched.Stop waits for its bundle's exit to be handled; a bundle that
|
||||||
// ignores SIGTERM longer than this is left to exit on its own (novox/hq issue 476).
|
// ignores SIGTERM longer than this is left to exit on its own (novox/hq issue 476).
|
||||||
var StopWait = 5 * time.Second
|
var StopWait = 5 * time.Second
|
||||||
@@ -182,6 +194,9 @@ var colour = regexp.MustCompile("\x1b\\[[0-9;]*m")
|
|||||||
// Start launches one bundle and learns its tools. It fails when the child cannot be started or does
|
// Start launches one bundle and learns its tools. It fails when the child cannot be started or does
|
||||||
// not complete the handshake. A child that exits later is started again on its next call.
|
// not complete the handshake. A child that exits later is started again on its next call.
|
||||||
func Start(module, entry string, env []string, mesh Bus, logf func(string, ...any)) (*Launched, error) {
|
func Start(module, entry string, env []string, mesh Bus, logf func(string, ...any)) (*Launched, error) {
|
||||||
|
if shuttingDown.Load() {
|
||||||
|
return nil, fmt.Errorf("%s's bundle is not started: the runtime is stopping", module)
|
||||||
|
}
|
||||||
var mu sync.Mutex
|
var mu sync.Mutex
|
||||||
var current *child
|
var current *child
|
||||||
// The child that last subscribed is the one events are handed to: a restarted child subscribes
|
// The child that last subscribed is the one events are handed to: a restarted child subscribes
|
||||||
@@ -444,7 +459,10 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
|||||||
pause = pauseFirst
|
pause = pauseFirst
|
||||||
}
|
}
|
||||||
mu.Unlock()
|
mu.Unlock()
|
||||||
if !wasStopped && wasServing {
|
if wasServing && !wasStopped && shuttingDown.Load() {
|
||||||
|
logf("[mesh-tools] %s; not started again: the runtime is stopping", why)
|
||||||
|
}
|
||||||
|
if !wasStopped && wasServing && !shuttingDown.Load() {
|
||||||
// Every bundle stays up (ADR 0198): code that runs long — a handler, a provisioner —
|
// Every bundle stays up (ADR 0198): code that runs long — a handler, a provisioner —
|
||||||
// is not waiting for a call to bring it back, and a tool bundle back early costs nothing.
|
// is not waiting for a call to bring it back, and a tool bundle back early costs nothing.
|
||||||
logf("[mesh-tools] %s; started again in %s", why, wait.Round(100*time.Millisecond))
|
logf("[mesh-tools] %s; started again in %s", why, wait.Round(100*time.Millisecond))
|
||||||
@@ -470,6 +488,9 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
|||||||
return nil, fmt.Errorf("%s's bundle is stopped; it answers no call", module)
|
return nil, fmt.Errorf("%s's bundle is stopped; it answers no call", module)
|
||||||
}
|
}
|
||||||
if c == nil {
|
if c == nil {
|
||||||
|
if shuttingDown.Load() {
|
||||||
|
return nil, fmt.Errorf("%s's bundle is not started again: the runtime is stopping", module)
|
||||||
|
}
|
||||||
fresh, err := start()
|
fresh, err := start()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -497,7 +518,7 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
|||||||
go func() {
|
go func() {
|
||||||
time.Sleep(after)
|
time.Sleep(after)
|
||||||
mu.Lock()
|
mu.Lock()
|
||||||
if stopped || current != nil {
|
if stopped || current != nil || shuttingDown.Load() {
|
||||||
mu.Unlock()
|
mu.Unlock()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -506,7 +527,7 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
|||||||
mu.Lock()
|
mu.Lock()
|
||||||
defer mu.Unlock()
|
defer mu.Unlock()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !stopped {
|
if !stopped && !shuttingDown.Load() {
|
||||||
wait := pause
|
wait := pause
|
||||||
if pause *= 2; pause > pauseMost {
|
if pause *= 2; pause > pauseMost {
|
||||||
pause = pauseMost
|
pause = pauseMost
|
||||||
|
|||||||
Reference in New Issue
Block a user