The reader of a bundle's stdout stopped at a line over 16 MiB without a word and the bundle's next write blocked for good: every call of the module then timed out, unheard of. Now the line is said, the bundle ended and started again by the exit path. A call the bundle does not answer in time is logged with tool and caller, and a refused publish of an answer (to a caller's inbox) is said once per subject per minute. Quick fix: only the bundle that met it (mesh-hand-acts' answer is cut by its own module). Thorough fix, chosen: the runner guards every bundle, so the kind (a silent freeze of a module runtime) cannot recur for any module. Claude-Session: https://claude.ai/code/session_01NcbgfFfQ4Z9UE5jK32PmE5
140 lines
5.1 KiB
Go
140 lines
5.1 KiB
Go
package launch
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// changedError is a lost compare-and-set as the bus answers it: told by its code (novox/hq ADR 0259).
|
|
type changedError struct{}
|
|
|
|
func (changedError) Error() string { return "the key changed since revision 3" }
|
|
func (changedError) ErrorCode() int { return CodeChanged }
|
|
|
|
// refusingState answers the state "lost" with a lost compare-and-set and every other state with a time-out.
|
|
type refusingState struct{ subscribing }
|
|
|
|
func (*refusingState) State(_ string, params json.RawMessage) (json.RawMessage, error) {
|
|
if strings.Contains(string(params), `"state":"lost"`) {
|
|
return nil, changedError{}
|
|
}
|
|
return nil, errors.New("the bus did not answer in 10s")
|
|
}
|
|
|
|
// A bundle that asks the runtime for the state "lost" once it is started, and for the state "copied" once that is
|
|
// answered, so the first answer's handling has ended before the second is asked.
|
|
const askingBundle = `#!/bin/sh
|
|
while IFS= read -r line; do
|
|
id=$(printf '%s' "$line" | sed -n 's/.*"id":\([0-9]*\)[,}].*/\1/p')
|
|
case "$line" in
|
|
*'"method":"initialize"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{}}\n' "$id" ;;
|
|
*'"method":"tools/list"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{"tools":[]}}\n' "$id"
|
|
printf '{"jsonrpc":"2.0","id":"q1","method":"mesh/state.update","params":{"state":"lost","key":"k"}}\n' ;;
|
|
*'"id":"q1"'*) printf '{"jsonrpc":"2.0","id":"q2","method":"mesh/state.get","params":{"state":"copied","key":"k"}}\n' ;;
|
|
esac
|
|
done
|
|
`
|
|
|
|
// A state operation the bus refuses or lets time out is said in the runtime's log with its method, its bundle and its
|
|
// time (issue 1966: a hand-act seat whose calls hung said nothing); a lost compare-and-set, asked first, is not.
|
|
func TestAFailedStateOperationIsSaidInTheRuntimesLog(t *testing.T) {
|
|
dir := t.TempDir()
|
|
entry := filepath.Join(dir, "bundle")
|
|
if err := os.WriteFile(entry, []byte(askingBundle), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
said := make(chan string, 16)
|
|
logf := func(f string, a ...any) { said <- fmt.Sprintf(f, a...) }
|
|
l, err := Start("asker", entry, os.Environ(), &refusingState{}, logf)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(l.Stop)
|
|
var heard []string
|
|
timeout := time.After(5 * time.Second)
|
|
for {
|
|
select {
|
|
case line := <-said:
|
|
heard = append(heard, line)
|
|
if strings.Contains(line, "mesh/state.update") {
|
|
t.Fatalf("a lost compare-and-set was said as a failure: %q", line)
|
|
}
|
|
if strings.Contains(line, "asker's bundle asked mesh/state.get and it failed after") && strings.Contains(line, "did not answer in 10s") {
|
|
return
|
|
}
|
|
case <-timeout:
|
|
t.Fatalf("the failed state operation was not said; the log: %v", heard)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A bundle that lists one tool, "t", and answers a call to it as its MODE says: "long" with a line over the runner's cap,
|
|
// "silent" with nothing at all.
|
|
const callingBundle = `#!/bin/sh
|
|
while IFS= read -r line; do
|
|
id=$(printf '%s' "$line" | sed -n 's/.*"id":\([0-9]*\)[,}].*/\1/p')
|
|
case "$line" in
|
|
*'"method":"initialize"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{}}\n' "$id" ;;
|
|
*'"method":"tools/list"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{"tools":[{"name":"t","description":"d"}]}}\n' "$id" ;;
|
|
*'"method":"tools/call"'*) [ "$MODE" = long ] && printf '%4096s\n' x ;;
|
|
esac
|
|
done
|
|
`
|
|
|
|
// Issue 1966: a bundle's line over the runner's cap, and a call the bundle never answers, are said in the runtime's log;
|
|
// the over-long line ends the bundle and fails the call at once, never leaving the runner reading nothing while the
|
|
// bundle's next write blocks.
|
|
func TestAnOverlongLineAndAnUnansweredCallAreSaid(t *testing.T) {
|
|
oldMax, oldCall := maxLine, CallTimeout
|
|
maxLine, CallTimeout = 1024, 300*time.Millisecond
|
|
t.Cleanup(func() { maxLine, CallTimeout = oldMax, oldCall })
|
|
for _, row := range []struct{ mode, said string }{
|
|
{"long", "bundle wrote a line over 1024 bytes"},
|
|
{"silent", ""},
|
|
} {
|
|
t.Run(row.mode, func(t *testing.T) {
|
|
entry := filepath.Join(t.TempDir(), "bundle")
|
|
if err := os.WriteFile(entry, []byte(callingBundle), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
said := make(chan string, 32)
|
|
logf := func(f string, a ...any) { said <- fmt.Sprintf(f, a...) }
|
|
l, err := Start("caller", entry, append(os.Environ(), "MODE="+row.mode), &refusingState{}, logf)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(l.Stop)
|
|
begun := time.Now()
|
|
_, err = l.Registrations[0].Tools[0].Run("caller-1", nil)
|
|
if err == nil {
|
|
t.Fatal("the call was answered")
|
|
}
|
|
if took := time.Since(begun); took > 5*time.Second {
|
|
t.Fatalf("the call hung for %s", took)
|
|
}
|
|
var heard []string
|
|
timeout := time.After(5 * time.Second)
|
|
for {
|
|
select {
|
|
case line := <-said:
|
|
heard = append(heard, line)
|
|
if row.mode == "long" && strings.Contains(line, row.said) && strings.Contains(line, "ending the bundle") {
|
|
return
|
|
}
|
|
if row.mode == "silent" && strings.Contains(line, "caller.t called by \"caller-1\"") && strings.Contains(line, "did not answer tools/call") {
|
|
return
|
|
}
|
|
case <-timeout:
|
|
t.Fatalf("not said; the log: %v", heard)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|