jschoubben
fcab25604e
mesh/merge-gate pass: builds claude-code, mesh-knowledge → ace, g14, novox, shanks; no bus step; every machine composes with the change as it did without (…
mesh/repo-check pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-check-typescript.sh (typescript)
mesh/delivery delivered
A judgement over one bus call's 30 seconds is collected by its ticket from claude-code, so an eleven-clause record can be reviewed; a proposal out of review tries is told to the operator, not raised as the module's fault, so it no longer holds the module's own rollouts.
268 lines
9.9 KiB
Go
268 lines
9.9 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func testJudge(confined bool, licensed bool, exec func(ctx context.Context, argv []string, stdin []byte) ([]byte, error)) *Judge {
|
|
p := Paths{Node: "control", Operator: "op", Agent: "op"}
|
|
if confined {
|
|
p.Agent = "nox"
|
|
}
|
|
return &Judge{P: p, Licensed: func() bool { return licensed }, Exec: exec, Now: time.Now, turn: make(chan struct{}, 1)}
|
|
}
|
|
|
|
var verdictSchema = map[string]any{"type": "object", "properties": map[string]any{"verdict": map[string]any{"type": "string"}}}
|
|
|
|
// The session can do nothing but answer: built-in tools off, safe mode, no session kept, as the agent's account.
|
|
func TestAJudgementRunsTheAgentWithNoToolsAsItsAccount(t *testing.T) {
|
|
var got []string
|
|
var input string
|
|
j := testJudge(true, true, func(_ context.Context, argv []string, stdin []byte) ([]byte, error) {
|
|
got, input = argv, string(stdin)
|
|
return []byte(`{"is_error":false,"subtype":"success","structured_output":{"verdict":"none"}}`), nil
|
|
})
|
|
out, err := j.Judge(JudgeAsk{Question: "is 1 the same as 2?", Schema: verdictSchema})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(out.Answer) != `{"verdict":"none"}` || out.Model != "sonnet" || out.Node != "control" {
|
|
t.Fatalf("%+v", out)
|
|
}
|
|
if input != "is 1 the same as 2?" {
|
|
t.Fatalf("the question goes on the input, never the command line: %q", input)
|
|
}
|
|
line := strings.Join(got, " ")
|
|
if !strings.HasPrefix(line, "sudo -n -H -u nox -- "+JudgeProgram+" -p ") {
|
|
t.Fatalf("not run as the agent's account: %s", line)
|
|
}
|
|
i := slices.Index(got, "--tools")
|
|
if i < 0 || got[i+1] != "" {
|
|
t.Fatalf("built-in tools are not off: %q", got)
|
|
}
|
|
for _, want := range []string{"--safe-mode", "--no-session-persistence", "--json-schema"} {
|
|
if !slices.Contains(got, want) {
|
|
t.Errorf("%s missing: %q", want, got)
|
|
}
|
|
}
|
|
for _, a := range got {
|
|
if strings.Contains(a, "is 1 the same") {
|
|
t.Fatal("the question is on the command line")
|
|
}
|
|
}
|
|
|
|
// Where agents run as the operator's account, no sudo.
|
|
j = testJudge(false, true, func(_ context.Context, argv []string, _ []byte) ([]byte, error) {
|
|
got = argv
|
|
return []byte(`{"structured_output":{"verdict":"none"}}`), nil
|
|
})
|
|
if _, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema, Model: "haiku"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got[0] != JudgeProgram || !slices.Contains(got, "haiku") {
|
|
t.Fatalf("%q", got)
|
|
}
|
|
}
|
|
|
|
func TestAJudgementRefusesWhatItCannotAnswer(t *testing.T) {
|
|
ran := false
|
|
ok := func(context.Context, []string, []byte) ([]byte, error) {
|
|
ran = true
|
|
return []byte(`{"structured_output":{}}`), nil
|
|
}
|
|
cases := map[string]struct {
|
|
j *Judge
|
|
ask JudgeAsk
|
|
say string
|
|
}{
|
|
"empty": {testJudge(true, true, ok), JudgeAsk{Question: " ", Schema: verdictSchema}, "needs a question"},
|
|
"too large": {testJudge(true, true, ok), JudgeAsk{Question: strings.Repeat("x", MostQuestion+1), Schema: verdictSchema}, "64 KiB"},
|
|
"no schema": {testJudge(true, true, ok), JudgeAsk{Question: "q"}, "needs schema"},
|
|
"not object": {testJudge(true, true, ok), JudgeAsk{Question: "q", Schema: map[string]any{"type": "string"}}, "object"},
|
|
"model": {testJudge(true, true, ok), JudgeAsk{Question: "q", Schema: verdictSchema, Model: "gpt"}, "not one of"},
|
|
"no licence": {testJudge(true, false, ok), JudgeAsk{Question: "q", Schema: verdictSchema}, "holds no licence"},
|
|
}
|
|
for name, c := range cases {
|
|
_, err := c.j.Judge(c.ask)
|
|
if err == nil || !strings.Contains(err.Error(), c.say) {
|
|
t.Errorf("%s: %v", name, err)
|
|
}
|
|
}
|
|
if ran {
|
|
t.Fatal("a refused judgement ran the agent")
|
|
}
|
|
}
|
|
|
|
func TestAJudgementThatFailsSaysWhy(t *testing.T) {
|
|
answers := map[string][]byte{
|
|
"an error": []byte(`{"is_error":true,"subtype":"error_during_execution","result":"rate limited"}`),
|
|
"no structure": []byte(`{"is_error":false,"result":"it is a duplicate"}`),
|
|
"not an object": []byte(`{"structured_output":[1,2]}`),
|
|
"not json": []byte(`Invalid API key`),
|
|
}
|
|
for name, out := range answers {
|
|
j := testJudge(true, true, func(context.Context, []string, []byte) ([]byte, error) { return out, nil })
|
|
if _, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema}); err == nil || !strings.Contains(err.Error(), "not in the shape") {
|
|
t.Errorf("%s: %v", name, err)
|
|
}
|
|
}
|
|
j := testJudge(true, true, func(context.Context, []string, []byte) ([]byte, error) {
|
|
return nil, errors.New("exit status 1: sudo: a password is required")
|
|
})
|
|
if _, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema}); err == nil || !strings.Contains(err.Error(), "could not judge") {
|
|
t.Errorf("%v", err)
|
|
}
|
|
}
|
|
|
|
// One at a time on a node, and never past the bound: a judgement still waiting at the bound is refused.
|
|
func TestAJudgementWaitsItsTurnWithinTheBound(t *testing.T) {
|
|
j := testJudge(true, true, func(ctx context.Context, _ []string, _ []byte) ([]byte, error) {
|
|
<-ctx.Done()
|
|
return nil, ctx.Err()
|
|
})
|
|
j.Bound = 300 * time.Millisecond
|
|
j.turn <- struct{}{} // another judgement holds the agent
|
|
start := time.Now()
|
|
_, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema})
|
|
if err == nil || !strings.Contains(err.Error(), "another judgement") {
|
|
t.Fatalf("%v", err)
|
|
}
|
|
if time.Since(start) > 2*time.Second {
|
|
t.Fatal("waited past the bound")
|
|
}
|
|
}
|
|
|
|
func TestAJudgementPastTheBoundIsEnded(t *testing.T) {
|
|
j := testJudge(true, true, func(ctx context.Context, _ []string, _ []byte) ([]byte, error) {
|
|
<-ctx.Done()
|
|
return nil, ctx.Err()
|
|
})
|
|
j.Bound = 200 * time.Millisecond
|
|
if _, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema}); err == nil || !strings.Contains(err.Error(), "did not answer within") {
|
|
t.Fatalf("%v", err)
|
|
}
|
|
}
|
|
|
|
func TestTheToolTakesOnlyItsArguments(t *testing.T) {
|
|
j := testJudge(true, true, func(context.Context, []string, []byte) ([]byte, error) {
|
|
return []byte(`{"structured_output":{"verdict":"none"}}`), nil
|
|
})
|
|
tool := judgeTool(j)
|
|
if _, err := tool.Run(map[string]any{"question": "q", "schema": verdictSchema, "by": "x"}); err == nil {
|
|
t.Fatal("an unknown argument was taken")
|
|
}
|
|
out, err := tool.Run(map[string]any{"question": "q", "schema": verdictSchema})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
raw, _ := json.Marshal(out)
|
|
if !strings.Contains(string(raw), `"answer":{"verdict":"none"}`) {
|
|
t.Fatalf("%s", raw)
|
|
}
|
|
}
|
|
|
|
// A long question may ask a longer bound than one bus call allows (novox/hq issue 508): the call answers a ticket
|
|
// when the judgement is not done within the call's wait, the judgement runs on to its own bound, and
|
|
// claude_code_judged collects its answer.
|
|
func TestALongJudgementIsCollectedByItsTicket(t *testing.T) {
|
|
var deadline time.Time
|
|
j := testJudge(true, true, func(ctx context.Context, _ []string, _ []byte) ([]byte, error) {
|
|
deadline, _ = ctx.Deadline()
|
|
select {
|
|
case <-time.After(400 * time.Millisecond):
|
|
return []byte(`{"structured_output":{"verdict":"none"}}`), nil
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
}
|
|
})
|
|
j.Wait = 50 * time.Millisecond
|
|
start := time.Now()
|
|
out, err := j.Judge(JudgeAsk{Question: "a long record", Schema: verdictSchema, Within: 90 * time.Second})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !out.Pending || out.Ticket == "" || len(out.Answer) != 0 || out.Node != "control" {
|
|
t.Fatalf("not a ticket: %+v", out)
|
|
}
|
|
var got *Judged
|
|
for i := 0; i < 100; i++ {
|
|
got, err = j.Collect(out.Ticket)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !got.Pending {
|
|
break
|
|
}
|
|
}
|
|
if got.Pending || string(got.Answer) != `{"verdict":"none"}` || got.Ticket != out.Ticket {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
// Read once the judgement is collected: its end is what orders the agent's write before this read.
|
|
if left := deadline.Sub(start); left < 80*time.Second || left > 91*time.Second {
|
|
t.Fatalf("the judgement's bound is %s, not the 90s asked", left)
|
|
}
|
|
if _, err := j.Collect(out.Ticket); err == nil || !strings.Contains(err.Error(), "no judgement") {
|
|
t.Fatalf("a ticket collected twice: %v", err)
|
|
}
|
|
}
|
|
|
|
// A long judgement that fails says why when it is collected, and its bound is never past MostJudgeBound.
|
|
func TestALongJudgementIsBoundedAndSaysWhyItFailed(t *testing.T) {
|
|
j := testJudge(true, true, func(ctx context.Context, _ []string, _ []byte) ([]byte, error) {
|
|
<-ctx.Done()
|
|
return nil, ctx.Err()
|
|
})
|
|
if _, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema, Within: MostJudgeBound + time.Second}); err == nil ||
|
|
!strings.Contains(err.Error(), "at most") {
|
|
t.Fatalf("a bound past the most taken: %v", err)
|
|
}
|
|
j.Wait = 20 * time.Millisecond
|
|
j.Bound = 30 * time.Millisecond // the test's stand-in for JudgeBound: within is past it, so a ticket
|
|
out, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema, Within: 200 * time.Millisecond})
|
|
if err != nil || !out.Pending {
|
|
t.Fatalf("%+v %v", out, err)
|
|
}
|
|
for i := 0; i < 100; i++ {
|
|
got, err := j.Collect(out.Ticket)
|
|
if err != nil {
|
|
if !strings.Contains(err.Error(), "did not answer within") {
|
|
t.Fatalf("%v", err)
|
|
}
|
|
return
|
|
}
|
|
if !got.Pending {
|
|
t.Fatalf("answered: %+v", got)
|
|
}
|
|
}
|
|
t.Fatal("never ended")
|
|
}
|
|
|
|
// The tools take within and ticket, and nothing else.
|
|
func TestTheToolsTakeWithinAndTicket(t *testing.T) {
|
|
j := testJudge(true, true, func(context.Context, []string, []byte) ([]byte, error) {
|
|
return []byte(`{"structured_output":{"verdict":"none"}}`), nil
|
|
})
|
|
out, err := judgeTool(j).Run(map[string]any{"question": "q", "schema": verdictSchema, "within": float64(60)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := out.(*Judged); got.Pending || string(got.Answer) != `{"verdict":"none"}` {
|
|
t.Fatalf("a quick long judgement is answered in its call: %+v", got)
|
|
}
|
|
if _, err := judgeTool(j).Run(map[string]any{"question": "q", "schema": verdictSchema, "within": "long"}); err == nil {
|
|
t.Fatal("within as text was taken")
|
|
}
|
|
if _, err := judgedTool(j).Run(map[string]any{"ticket": "nope"}); err == nil || !strings.Contains(err.Error(), "no judgement") {
|
|
t.Fatalf("%v", err)
|
|
}
|
|
if _, err := judgedTool(j).Run(map[string]any{"ticket": "x", "by": "y"}); err == nil {
|
|
t.Fatal("an unknown argument was taken")
|
|
}
|
|
}
|