268 lines
10 KiB
Go
268 lines
10 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func testJudge(confined bool, licensed bool, exec func(ctx context.Context, argv []string, stdin []byte) ([]byte, error)) *Judge {
|
|
p := Paths{Node: "control", Operator: "op", Agent: "op"}
|
|
if confined {
|
|
p.Agent = "nox"
|
|
}
|
|
return &Judge{P: p, Licensed: func() bool { return licensed }, Exec: exec, Now: time.Now, turn: make(chan struct{}, 1)}
|
|
}
|
|
|
|
var verdictSchema = map[string]any{"type": "object", "properties": map[string]any{"verdict": map[string]any{"type": "string"}}}
|
|
|
|
// The session can do nothing but answer: built-in tools off, safe mode, no session kept, as the agent's account.
|
|
func TestAJudgementRunsTheAgentWithNoToolsAsItsAccount(t *testing.T) {
|
|
var got []string
|
|
var input string
|
|
j := testJudge(true, true, func(_ context.Context, argv []string, stdin []byte) ([]byte, error) {
|
|
got, input = argv, string(stdin)
|
|
return []byte(`{"is_error":false,"subtype":"success","structured_output":{"verdict":"none"}}`), nil
|
|
})
|
|
out, err := j.Judge(JudgeAsk{Question: "is 1 the same as 2?", Schema: verdictSchema})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(out.Answer) != `{"verdict":"none"}` || out.Model != "sonnet" || out.Node != "control" {
|
|
t.Fatalf("%+v", out)
|
|
}
|
|
if input != "is 1 the same as 2?" {
|
|
t.Fatalf("the question goes on the input, never the command line: %q", input)
|
|
}
|
|
line := strings.Join(got, " ")
|
|
if !strings.HasPrefix(line, "sudo -n -H -u nox -- "+JudgeProgram+" -p ") {
|
|
t.Fatalf("not run as the agent's account: %s", line)
|
|
}
|
|
i := slices.Index(got, "--tools")
|
|
if i < 0 || got[i+1] != "" {
|
|
t.Fatalf("built-in tools are not off: %q", got)
|
|
}
|
|
for _, want := range []string{"--safe-mode", "--no-session-persistence", "--json-schema"} {
|
|
if !slices.Contains(got, want) {
|
|
t.Errorf("%s missing: %q", want, got)
|
|
}
|
|
}
|
|
for _, a := range got {
|
|
if strings.Contains(a, "is 1 the same") {
|
|
t.Fatal("the question is on the command line")
|
|
}
|
|
}
|
|
|
|
// Where agents run as the operator's account, no sudo.
|
|
j = testJudge(false, true, func(_ context.Context, argv []string, _ []byte) ([]byte, error) {
|
|
got = argv
|
|
return []byte(`{"structured_output":{"verdict":"none"}}`), nil
|
|
})
|
|
if _, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema, Model: "haiku"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got[0] != JudgeProgram || !slices.Contains(got, "haiku") {
|
|
t.Fatalf("%q", got)
|
|
}
|
|
}
|
|
|
|
func TestAJudgementRefusesWhatItCannotAnswer(t *testing.T) {
|
|
ran := false
|
|
ok := func(context.Context, []string, []byte) ([]byte, error) {
|
|
ran = true
|
|
return []byte(`{"structured_output":{}}`), nil
|
|
}
|
|
cases := map[string]struct {
|
|
j *Judge
|
|
ask JudgeAsk
|
|
say string
|
|
}{
|
|
"empty": {testJudge(true, true, ok), JudgeAsk{Question: " ", Schema: verdictSchema}, "needs a question"},
|
|
"too large": {testJudge(true, true, ok), JudgeAsk{Question: strings.Repeat("x", MostQuestion+1), Schema: verdictSchema}, "64 KiB"},
|
|
"no schema": {testJudge(true, true, ok), JudgeAsk{Question: "q"}, "needs schema"},
|
|
"not object": {testJudge(true, true, ok), JudgeAsk{Question: "q", Schema: map[string]any{"type": "string"}}, "object"},
|
|
"model": {testJudge(true, true, ok), JudgeAsk{Question: "q", Schema: verdictSchema, Model: "gpt"}, "not one of"},
|
|
"no licence": {testJudge(true, false, ok), JudgeAsk{Question: "q", Schema: verdictSchema}, "holds no licence"},
|
|
}
|
|
for name, c := range cases {
|
|
_, err := c.j.Judge(c.ask)
|
|
if err == nil || !strings.Contains(err.Error(), c.say) {
|
|
t.Errorf("%s: %v", name, err)
|
|
}
|
|
}
|
|
if ran {
|
|
t.Fatal("a refused judgement ran the agent")
|
|
}
|
|
}
|
|
|
|
func TestAJudgementThatFailsSaysWhy(t *testing.T) {
|
|
answers := map[string][]byte{
|
|
"an error": []byte(`{"is_error":true,"subtype":"error_during_execution","result":"rate limited"}`),
|
|
"no structure": []byte(`{"is_error":false,"result":"it is a duplicate"}`),
|
|
"not an object": []byte(`{"structured_output":[1,2]}`),
|
|
"not json": []byte(`Invalid API key`),
|
|
}
|
|
for name, out := range answers {
|
|
j := testJudge(true, true, func(context.Context, []string, []byte) ([]byte, error) { return out, nil })
|
|
if _, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema}); err == nil || !strings.Contains(err.Error(), "not in the shape") {
|
|
t.Errorf("%s: %v", name, err)
|
|
}
|
|
}
|
|
j := testJudge(true, true, func(context.Context, []string, []byte) ([]byte, error) {
|
|
return nil, errors.New("exit status 1: sudo: a password is required")
|
|
})
|
|
if _, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema}); err == nil || !strings.Contains(err.Error(), "could not judge") {
|
|
t.Errorf("%v", err)
|
|
}
|
|
}
|
|
|
|
// One at a time on a node, and never past the bound: a judgement still waiting at the bound is refused.
|
|
func TestAJudgementWaitsItsTurnWithinTheBound(t *testing.T) {
|
|
j := testJudge(true, true, func(ctx context.Context, _ []string, _ []byte) ([]byte, error) {
|
|
<-ctx.Done()
|
|
return nil, ctx.Err()
|
|
})
|
|
j.Bound = 300 * time.Millisecond
|
|
j.turn <- struct{}{} // another judgement holds the agent
|
|
start := time.Now()
|
|
_, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema})
|
|
if err == nil || !strings.Contains(err.Error(), "another judgement") {
|
|
t.Fatalf("%v", err)
|
|
}
|
|
if time.Since(start) > 10*time.Second {
|
|
t.Fatal("waited past the bound")
|
|
}
|
|
}
|
|
|
|
func TestAJudgementPastTheBoundIsEnded(t *testing.T) {
|
|
j := testJudge(true, true, func(ctx context.Context, _ []string, _ []byte) ([]byte, error) {
|
|
<-ctx.Done()
|
|
return nil, ctx.Err()
|
|
})
|
|
j.Bound = 200 * time.Millisecond
|
|
if _, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema}); err == nil || !strings.Contains(err.Error(), "did not answer within") {
|
|
t.Fatalf("%v", err)
|
|
}
|
|
}
|
|
|
|
func TestTheToolTakesOnlyItsArguments(t *testing.T) {
|
|
j := testJudge(true, true, func(context.Context, []string, []byte) ([]byte, error) {
|
|
return []byte(`{"structured_output":{"verdict":"none"}}`), nil
|
|
})
|
|
tool := judgeTool(j)
|
|
if _, err := tool.Run(map[string]any{"question": "q", "schema": verdictSchema, "by": "x"}); err == nil {
|
|
t.Fatal("an unknown argument was taken")
|
|
}
|
|
out, err := tool.Run(map[string]any{"question": "q", "schema": verdictSchema})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
raw, _ := json.Marshal(out)
|
|
if !strings.Contains(string(raw), `"answer":{"verdict":"none"}`) {
|
|
t.Fatalf("%s", raw)
|
|
}
|
|
}
|
|
|
|
// A long question may ask a longer bound than one bus call allows (novox/hq issue 508): the call answers a ticket
|
|
// when the judgement is not done within the call's wait, the judgement runs on to its own bound, and
|
|
// claude_code_judged collects its answer.
|
|
func TestALongJudgementIsCollectedByItsTicket(t *testing.T) {
|
|
var deadline time.Time
|
|
j := testJudge(true, true, func(ctx context.Context, _ []string, _ []byte) ([]byte, error) {
|
|
deadline, _ = ctx.Deadline()
|
|
select {
|
|
case <-time.After(1500 * time.Millisecond): // well past Wait, however loaded the machine
|
|
return []byte(`{"structured_output":{"verdict":"none"}}`), nil
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
}
|
|
})
|
|
j.Wait = 50 * time.Millisecond
|
|
start := time.Now()
|
|
out, err := j.Judge(JudgeAsk{Question: "a long record", Schema: verdictSchema, Within: 90 * time.Second})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !out.Pending || out.Ticket == "" || len(out.Answer) != 0 || out.Node != "control" {
|
|
t.Fatalf("not a ticket: %+v", out)
|
|
}
|
|
var got *Judged
|
|
for i := 0; i < 600; i++ {
|
|
got, err = j.Collect(out.Ticket)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !got.Pending {
|
|
break
|
|
}
|
|
}
|
|
if got.Pending || string(got.Answer) != `{"verdict":"none"}` || got.Ticket != out.Ticket {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
// Read once the judgement is collected: its end is what orders the agent's write before this read.
|
|
if left := deadline.Sub(start); left < 80*time.Second || left > 91*time.Second {
|
|
t.Fatalf("the judgement's bound is %s, not the 90s asked", left)
|
|
}
|
|
if _, err := j.Collect(out.Ticket); err == nil || !strings.Contains(err.Error(), "no judgement") {
|
|
t.Fatalf("a ticket collected twice: %v", err)
|
|
}
|
|
}
|
|
|
|
// A long judgement that fails says why when it is collected, and its bound is never past MostJudgeBound.
|
|
func TestALongJudgementIsBoundedAndSaysWhyItFailed(t *testing.T) {
|
|
j := testJudge(true, true, func(ctx context.Context, _ []string, _ []byte) ([]byte, error) {
|
|
<-ctx.Done()
|
|
return nil, ctx.Err()
|
|
})
|
|
if _, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema, Within: MostJudgeBound + time.Second}); err == nil ||
|
|
!strings.Contains(err.Error(), "at most") {
|
|
t.Fatalf("a bound past the most taken: %v", err)
|
|
}
|
|
j.Wait = 20 * time.Millisecond
|
|
j.Bound = time.Second // the test's stand-in for JudgeBound: within is past it, so a ticket; far past Wait on a loaded machine
|
|
out, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema, Within: 2 * time.Second})
|
|
if err != nil || !out.Pending {
|
|
t.Fatalf("%+v %v", out, err)
|
|
}
|
|
for i := 0; i < 3000; i++ {
|
|
got, err := j.Collect(out.Ticket)
|
|
if err != nil {
|
|
if !strings.Contains(err.Error(), "did not answer within") {
|
|
t.Fatalf("%v", err)
|
|
}
|
|
return
|
|
}
|
|
if !got.Pending {
|
|
t.Fatalf("answered: %+v", got)
|
|
}
|
|
}
|
|
t.Fatal("never ended")
|
|
}
|
|
|
|
// The tools take within and ticket, and nothing else.
|
|
func TestTheToolsTakeWithinAndTicket(t *testing.T) {
|
|
j := testJudge(true, true, func(context.Context, []string, []byte) ([]byte, error) {
|
|
return []byte(`{"structured_output":{"verdict":"none"}}`), nil
|
|
})
|
|
out, err := judgeTool(j).Run(map[string]any{"question": "q", "schema": verdictSchema, "within": float64(60)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := out.(*Judged); got.Pending || string(got.Answer) != `{"verdict":"none"}` {
|
|
t.Fatalf("a quick long judgement is answered in its call: %+v", got)
|
|
}
|
|
if _, err := judgeTool(j).Run(map[string]any{"question": "q", "schema": verdictSchema, "within": "long"}); err == nil {
|
|
t.Fatal("within as text was taken")
|
|
}
|
|
if _, err := judgedTool(j).Run(map[string]any{"ticket": "nope"}); err == nil || !strings.Contains(err.Error(), "no judgement") {
|
|
t.Fatalf("%v", err)
|
|
}
|
|
if _, err := judgedTool(j).Run(map[string]any{"ticket": "x", "by": "y"}); err == nil {
|
|
t.Fatal("an unknown argument was taken")
|
|
}
|
|
}
|