package main import ( "context" "encoding/json" "errors" "slices" "strings" "testing" "time" ) func testJudge(confined bool, licensed bool, exec func(ctx context.Context, argv []string, stdin []byte) ([]byte, error)) *Judge { p := Paths{Node: "control", Operator: "op", Agent: "op"} if confined { p.Agent = "nox" } return &Judge{P: p, Licensed: func() bool { return licensed }, Exec: exec, Now: time.Now, turn: make(chan struct{}, 1)} } var verdictSchema = map[string]any{"type": "object", "properties": map[string]any{"verdict": map[string]any{"type": "string"}}} // The session can do nothing but answer: built-in tools off, safe mode, no session kept, as the agent's account. func TestAJudgementRunsTheAgentWithNoToolsAsItsAccount(t *testing.T) { var got []string var input string j := testJudge(true, true, func(_ context.Context, argv []string, stdin []byte) ([]byte, error) { got, input = argv, string(stdin) return []byte(`{"is_error":false,"subtype":"success","structured_output":{"verdict":"none"}}`), nil }) out, err := j.Judge(JudgeAsk{Question: "is 1 the same as 2?", Schema: verdictSchema}) if err != nil { t.Fatal(err) } if string(out.Answer) != `{"verdict":"none"}` || out.Model != "sonnet" || out.Node != "control" { t.Fatalf("%+v", out) } if input != "is 1 the same as 2?" { t.Fatalf("the question goes on the input, never the command line: %q", input) } line := strings.Join(got, " ") if !strings.HasPrefix(line, "sudo -n -H -u nox -- "+JudgeProgram+" -p ") { t.Fatalf("not run as the agent's account: %s", line) } i := slices.Index(got, "--tools") if i < 0 || got[i+1] != "" { t.Fatalf("built-in tools are not off: %q", got) } for _, want := range []string{"--safe-mode", "--no-session-persistence", "--json-schema"} { if !slices.Contains(got, want) { t.Errorf("%s missing: %q", want, got) } } for _, a := range got { if strings.Contains(a, "is 1 the same") { t.Fatal("the question is on the command line") } } // Where agents run as the operator's account, no sudo. j = testJudge(false, true, func(_ context.Context, argv []string, _ []byte) ([]byte, error) { got = argv return []byte(`{"structured_output":{"verdict":"none"}}`), nil }) if _, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema, Model: "haiku"}); err != nil { t.Fatal(err) } if got[0] != JudgeProgram || !slices.Contains(got, "haiku") { t.Fatalf("%q", got) } } func TestAJudgementRefusesWhatItCannotAnswer(t *testing.T) { ran := false ok := func(context.Context, []string, []byte) ([]byte, error) { ran = true return []byte(`{"structured_output":{}}`), nil } cases := map[string]struct { j *Judge ask JudgeAsk say string }{ "empty": {testJudge(true, true, ok), JudgeAsk{Question: " ", Schema: verdictSchema}, "needs a question"}, "too large": {testJudge(true, true, ok), JudgeAsk{Question: strings.Repeat("x", MostQuestion+1), Schema: verdictSchema}, "64 KiB"}, "no schema": {testJudge(true, true, ok), JudgeAsk{Question: "q"}, "needs schema"}, "not object": {testJudge(true, true, ok), JudgeAsk{Question: "q", Schema: map[string]any{"type": "string"}}, "object"}, "model": {testJudge(true, true, ok), JudgeAsk{Question: "q", Schema: verdictSchema, Model: "gpt"}, "not one of"}, "no licence": {testJudge(true, false, ok), JudgeAsk{Question: "q", Schema: verdictSchema}, "holds no licence"}, } for name, c := range cases { _, err := c.j.Judge(c.ask) if err == nil || !strings.Contains(err.Error(), c.say) { t.Errorf("%s: %v", name, err) } } if ran { t.Fatal("a refused judgement ran the agent") } } func TestAJudgementThatFailsSaysWhy(t *testing.T) { answers := map[string][]byte{ "an error": []byte(`{"is_error":true,"subtype":"error_during_execution","result":"rate limited"}`), "no structure": []byte(`{"is_error":false,"result":"it is a duplicate"}`), "not an object": []byte(`{"structured_output":[1,2]}`), "not json": []byte(`Invalid API key`), } for name, out := range answers { j := testJudge(true, true, func(context.Context, []string, []byte) ([]byte, error) { return out, nil }) if _, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema}); err == nil || !strings.Contains(err.Error(), "not in the shape") { t.Errorf("%s: %v", name, err) } } j := testJudge(true, true, func(context.Context, []string, []byte) ([]byte, error) { return nil, errors.New("exit status 1: sudo: a password is required") }) if _, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema}); err == nil || !strings.Contains(err.Error(), "could not judge") { t.Errorf("%v", err) } } // One at a time on a node, and never past the bound: a judgement still waiting at the bound is refused. func TestAJudgementWaitsItsTurnWithinTheBound(t *testing.T) { j := testJudge(true, true, func(ctx context.Context, _ []string, _ []byte) ([]byte, error) { <-ctx.Done() return nil, ctx.Err() }) j.Bound = 300 * time.Millisecond j.turn <- struct{}{} // another judgement holds the agent start := time.Now() _, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema}) if err == nil || !strings.Contains(err.Error(), "another judgement") { t.Fatalf("%v", err) } if time.Since(start) > 2*time.Second { t.Fatal("waited past the bound") } } func TestAJudgementPastTheBoundIsEnded(t *testing.T) { j := testJudge(true, true, func(ctx context.Context, _ []string, _ []byte) ([]byte, error) { <-ctx.Done() return nil, ctx.Err() }) j.Bound = 200 * time.Millisecond if _, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema}); err == nil || !strings.Contains(err.Error(), "did not answer within") { t.Fatalf("%v", err) } } func TestTheToolTakesOnlyItsArguments(t *testing.T) { j := testJudge(true, true, func(context.Context, []string, []byte) ([]byte, error) { return []byte(`{"structured_output":{"verdict":"none"}}`), nil }) tool := judgeTool(j) if _, err := tool.Run(map[string]any{"question": "q", "schema": verdictSchema, "by": "x"}); err == nil { t.Fatal("an unknown argument was taken") } out, err := tool.Run(map[string]any{"question": "q", "schema": verdictSchema}) if err != nil { t.Fatal(err) } raw, _ := json.Marshal(out) if !strings.Contains(string(raw), `"answer":{"verdict":"none"}`) { t.Fatalf("%s", raw) } } // A long question may ask a longer bound than one bus call allows (novox/hq issue 508): the call answers a ticket // when the judgement is not done within the call's wait, the judgement runs on to its own bound, and // claude_code_judged collects its answer. func TestALongJudgementIsCollectedByItsTicket(t *testing.T) { var deadline time.Time j := testJudge(true, true, func(ctx context.Context, _ []string, _ []byte) ([]byte, error) { deadline, _ = ctx.Deadline() select { case <-time.After(400 * time.Millisecond): return []byte(`{"structured_output":{"verdict":"none"}}`), nil case <-ctx.Done(): return nil, ctx.Err() } }) j.Wait = 50 * time.Millisecond start := time.Now() out, err := j.Judge(JudgeAsk{Question: "a long record", Schema: verdictSchema, Within: 90 * time.Second}) if err != nil { t.Fatal(err) } if !out.Pending || out.Ticket == "" || len(out.Answer) != 0 || out.Node != "control" { t.Fatalf("not a ticket: %+v", out) } var got *Judged for i := 0; i < 100; i++ { got, err = j.Collect(out.Ticket) if err != nil { t.Fatal(err) } if !got.Pending { break } } if got.Pending || string(got.Answer) != `{"verdict":"none"}` || got.Ticket != out.Ticket { t.Fatalf("%+v", got) } // Read once the judgement is collected: its end is what orders the agent's write before this read. if left := deadline.Sub(start); left < 80*time.Second || left > 91*time.Second { t.Fatalf("the judgement's bound is %s, not the 90s asked", left) } if _, err := j.Collect(out.Ticket); err == nil || !strings.Contains(err.Error(), "no judgement") { t.Fatalf("a ticket collected twice: %v", err) } } // A long judgement that fails says why when it is collected, and its bound is never past MostJudgeBound. func TestALongJudgementIsBoundedAndSaysWhyItFailed(t *testing.T) { j := testJudge(true, true, func(ctx context.Context, _ []string, _ []byte) ([]byte, error) { <-ctx.Done() return nil, ctx.Err() }) if _, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema, Within: MostJudgeBound + time.Second}); err == nil || !strings.Contains(err.Error(), "at most") { t.Fatalf("a bound past the most taken: %v", err) } j.Wait = 20 * time.Millisecond j.Bound = 30 * time.Millisecond // the test's stand-in for JudgeBound: within is past it, so a ticket out, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema, Within: 200 * time.Millisecond}) if err != nil || !out.Pending { t.Fatalf("%+v %v", out, err) } for i := 0; i < 100; i++ { got, err := j.Collect(out.Ticket) if err != nil { if !strings.Contains(err.Error(), "did not answer within") { t.Fatalf("%v", err) } return } if !got.Pending { t.Fatalf("answered: %+v", got) } } t.Fatal("never ended") } // The tools take within and ticket, and nothing else. func TestTheToolsTakeWithinAndTicket(t *testing.T) { j := testJudge(true, true, func(context.Context, []string, []byte) ([]byte, error) { return []byte(`{"structured_output":{"verdict":"none"}}`), nil }) out, err := judgeTool(j).Run(map[string]any{"question": "q", "schema": verdictSchema, "within": float64(60)}) if err != nil { t.Fatal(err) } if got := out.(*Judged); got.Pending || string(got.Answer) != `{"verdict":"none"}` { t.Fatalf("a quick long judgement is answered in its call: %+v", got) } if _, err := judgeTool(j).Run(map[string]any{"question": "q", "schema": verdictSchema, "within": "long"}); err == nil { t.Fatal("within as text was taken") } if _, err := judgedTool(j).Run(map[string]any{"ticket": "nope"}); err == nil || !strings.Contains(err.Error(), "no judgement") { t.Fatalf("%v", err) } if _, err := judgedTool(j).Run(map[string]any{"ticket": "x", "by": "y"}); err == nil { t.Fatal("an unknown argument was taken") } }