CAT-CC1: a judgement asked with a ticket says one judged when it ends, with its answer or why it failed
Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
This commit is contained in:
@@ -182,6 +182,9 @@ func TestALongJudgementIsCollectedByItsTicket(t *testing.T) {
|
||||
}
|
||||
})
|
||||
j.Wait = 50 * time.Millisecond
|
||||
said := make(chan JudgedEvent, 4)
|
||||
ids := make(chan string, 4)
|
||||
j.Say = func(e JudgedEvent, id string) error { said <- e; ids <- id; return nil }
|
||||
start := time.Now()
|
||||
out, err := j.Judge(JudgeAsk{Question: "a long record", Schema: verdictSchema, Within: 90 * time.Second})
|
||||
if err != nil {
|
||||
@@ -203,6 +206,21 @@ func TestALongJudgementIsCollectedByItsTicket(t *testing.T) {
|
||||
if got.Pending || string(got.Answer) != `{"verdict":"none"}` || got.Ticket != out.Ticket {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
// One `judged` per ticket, with the answer, under an id of the ticket's own; none for a judgement with no ticket.
|
||||
select {
|
||||
case e := <-said:
|
||||
if e.Ticket != out.Ticket || e.Node != "control" || string(e.Answer) != `{"verdict":"none"}` || e.Error != "" || e.Model != "sonnet" {
|
||||
t.Fatalf("judged: %+v", e)
|
||||
}
|
||||
if id := <-ids; id != "judged-control-"+out.Ticket {
|
||||
t.Fatalf("judged under %q", id)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the ended judgement was not said")
|
||||
}
|
||||
if len(said) != 0 {
|
||||
t.Fatal("judged said twice")
|
||||
}
|
||||
// Read once the judgement is collected: its end is what orders the agent's write before this read.
|
||||
if left := deadline.Sub(start); left < 80*time.Second || left > 91*time.Second {
|
||||
t.Fatalf("the judgement's bound is %s, not the 90s asked", left)
|
||||
@@ -224,10 +242,23 @@ func TestALongJudgementIsBoundedAndSaysWhyItFailed(t *testing.T) {
|
||||
}
|
||||
j.Wait = 20 * time.Millisecond
|
||||
j.Bound = time.Second // the test's stand-in for JudgeBound: within is past it, so a ticket; far past Wait on a loaded machine
|
||||
failed := make(chan JudgedEvent, 1)
|
||||
j.Say = func(e JudgedEvent, _ string) error { failed <- e; return nil }
|
||||
out, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema, Within: 2 * time.Second})
|
||||
if err != nil || !out.Pending {
|
||||
t.Fatalf("%+v %v", out, err)
|
||||
}
|
||||
defer func() {
|
||||
// A failed judgement is said too, with why and no answer.
|
||||
select {
|
||||
case e := <-failed:
|
||||
if e.Ticket != out.Ticket || e.Answer != nil || !strings.Contains(e.Error, "did not answer within") {
|
||||
t.Errorf("judged: %+v", e)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Error("the failed judgement was not said")
|
||||
}
|
||||
}()
|
||||
for i := 0; i < 3000; i++ {
|
||||
got, err := j.Collect(out.Ticket)
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user