CAT-CC1: a judgement asked with a ticket says one judged when it ends, with its answer or why it failed
Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
This commit is contained in:
@@ -71,6 +71,17 @@ type Judged struct {
|
||||
Pending bool `json:"pending,omitempty"`
|
||||
}
|
||||
|
||||
// JudgedEvent is the body of `judged`: one per judgement asked with a bound past JudgeBound, said when it ends, with
|
||||
// its answer or why there is none. The question and the schema are never in it.
|
||||
type JudgedEvent struct {
|
||||
Ticket string `json:"ticket"`
|
||||
Node string `json:"node"`
|
||||
Model string `json:"model,omitempty"`
|
||||
Seconds float64 `json:"seconds,omitempty"`
|
||||
Answer json.RawMessage `json:"answer,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// running is one judgement asked with a bound past JudgeBound, until it is collected.
|
||||
type running struct {
|
||||
done chan struct{}
|
||||
@@ -92,6 +103,9 @@ type Judge struct {
|
||||
// Wait is how long a call waits for a long judgement before it answers its ticket; JudgeWait unless a test
|
||||
// says less.
|
||||
Wait time.Duration
|
||||
// Say tells the bus a ticketed judgement ended: the event `judged`, once, under the message id it is given so a
|
||||
// repeat is dropped. Nil says nothing (a test's judge); the ticket is collected by claude_code_judged all the same.
|
||||
Say func(body JudgedEvent, id string) error
|
||||
turn chan struct{}
|
||||
|
||||
mu sync.Mutex
|
||||
@@ -208,10 +222,28 @@ func (j *Judge) start(schema, model, question string, bound time.Duration) (*Jud
|
||||
go func() {
|
||||
r.out, r.err = j.run(schema, model, question, bound)
|
||||
close(r.done)
|
||||
j.said(ticket, r)
|
||||
}()
|
||||
return j.Collect(ticket)
|
||||
}
|
||||
|
||||
// said tells the bus that the judgement under a ticket ended (the event `judged`), once. A failure to say is logged: the
|
||||
// ticket stays collectible by claude_code_judged, and the asker's own bound ends its wait.
|
||||
func (j *Judge) said(ticket string, r *running) {
|
||||
if j.Say == nil {
|
||||
return
|
||||
}
|
||||
e := JudgedEvent{Ticket: ticket, Node: j.P.Node}
|
||||
if r.err != nil {
|
||||
e.Error = r.err.Error()
|
||||
} else {
|
||||
e.Model, e.Seconds, e.Answer = r.out.Model, r.out.Seconds, r.out.Answer
|
||||
}
|
||||
if err := j.Say(e, "judged-"+j.P.Node+"-"+ticket); err != nil {
|
||||
say("the judgement under ticket %s ended and could not be said on the bus: %v", ticket, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Collect waits as long as a call may for the judgement under a ticket: its answer, why it failed, or that it is
|
||||
// still pending. A ticket is answered once; an answer nobody collects is kept JudgedKept past its bound.
|
||||
func (j *Judge) Collect(ticket string) (*Judged, error) {
|
||||
|
||||
@@ -182,6 +182,9 @@ func TestALongJudgementIsCollectedByItsTicket(t *testing.T) {
|
||||
}
|
||||
})
|
||||
j.Wait = 50 * time.Millisecond
|
||||
said := make(chan JudgedEvent, 4)
|
||||
ids := make(chan string, 4)
|
||||
j.Say = func(e JudgedEvent, id string) error { said <- e; ids <- id; return nil }
|
||||
start := time.Now()
|
||||
out, err := j.Judge(JudgeAsk{Question: "a long record", Schema: verdictSchema, Within: 90 * time.Second})
|
||||
if err != nil {
|
||||
@@ -203,6 +206,21 @@ func TestALongJudgementIsCollectedByItsTicket(t *testing.T) {
|
||||
if got.Pending || string(got.Answer) != `{"verdict":"none"}` || got.Ticket != out.Ticket {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
// One `judged` per ticket, with the answer, under an id of the ticket's own; none for a judgement with no ticket.
|
||||
select {
|
||||
case e := <-said:
|
||||
if e.Ticket != out.Ticket || e.Node != "control" || string(e.Answer) != `{"verdict":"none"}` || e.Error != "" || e.Model != "sonnet" {
|
||||
t.Fatalf("judged: %+v", e)
|
||||
}
|
||||
if id := <-ids; id != "judged-control-"+out.Ticket {
|
||||
t.Fatalf("judged under %q", id)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the ended judgement was not said")
|
||||
}
|
||||
if len(said) != 0 {
|
||||
t.Fatal("judged said twice")
|
||||
}
|
||||
// Read once the judgement is collected: its end is what orders the agent's write before this read.
|
||||
if left := deadline.Sub(start); left < 80*time.Second || left > 91*time.Second {
|
||||
t.Fatalf("the judgement's bound is %s, not the 90s asked", left)
|
||||
@@ -224,10 +242,23 @@ func TestALongJudgementIsBoundedAndSaysWhyItFailed(t *testing.T) {
|
||||
}
|
||||
j.Wait = 20 * time.Millisecond
|
||||
j.Bound = time.Second // the test's stand-in for JudgeBound: within is past it, so a ticket; far past Wait on a loaded machine
|
||||
failed := make(chan JudgedEvent, 1)
|
||||
j.Say = func(e JudgedEvent, _ string) error { failed <- e; return nil }
|
||||
out, err := j.Judge(JudgeAsk{Question: "q", Schema: verdictSchema, Within: 2 * time.Second})
|
||||
if err != nil || !out.Pending {
|
||||
t.Fatalf("%+v %v", out, err)
|
||||
}
|
||||
defer func() {
|
||||
// A failed judgement is said too, with why and no answer.
|
||||
select {
|
||||
case e := <-failed:
|
||||
if e.Ticket != out.Ticket || e.Answer != nil || !strings.Contains(e.Error, "did not answer within") {
|
||||
t.Errorf("judged: %+v", e)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Error("the failed judgement was not said")
|
||||
}
|
||||
}()
|
||||
for i := 0; i < 3000; i++ {
|
||||
got, err := j.Collect(out.Ticket)
|
||||
if err != nil {
|
||||
|
||||
@@ -372,6 +372,10 @@ func tools(p Paths, servers ServerState, view *ServerView, config ConfigState, c
|
||||
out = append(out, proposalTools(p)...)
|
||||
out = append(out, waitTools(p)...)
|
||||
judge := NewJudge(p)
|
||||
// The event `judged`, one per ticket when its judgement ends: the asker hears it instead of asking again.
|
||||
judge.Say = func(e JudgedEvent, id string) error {
|
||||
return stdio.Emit("judged", e, stdio.EmitOptions{MessageID: id})
|
||||
}
|
||||
out = append(out, judgeTool(judge), judgedTool(judge))
|
||||
return append(out, configTools(p, config, configView)...)
|
||||
}
|
||||
|
||||
@@ -34,6 +34,9 @@
|
||||
"memory",
|
||||
"waits"
|
||||
],
|
||||
"emits": [
|
||||
"judged"
|
||||
],
|
||||
"reads": [
|
||||
"claude-licence-manager.bindings",
|
||||
"mesh-delivery-assigner.placement",
|
||||
|
||||
Reference in New Issue
Block a user