133 lines
5.4 KiB
Go
133 lines
5.4 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// novox/hq issue 508: ADR 0297, eleven clauses and eleven links, could not be reviewed within the judgement's 25
|
|
// seconds five times running, while short records were reviewed in 14 to 25. A review's bound scales with what is
|
|
// judged, up to a cap, and a judgement past one bus call's time is collected by its ticket.
|
|
|
|
// aRecord is a decision of n clauses, each about size bytes.
|
|
func aRecord(n, size int) *Record {
|
|
r := &Record{Kind: KindDecision, Number: 297, Title: "Issues are kept as a log of operations", Status: StatusProposed}
|
|
for i := 1; i <= n; i++ {
|
|
r.Clauses = append(r.Clauses, Clause{N: i, Text: fmt.Sprintf("**Clause %d.** %s", i, strings.Repeat("A word of reasoning. ", size/20))})
|
|
}
|
|
r.Sections = []Section{{Name: "Context", Text: strings.Repeat("Why it was decided. ", n*size/40)}}
|
|
return r
|
|
}
|
|
|
|
// asked is the agent module, played over the calls the knowledge base makes to it.
|
|
type asked struct {
|
|
keys []string
|
|
bodies []map[string]any
|
|
pending int // how many times a ticketed judgement answers pending before its answer
|
|
refuse bool
|
|
}
|
|
|
|
func (a *asked) ask(key string, body any) (json.RawMessage, error) {
|
|
b, _ := body.(map[string]any)
|
|
a.keys, a.bodies = append(a.keys, key), append(a.bodies, b)
|
|
if strings.HasPrefix(key, "claude-code.claude_code_judge@") {
|
|
if _, long := b["within"]; long && a.refuse {
|
|
return nil, errors.New("claude_code_judge takes question, schema and model; not within")
|
|
}
|
|
if _, long := b["within"]; long && a.pending > 0 {
|
|
return json.RawMessage(`{"node":"novox","ticket":"t1","pending":true}`), nil
|
|
}
|
|
return json.RawMessage(`{"answer":{"verdict":"approve","findings":[]},"model":"sonnet","node":"novox"}`), nil
|
|
}
|
|
if key == "claude-code.claude_code_judged@novox" && b["ticket"] == "t1" {
|
|
if a.pending--; a.pending > 0 {
|
|
return json.RawMessage(`{"node":"novox","ticket":"t1","pending":true}`), nil
|
|
}
|
|
return json.RawMessage(`{"answer":{"verdict":"approve","findings":[]},"model":"sonnet","node":"novox","ticket":"t1"}`), nil
|
|
}
|
|
return nil, fmt.Errorf("nothing answers %s", key)
|
|
}
|
|
|
|
func withinOf(t *testing.T, b map[string]any) time.Duration {
|
|
t.Helper()
|
|
v, ok := b["within"]
|
|
if !ok {
|
|
return 0
|
|
}
|
|
s, ok := v.(int)
|
|
if !ok {
|
|
t.Fatalf("within is not whole seconds: %#v", v)
|
|
}
|
|
return time.Duration(s) * time.Second
|
|
}
|
|
|
|
func TestALongRecordsReviewGetsTheLargerBound(t *testing.T) {
|
|
short, long := ReviewQuestion(aRecord(2, 200), nil), ReviewQuestion(aRecord(12, 2000), nil)
|
|
a := &asked{}
|
|
j := judgementOver(a.ask, "novox", newJudgedHub())
|
|
for _, q := range []string{short, long} {
|
|
if _, _, err := j(q, reviewSchema, "sonnet"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
s, l := withinOf(t, a.bodies[0]), withinOf(t, a.bodies[1])
|
|
if l <= s || l <= 25*time.Second || l > MostJudgeWithin {
|
|
t.Fatalf("a %d-byte review is bounded %s and a %d-byte one %s: the longer must get the larger bound, past 25s and at most %s",
|
|
len(short), s, len(long), l, MostJudgeWithin)
|
|
}
|
|
if got := JudgeWithin(strings.Repeat("x", MostReviewQuestion)); got != MostJudgeWithin {
|
|
t.Fatalf("the largest question is bounded %s, not the cap %s", got, MostJudgeWithin)
|
|
}
|
|
}
|
|
|
|
// A judgement answered pending waits for the agent module's word `judged` under its ticket, whether the word comes
|
|
// after the answer or before it, and asks nothing again; its error is the judgement's, and a word that never comes
|
|
// ends the wait in words.
|
|
func TestAPendingJudgementWaitsForTheWordOfItsTicket(t *testing.T) {
|
|
word := `{"ticket":"t1","node":"novox","model":"sonnet","answer":{"verdict":"approve","findings":[]}}`
|
|
for name, c := range map[string]struct{ before bool }{"after": {false}, "before": {true}} {
|
|
a := &asked{pending: 1}
|
|
hub := newJudgedHub()
|
|
if c.before {
|
|
hub.Heard(json.RawMessage(word))
|
|
} else {
|
|
time.AfterFunc(50*time.Millisecond, func() { hub.Heard(json.RawMessage(word)) })
|
|
}
|
|
answer, by, err := judgementOver(a.ask, "novox", hub)(ReviewQuestion(aRecord(12, 2000), nil), reviewSchema, "sonnet")
|
|
if err != nil {
|
|
t.Fatalf("%s: %v", name, err)
|
|
}
|
|
if string(answer) != `{"verdict":"approve","findings":[]}` || by != "the agent on novox (sonnet)" {
|
|
t.Fatalf("%s: %s by %s", name, answer, by)
|
|
}
|
|
if strings.Join(a.keys, " ") != "claude-code.claude_code_judge@novox" {
|
|
t.Fatalf("%s: asked %q", name, a.keys)
|
|
}
|
|
}
|
|
hub := newJudgedHub()
|
|
hub.Heard(json.RawMessage(`{"ticket":"t1","node":"novox","error":"the agent on novox did not answer within 90s"}`))
|
|
if _, _, err := judgementOver((&asked{pending: 1}).ask, "novox", hub)(ReviewQuestion(aRecord(12, 2000), nil), reviewSchema, "sonnet"); err == nil ||
|
|
!strings.Contains(err.Error(), "did not answer within") {
|
|
t.Fatalf("the judgement's own failure: %v", err)
|
|
}
|
|
if _, err := newJudgedHub().Wait("novox", "t2", 20*time.Millisecond); err == nil || !strings.Contains(err.Error(), "t2") {
|
|
t.Fatalf("a word never said: %v", err)
|
|
}
|
|
}
|
|
|
|
// An agent module from before issue 508, which takes no within, is asked again without it: the review is had when
|
|
// it fits the old bound, and fails as before when it does not.
|
|
func TestAnAgentModuleThatTakesNoWithinIsAskedWithoutIt(t *testing.T) {
|
|
a := &asked{refuse: true}
|
|
if _, _, err := judgementOver(a.ask, "novox", newJudgedHub())(ReviewQuestion(aRecord(12, 2000), nil), reviewSchema, "sonnet"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(a.bodies) != 2 || withinOf(t, a.bodies[1]) != 0 {
|
|
t.Fatalf("asked %v", a.bodies)
|
|
}
|
|
}
|