Compare commits

..
19 changed files with 290 additions and 134 deletions
@@ -193,6 +193,24 @@ func Tools(r check.Reader) []stdio.Tool {
n, _ := args["number"].(float64)
return r.Pull(repository, int(n))
}},
{Name: check.FailureReadTool,
Description: "The full output of a pull request's failed test, which the verdict and its mesh/check status cut to their last lines " +
"(\"... (N earlier lines)\"): the test, the words the verdict keeps, the build step that holds the rest (its id, build module, " +
"machine, the check it ran for, and the call that reads it) and the first page of its build log. Read on with from = the " +
"output's next. Answers from the pull request's last failed verdict also once no candidate holds it. Read-only.",
Input: map[string]any{"type": "object", "required": []string{"repository", "number"}, "properties": map[string]any{
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
"number": map[string]any{"type": "integer", "description": "the pull request's number"},
"from": map[string]any{"type": "integer", "description": "the log's stream sequence to read on from: the output's next; 0 is the start"}}},
Run: func(args map[string]any) (any, error) {
repository, _ := args["repository"].(string)
n, _ := args["number"].(float64)
from, _ := args["from"].(float64)
if from < 0 {
return nil, fmt.Errorf("from is a stream sequence, 0 or more")
}
return r.Failure(repository, int(n), uint64(from))
}},
{Name: check.KeptReadTool,
Description: "The kept verdicts of passing packages, each as stored under kept.<level>.<digest>: the package, its tests, the time they took, the toolchain " +
"and when it was kept. With a level, that level's; with a level and a digest, that one. Read-only.",
@@ -229,8 +247,7 @@ func Tools(r check.Reader) []stdio.Tool {
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). held_by_turn names the candidates whose check an approved " +
"merge's turn holds off (novox/hq ADR 0548 §3), with the repository and the merge that waits; the gate's turns tool reads every turn and its " +
"hand-now tool steers one. dead_letters says whether the checker hears the asks the build seats give up on (issue 734), and why not: " +
"deaf, a try or check whose ask is given up waits until its bound. judging names the candidates whose newest step ended and whose end " +
"the checker judges now (levels, kept verdicts, composition): their verdict follows by itself; lost names those nothing will judge. Read-only.",
"deaf, a try or check whose ask is given up waits until its bound. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) { return r.Status() }},
}
@@ -244,9 +261,8 @@ func RecoveryTools(r check.Reader, w check.VerdictEntry) []stdio.Tool {
Description: "Ask a candidate's check again by hand, whose verdict stays running while nothing runs for it (" + check.StatusReadTool + " names it " +
"under lost): its verdict is ended as asked again, saying why, and the checker, which hears its verdicts, stops the wait that judged nothing " +
"and asks the check anew, whole. Refused for a group with no candidate, a verdict that is not running (a pass or failure stands until a push or " +
"a trunk move), while a step of its pull requests can still run on a build module (its end judges it), and while the checker judges the end " +
"of its newest step (" + check.StatusReadTool + " names it under judging: its verdict follows). The checker reads a step that ended while " +
"its own watch was deaf by itself at the next step end of any step of its build module; this is for what that does not reach.",
"a trunk move), and while a step of its pull requests can still run on a build module (its end judges it). The checker reads a step that " +
"ended while its watch was deaf by itself at the next step end of its build module; this is for what that does not reach.",
Input: map[string]any{"type": "object", "required": []string{"group", "why"}, "properties": map[string]any{
"group": map[string]any{"type": "string", "description": "the delivery group's branch name"},
"why": map[string]any{"type": "string", "description": "why it is asked again, in words; kept in the verdict it ends"}}},
@@ -271,7 +287,8 @@ func main() {
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound(),
TurnState: read(stdio.State("mesh-delivery-gate.turns")), HearingState: read(stdio.State(buildrun.HearingState)),
AskState: read(stdio.State(check.AsksState))}
r.Runs = runsOf{read(stdio.State("mesh-delivery-indexer.modules"))} // the node-engine at the commit the mesh runs, as the checker reads it
r.Logs = func(tool string, args map[string]any) (json.RawMessage, error) { return stdio.Ask(tool, args) } // the module's invokes
r.Runs = runsOf{read(stdio.State("mesh-delivery-indexer.modules"))} // the node-engine at the commit the mesh runs, as the checker reads it
tries := stdio.State(check.TriesState)
source := stdio.State("mesh-delivery-gate.source")
trunks := func() (map[string]string, error) {
@@ -11,7 +11,7 @@ import (
"mesh-delivery-checker/internal/check"
)
func TestTheToolsServedAreTheFiveReadTools(t *testing.T) {
func TestTheToolsServedAreTheSixReadTools(t *testing.T) {
var names []string
for _, tl := range Tools(check.Reader{}) {
names = append(names, tl.Name)
@@ -19,7 +19,7 @@ func TestTheToolsServedAreTheFiveReadTools(t *testing.T) {
t.Errorf("%s has no description or no run", tl.Name)
}
}
want := []string{check.VerdictsReadTool, check.PullReadTool, check.KeptReadTool, check.PreviewsReadTool, check.StatusReadTool}
want := []string{check.VerdictsReadTool, check.PullReadTool, check.FailureReadTool, check.KeptReadTool, check.PreviewsReadTool, check.StatusReadTool}
if !slices.Equal(names, want) {
t.Errorf("tools = %v, want %v", names, want)
}
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-checker
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030100935-c2ec8b6b2fde
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+4
View File
@@ -1,5 +1,9 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 h1:Tt16HJk8JZRce1i3JzzDSNdCWOrv7dw8FQ/bckNQ5Ig=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030095816-530d8d3007dc h1:Wdjl2gomooHswJ5qpkwZPq/ureZgXVw6FMCsgzPJhPo=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030095816-530d8d3007dc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030100935-c2ec8b6b2fde h1:eKXSllHCS31GgMazfvZ7BeiLoXIUibnUmzxWVkj+Vcw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030100935-c2ec8b6b2fde/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -6,7 +6,6 @@ import (
"errors"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -30,18 +29,6 @@ const (
// CheckAsks are the asks a check put, in order, as the state asks keeps them while it runs; its end deletes the key.
type CheckAsks struct {
Asks []string `json:"asks"`
// Judging names the step whose end the check heard and judges now (its levels, kept verdicts, the composition), until
// its next ask or its verdict: between a step's end and its verdict no step of the check runs, and without it the read
// tools took that judging for a check nothing will judge (issue 1721: mesh-catalog #562 was named lost ten seconds
// after its step passed, and checker_ask_again ended the judging that would have passed it). Nil while an ask waits.
Judging *Judging `json:"judging,omitempty"`
}
// Judging is the step whose end a running check heard and judges now, and since when.
type Judging struct {
Step string `json:"step"`
Node string `json:"node"`
Since time.Time `json:"since"`
}
// AsksKV is the state asks: a KV whose key a check's end deletes, so the state holds only the checks that run.
@@ -58,14 +45,13 @@ func (ch *Checker) forgetAsks(ctx context.Context, key string) {
}
}
// recordAsks keeps the asks a check put under its verdict key, with the step whose end it judges now (nil while an ask
// waits); nil Asks keeps nothing. A record that cannot be written is said: checker_ask_again then refuses the check, as it
// cannot tell whether its ask waits in a stream.
func (ch *Checker) recordAsks(ctx context.Context, key string, asks []string, judging *Judging) {
// recordAsks keeps the asks a check put under its verdict key; nil Asks keeps nothing. A record that cannot be written
// is said: checker_ask_again then refuses the check, as it cannot tell whether its ask waits in a stream.
func (ch *Checker) recordAsks(ctx context.Context, key string, asks []string) {
if ch.Asks == nil {
return
}
data, err := json.Marshal(CheckAsks{Asks: asks, Judging: judging})
data, err := json.Marshal(CheckAsks{Asks: asks})
if err == nil {
_, err = putOnce(ctx, ch.Asks, key, data, func(cur []byte) bool { return string(cur) == string(data) })
}
@@ -153,9 +139,6 @@ func (r Reader) AskAgain(w VerdictEntry, group, why string) (AskedAgain, error)
if err != nil || len(put.Asks) == 0 {
return AskedAgain{}, fmt.Errorf("the checker names no ask of %s (%v), so whether one waits in a build seat's stream cannot be told; asking again could run it twice", key, err)
}
if why := r.judgingWhy(put.Judging); why != "" {
return AskedAgain{}, fmt.Errorf("%s: asking again would end that judging and run the check twice", why)
}
taken := map[string]bool{}
for _, s := range steps.all {
taken[s.step.ID] = true
@@ -749,11 +749,8 @@ func (ch *Checker) try(ctx, run context.Context, key string, cs *Case) {
began := ch.now()
if cs.Watch == nil && ch.Asks != nil {
cs.Watch = func(_ string, step *delivery.Step) {
switch {
case step == nil: // put: kept before its wait starts
ch.recordAsks(ctx, cs.Key, cs.Asks(), nil)
case step.State != delivery.StepRunning: // its end, heard: judged from now until the next ask or the verdict
ch.recordAsks(ctx, cs.Key, cs.Asks(), &Judging{Step: step.ID, Node: step.Node, Since: ch.now().UTC()})
if step == nil { // put: kept before its wait starts
ch.recordAsks(ctx, cs.Key, cs.Asks())
}
}
defer ch.forgetAsks(ctx, cs.Key)
@@ -14,21 +14,10 @@ type fakeCompositions struct {
v delivery.CompositionVerdict
err error
asked []string
// hold, when set, is handed a value when Await is entered and holds it until it is handed one back: a check judging
// its step's end, its verdict not written yet.
hold chan struct{}
}
func (f *fakeCompositions) Await(ctx context.Context, key string) (delivery.CompositionVerdict, error) {
func (f *fakeCompositions) Await(_ context.Context, key string) (delivery.CompositionVerdict, error) {
f.asked = append(f.asked, key)
if f.hold != nil {
f.hold <- struct{}{}
select {
case <-f.hold:
case <-ctx.Done():
return delivery.CompositionVerdict{}, ctx.Err()
}
}
return f.v, f.err
}
@@ -432,6 +432,8 @@ func TestTheChangesOwnFailingTestRunsFirstThenTheRestAndNoLaterLevel(t *testing.
b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"}
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}}
failing := captured[Tested](t, "go-test-failing")
// The step said the failure's whole output on its log (the operator, 2026-10-30): the verdict names where.
failing.Failed[0].Step, failing.Failed[0].Module, failing.Failed[0].Machine, failing.Failed[0].Run = "s1", "mesh-build-go", "ace", "r1"
b.test = func(c stepCall) (any, error) {
if got := packagesOf(c); len(got) == 1 && got[0] == "m/internal/b" {
return failing, nil
@@ -439,7 +441,8 @@ func TestTheChangesOwnFailingTestRunsFirstThenTheRestAndNoLaterLevel(t *testing.
return captured[Tested](t, "go-test-slow"), nil
}
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
if out.Verdict.State != delivery.VerdictFail || !strings.Contains(out.Verdict.Failed.Test, "TestThreeIsFour") {
if f := out.Verdict.Failed; out.Verdict.State != delivery.VerdictFail || !strings.Contains(f.Test, "TestThreeIsFour") ||
f.Step != "s1" || f.Module != "mesh-build-go" || f.Machine != "ace" || f.Run != "r1" || f.Pull != repoA+"#7" || out.Verdict.Failures[0].Step != "s1" {
t.Fatalf("verdict = %+v", out.Verdict)
}
tests := b.steps.tool("test")
@@ -0,0 +1,132 @@
package check
import (
"encoding/json"
"fmt"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// A pull request's failed test, its whole output read from the build step that said it (the operator, 2026-10-30: the
// verdict cuts a failure to its last lines, and its cause was guessed). The verdict names the step, its build module and
// machine and the check it ran for (delivery.FailedFirst); the build module's log tool reads the step's log from its own
// stream (mesh-sdk buildstep.LogTool). The verdict is found through the pull request's candidate, or, once no candidate
// holds it, as the newest verdict that names the pull request's failure.
// FailureStep is where a failure's whole output is: the step id, its build module and machine, the run it ran for, and the
// call that reads it.
type FailureStep struct {
ID string `json:"id"`
Module string `json:"module"`
Machine string `json:"machine,omitempty"`
Run string `json:"run"`
Read string `json:"read"`
}
// FailureRead is a pull request's failed test: the verdict that names it, the test and the words the verdict keeps, the
// step that holds the rest, and the first page of its log (Output, the log tool's answer), or why there is none.
type FailureRead struct {
Pull string `json:"pull"`
Verdict string `json:"verdict"`
Test string `json:"test,omitempty"`
Words string `json:"words,omitempty"`
Step *FailureStep `json:"step,omitempty"`
Output json.RawMessage `json:"output,omitempty"`
Note string `json:"note,omitempty"`
}
// logToolOf is a build module's log tool: mesh-build-go's build_go_log.
func logToolOf(module string) string {
lang := strings.TrimPrefix(module, "mesh-build-")
return module + ".build_" + lang + "_log"
}
// failureOf is the failure a verdict names of the pull request pull: the first that names it, or, in a verdict that names
// no pull request (written before), the first failure.
func failureOf(v delivery.Verdict, pull string) (delivery.FailedFirst, bool) {
for _, f := range append([]delivery.FailedFirst{v.Failed}, v.Failures...) {
if f.Pull == pull && f.Test != "" {
return f, true
}
}
if v.Failed.Pull == "" && v.Failed.Test != "" {
return v.Failed, true
}
return delivery.FailedFirst{}, false
}
// Failure answers a pull request's failed test, with the page of its log from the stream sequence from on (0 the start).
func (r Reader) Failure(repository string, number int, from uint64) (FailureRead, error) {
pull := fmt.Sprintf("%s#%d", repository, number)
out := FailureRead{Pull: pull}
var v delivery.Verdict
found := false
if read, err := r.Pull(repository, number); err == nil && len(read.Verdict) > 0 {
if v, err = delivery.DecodeVerdict(read.Verdict); err != nil {
return out, fmt.Errorf("%s holds what is not a verdict: %w", read.Key, err)
}
out.Verdict, found = read.Key, true
} else if err != nil && !strings.HasPrefix(err.Error(), "no candidate") {
return out, err
}
if _, ok := failureOf(v, pull); !found || v.State != delivery.VerdictFail || !ok {
// No candidate holds it, or its candidate's verdict did not fail of it: the newest verdict that names its failure.
key, last, err := r.lastFailure(pull)
if err != nil {
return out, err
}
if key == "" {
if found {
out.Note = "its verdict names no failed test of it, and no other verdict does"
return out, nil
}
return out, fmt.Errorf("no candidate holds %s, and no verdict names a failed test of it", pull)
}
out.Verdict, v = key, last
}
f, _ := failureOf(v, pull)
out.Test, out.Words = f.Test, f.Words
if f.Step == "" {
out.Note = "the verdict names no build step that kept the whole output (a build module or a checker from before it)"
return out, nil
}
out.Step = &FailureStep{ID: f.Step, Module: f.Module, Machine: f.Machine, Run: f.Run,
Read: fmt.Sprintf("%s {run: %s, step: %s}", logToolOf(f.Module), f.Run, f.Step)}
if r.Logs == nil {
out.Note = "the build module's log cannot be asked here"
return out, nil
}
page, err := r.Logs(logToolOf(f.Module), map[string]any{"run": f.Run, "step": f.Step, "from": from})
if err != nil {
out.Note = "its log could not be read: " + err.Error()
return out, nil
}
out.Output = page
return out, nil
}
// lastFailure is the newest verdict that names a failed test of pull, by its step id (an id begins with its time), and
// its key; "" when none does among the first MaxReads.
func (r Reader) lastFailure(pull string) (string, delivery.Verdict, error) {
rows, _, err := r.Verdicts("")
if err != nil {
return "", delivery.Verdict{}, err
}
var key, newest string
var last delivery.Verdict
for _, row := range rows {
v, err := delivery.DecodeVerdict(row.Verdict)
if err != nil || v.State != delivery.VerdictFail {
continue
}
f, ok := failureOf(v, pull)
if !ok || f.Pull != pull {
continue
}
if key == "" || len(f.Step) > len(newest) || (len(f.Step) == len(newest) && f.Step > newest) {
key, newest, last = row.Key, f.Step, v
}
}
return key, last, nil
}
@@ -318,6 +318,15 @@ func (st *state) outcome(v delivery.Verdict) Outcome {
return out
}
// runOf is the run a case's asks name (delivery.StepAsk.Run), under which their steps say their logs: the check's heads
// hash, or delivery.NoRun when it is no token.
func runOf(c *Case) string {
if c == nil || c.HeadsHash == "" || strings.ContainsAny(c.HeadsHash, ".*> ") {
return delivery.NoRun
}
return c.HeadsHash
}
// fail is a verdict of failure at a machine.
func fail(machine, words string) *delivery.Verdict {
v := failure(machine, words)
@@ -430,7 +439,7 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
kind = delivery.AskRecheck // a re-judge after a trunk move goes after every new head's check (ADR 0536 §4)
}
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: askLanguage[t.module], Asker: delivery.CheckerModule,
At: st.j.now().UTC(), Check: job, Needs: needs}
At: st.j.now().UTC(), Check: job, Needs: needs, Run: runOf(st.c)}
var step delivery.Step
var firstDecline time.Time
for {
@@ -831,7 +840,12 @@ func levelFailure(t *target, level string, results ...Tested) *delivery.Verdict
if t.module == buildrun.NodeBuildModule && f.Package == "" {
name = res.Module
}
all = append(all, delivery.FailedFirst{Test: level + ": " + name, Words: failureWordsAfter(f, hidWords(t.module, res.By, f))})
ff := delivery.FailedFirst{Test: level + ": " + name, Words: failureWordsAfter(f, hidWords(t.module, res.By, f)),
Pull: fmt.Sprintf("%s#%d", t.pull.Repository, t.pull.Number)}
if f.Step != "" { // its build step said its whole output on the module's log: where (checker_failure reads it)
ff.Step, ff.Module, ff.Machine, ff.Run = f.Step, f.Module, f.Machine, f.Run
}
all = append(all, ff)
}
}
if len(all) == 0 { // no result: a level said failed with nothing to name
@@ -848,7 +862,8 @@ func failureLines(all []delivery.FailedFirst) []delivery.FailedFirst {
n := min(len(all), delivery.MaxFailures)
out := make([]delivery.FailedFirst, 0, n+1)
for _, f := range all[:n] {
out = append(out, delivery.FailedFirst{Test: f.Test, Machine: f.Machine, Words: clipLine(firstLine(f.Words), MaxFailureLine)})
f.Words = clipLine(firstLine(f.Words), MaxFailureLine)
out = append(out, f)
}
if more := len(all) - n; more > 0 {
out = append(out, delivery.FailedFirst{Test: fmt.Sprintf("and %d more", more)})
@@ -312,7 +312,7 @@ func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []s
// Report is the full line of a pull request's status, with no limit: what the read tools and the full report say.
func Report(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string, judged map[string]string) string {
return parts(v, cs, p, builds, slowest, judged).join(0)
return joinSet(parts(v, cs, p, builds, slowest, judged).join(0), fullOutputWords(v, p))
}
// Line is the words of a status for one pull request of a case (ADR 0447 clause 3, ADR 0484 clause 7): the verdict first (a
@@ -324,26 +324,52 @@ func Report(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []
// The program that judged is never dropped: a line naming it gives the counts in their compact form, then drops them and
// what follows, and is cut in its first part rather than lose it.
func Line(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string, judged map[string]string) string {
pc := parts(v, cs, p, builds, slowest, judged)
full := fullOutputWords(v, p)
if full == "" {
return lineWithin(parts(v, cs, p, builds, slowest, judged), MaxLine)
}
return lineWithin(parts(v, cs, p, builds, slowest, judged), MaxLine-utf8.RuneCountInString(full)-3) + " | " + full
}
// FullOutputTool is the read tool that answers a failed test's whole output (the checker's tools).
const FullOutputTool = "checker_failure"
// fullOutputWords end the line of a pull request whose failed test's whole output a build step said on its log: where to
// read it, "full output: checker_failure <repo>#<n>"; "" when none was said (no test failed, or an older build module).
func fullOutputWords(v delivery.Verdict, p Pull) string {
if v.State != delivery.VerdictFail {
return ""
}
pull := fmt.Sprintf("%s#%d", p.Repository, p.Number)
for _, f := range append([]delivery.FailedFirst{v.Failed}, v.Failures...) {
if f.Step != "" && (f.Pull == "" || f.Pull == pull) {
return "full output: " + FullOutputTool + " " + pull
}
}
return ""
}
// lineWithin is the line of the pieces in at most most characters (Line).
func lineWithin(pc pieces, most int) string {
for drop := 0; drop <= 4; drop++ {
if s := pc.join(drop); utf8.RuneCountInString(s) <= MaxLine {
if s := pc.join(drop); utf8.RuneCountInString(s) <= most {
return s
}
}
if pc.judged != "" {
for _, counts := range []string{pc.shortCounts, ""} {
if s := joinSet(pc.head, pc.commits, counts, pc.judged); utf8.RuneCountInString(s) <= MaxLine {
if s := joinSet(pc.head, pc.commits, counts, pc.judged); utf8.RuneCountInString(s) <= most {
return s
}
}
rest := " | " + joinSet(pc.commits, pc.judged)
room := MaxLine - utf8.RuneCountInString(rest) - 1
room := most - utf8.RuneCountInString(rest) - 1
if head := []rune(pc.head); room > 0 && len(head) > room {
return string(head[:room]) + "…" + rest
}
}
runes := []rune(pc.join(4))
return string(runes[:MaxLine-1]) + "…"
return string(runes[:most-1]) + "…"
}
// joinSet joins the parts that are not empty.
@@ -50,11 +50,18 @@ func TestAFailureNamesTheTestTheConflictOrTheRefusedStepFirst(t *testing.T) { //
"test": {delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Test: "unit: m/a.TestX", Words: "want 4"}}, "failed: unit: m/a.TestX: want 4"},
"conflict": {delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "shanks", Words: "novox/mesh-delivery#7 does not merge into the trunk: files in conflict: go.mod"}}, "failed: shanks: novox/mesh-delivery#7 does not merge into the trunk: files in conflict: go.mod"},
"refused": {delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "mesh-build-go", Words: "no go build node qualifies: builders disagree"}}, "failed: mesh-build-go: no go build node qualifies"},
// The operator, 2026-10-30: a test whose step said its whole output ends the line saying where to read it.
"whole output": {delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Test: "unit: m/a.TestX", Words: "want 4\n... " + strings.Repeat("a long line ", 40),
Step: "s1", Module: "mesh-build-go", Run: "r1", Pull: "novox/mesh-delivery#7"}}, "failed: unit: m/a.TestX: want 4"},
} {
line := Line(c.v, cs, p, nil, nil, nil)
if !strings.HasPrefix(line, c.want) || !strings.Contains(line, "111111111111 on aaaaaaaaaaaa") {
t.Errorf("%s: line = %q, want it to open with %q and name the commits", name, line, c.want)
}
full := strings.HasSuffix(line, " | full output: checker_failure novox/mesh-delivery#7")
if full != (c.v.Failed.Step != "") || utf8.RuneCountInString(line) > MaxLine {
t.Errorf("%s: line = %q: the full output named %v, want %v, within %d characters", name, line, full, !full, MaxLine)
}
}
}
@@ -266,8 +266,6 @@ func TestACheckWhoseAskJudgedNothingIsAskedAgainOrRead(t *testing.T) {
{name: "its watch deaf, a step of its build module ends", deaf: 1, how: "nudge", state: delivery.VerdictPass, asks: 1},
{name: "its watch deaf, asked again by hand", deaf: 1, how: "hand", state: delivery.VerdictPass, asks: 2},
{name: "its watch deaf, asked again by hand after its build module replaced it", deaf: 1, restarts: 1, how: "hand", state: delivery.VerdictPass, asks: 2},
// issue 1721: mesh-catalog #562 was named lost while the checker judged its step's end, and asked again by hand
{name: "its step's end heard and being judged", how: "judging", state: delivery.VerdictPass, asks: 1},
} {
t.Run(row.name, func(t *testing.T) {
b := newBench(t)
@@ -284,11 +282,6 @@ func TestACheckWhoseAskJudgedNothingIsAskedAgainOrRead(t *testing.T) {
b.ex.Asks = buildrun.Asker{Publisher: tr, Waiter: heard, Bound: 5 * time.Second}
asked = func() int { return len(tr.Asks()) }
}
var held *fakeCompositions
if row.how == "judging" {
held = &fakeCompositions{err: ErrAssignerNotRunning, hold: make(chan struct{})}
b.ex.Compositions = held
}
r := newRig(t, b.ex)
asksKept := newMemKV()
r.ch.Asks = asksKept
@@ -304,24 +297,6 @@ func TestACheckWhoseAskJudgedNothingIsAskedAgainOrRead(t *testing.T) {
}
}
switch row.how {
case "judging":
select {
case <-held.hold: // its step ended and was heard: the checker judges it, its verdict not written yet
case <-time.After(5 * time.Second):
t.Fatal("the check never came to judge its step's end")
}
cands := newMemKV()
ckey, _ := delivery.CandidatesKey("feat/x")
_, _ = cands.Create(ctx, ckey, cand)
rd := Reader{CandidateState: kvRead(cands), VerdictState: kvRead(r.verdicts), AskState: kvRead(asksKept),
StepStates: map[string]StateRead{buildrun.GoBuildModule: tr.Steps(false)}}
if st, _ := rd.Status(); st.Lost["feat/x"] != "" || !strings.Contains(st.Judging["feat/x"], "judges the end of its step") {
t.Errorf("status while its step's end is judged: lost %v, judging %v; want judging, not lost", st.Lost, st.Judging)
}
if _, err := rd.AskAgain(asVerdicts{r.verdicts}, "feat/x", "named lost"); err == nil || !strings.Contains(err.Error(), "judges the end of its step") {
t.Errorf("asked again while its step's end is judged: %v", err)
}
held.hold <- struct{}{}
case "nudge":
heard.Nudge(buildrun.NodeBuildModule) // another build module's step: not this one's to read
heard.Nudge(buildrun.GoBuildModule)
@@ -18,7 +18,9 @@ import (
const (
VerdictsReadTool = "checker_verdicts"
PullReadTool = "checker_pull"
KeptReadTool = "checker_kept"
// FailureReadTool is FullOutputTool: a pull request's failed test with its whole output (Failure).
FailureReadTool = FullOutputTool
KeptReadTool = "checker_kept"
// PreviewsReadTool reads the previews state (issue 1680); StatusReadTool reads what the checker runs (issue 1685: nothing holds it since R4g).
PreviewsReadTool = "checker_previews"
StatusReadTool = "checker_status"
@@ -49,7 +51,10 @@ type Reader struct {
HearingState StateRead
// AskState is the checker's state asks: the asks each running check put (AsksState), which checker_ask_again reads.
AskState StateRead
Now func() time.Time
// Logs asks a build module's log tool (`<module>.build_<lang>_log`) as the checker (stdio.Ask): a failed test's whole
// output (Failure); nil asks none.
Logs func(tool string, args map[string]any) (json.RawMessage, error)
Now func() time.Time
}
// VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
@@ -338,10 +343,6 @@ type Status struct {
// Lost are the candidates whose verdict is running while no step of theirs can still run and their newest step ended, by
// group, with that step: nothing will judge them (the lost check of 2026-10-28); checker_ask_again asks one again.
Lost map[string]string `json:"lost"`
// Judging are the candidates whose verdict is running while the checker judges the end of their newest step (its
// levels, kept verdicts, the composition), by group, with that step and since when: their verdict follows by itself,
// and checker_ask_again refuses them (issue 1721).
Judging map[string]string `json:"judging"`
// DeadLetters says whether the checker hears the asks the build seats give up on (issue 734): deaf, a try or check
// whose ask is given up waits until its bound.
DeadLetters DeadLettersHeard `json:"dead_letters"`
@@ -405,7 +406,7 @@ func heldBy(t *Turns, c delivery.Candidate, heads map[string]string) string {
// Status reads the build modules' steps and the candidates against their verdicts.
func (r Reader) Status() (Status, error) {
st := Status{RunningByNode: map[string]int{}, Lost: map[string]string{}, Judging: map[string]string{}}
st := Status{RunningByNode: map[string]int{}, Lost: map[string]string{}}
read := r.readSteps()
st.Running, st.LeftRunning, st.Notes = read.running, read.left, read.notes
for _, rs := range st.Running {
@@ -491,9 +492,7 @@ func (r Reader) Status() (Status, error) {
switch {
case err != nil:
case v.State == delivery.VerdictRunning:
if judging := r.judgingWhy(r.judgingOf(c)); judging != "" {
st.Judging[c.Group] = judging
} else if why := read.lostWhy(c); why != "" {
if why := read.lostWhy(c); why != "" {
st.Lost[c.Group] = why
}
case IsNodesFailure(v):
@@ -628,41 +627,6 @@ func (s stepsRead) lostWhy(c delivery.Candidate) string {
last.ID, last.Node, last.State, last.Ended.UTC().Format(time.RFC3339), AskAgainTool)
}
// judgingOf is the step whose end the candidate's running check judges now, as the checker's state asks keeps it; nil
// for none, or when the state cannot be read.
func (r Reader) judgingOf(c delivery.Candidate) *Judging {
key, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
if err != nil || r.AskState.Get == nil {
return nil
}
raw, err := r.AskState.Get(key)
var put CheckAsks
if err != nil || len(raw) == 0 || json.Unmarshal(raw, &put) != nil {
return nil
}
return put.Judging
}
// judgingWhy says that the checker judges a step's end now, or "" for none: a judging older than the step bound is not
// counted, as the checker that heard that end stopped before its verdict (one that starts again takes the check anew and
// writes its asks again), so the check is lost.
func (r Reader) judgingWhy(j *Judging) string {
if j == nil {
return ""
}
now, bound := time.Now(), r.StepBound
if r.Now != nil {
now = r.Now()
}
if bound <= 0 {
bound = DefaultStepBound
}
if now.Sub(j.Since) > bound {
return ""
}
return fmt.Sprintf("the checker judges the end of its step %s on %s since %s; its verdict follows", j.Step, j.Node, j.Since.UTC().Format(time.RFC3339))
}
// DefaultStepBound is how long one check ask's step may run once a build node took it (its key written running; the fetch,
// the digests and the levels), never counting its wait in the stream; past it the check fails naming the node.
const DefaultStepBound = 30 * time.Minute
@@ -67,6 +67,41 @@ func TestTheVerdictToolsAnswerTheStoredVerdictWithTheFullReportOfEachPullRequest
if only, _, _ := rd.Verdicts("feat/other"); len(only) != 0 {
t.Errorf("another group's verdicts = %+v", only)
}
// checker_failure (the operator, 2026-10-30): a passing pull request has no failure to read; one no candidate holds any
// more is answered from its newest failed verdict, with the step that holds its whole output and that log's first page.
if got, err := rd.Failure(repoA, 7, 0); err != nil || got.Step != nil || !strings.Contains(got.Note, "names no failed test") {
t.Errorf("a passing pull request's failure: %+v, %v", got, err)
}
if _, err := rd.Failure("novox/none", 1, 0); err == nil || !strings.Contains(err.Error(), "no verdict names a failed test") {
t.Errorf("a pull request no candidate and no verdict holds: %v", err)
}
verdicts := fakeState{}
keys, _ := rd.VerdictState.Keys()
for _, k := range keys {
verdicts[k], _ = rd.VerdictState.Get(k)
}
for i, step := range []string{"mh0aaaaa000001", "mh0aaaab000002"} {
v := delivery.Verdict{V: 1, Group: "fix/y", Heads: fmt.Sprintf("%012d", i), State: delivery.VerdictFail,
Failed: delivery.FailedFirst{Test: "unit: m/a.TestX", Words: "want 4", Step: step, Module: "mesh-build-go", Machine: "ace", Run: "r1", Pull: "novox/y#9"}}
raw, err := delivery.EncodeVerdict(v)
if err != nil {
t.Fatal(err)
}
key, _ := delivery.CheckerVerdictKey(v.Group, v.Heads)
verdicts[key] = raw
}
rd.VerdictState = verdicts.read()
var asked string
rd.Logs = func(tool string, args map[string]any) (json.RawMessage, error) {
asked = fmt.Sprintf("%s %v %v %v", tool, args["run"], args["step"], args["from"])
return json.RawMessage(`{"lines":["=== m/a TestX","want 4"],"next":0}`), nil
}
got, err := rd.Failure("novox/y", 9, 0)
if err != nil || got.Step == nil || got.Step.ID != "mh0aaaab000002" || got.Step.Read != "mesh-build-go.build_go_log {run: r1, step: mh0aaaab000002}" ||
asked != "mesh-build-go.build_go_log r1 mh0aaaab000002 0" || !strings.Contains(string(got.Output), "want 4") {
t.Errorf("the failure of novox/y#9: %+v (%s), asked %q, %v", got, got.Output, asked, err)
}
}
// 1887/A10: an hq pull request is answered as any other, its report its verdict's (ADR 0527 §2), never a note that a script
@@ -101,10 +101,12 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) {
t.Errorf("reads = %v, want %v", m.Reads, want)
}
// 1887/A5b: the checker reads nothing of the controller's plans-building (ADR 0483 superseded by ADR 0525 §7), calls no
// build module's tool and holds no capacity: it puts check asks on the Go and Node build seats (ADR 0525 §1, §4).
if want := []string{"gitea.gitea_commits", "gitea.gitea_compare", "gitea.gitea_get_file", "gitea.gitea_list_branches"}; !reflect.DeepEqual(sorted(m.Invokes), want) {
t.Errorf("invokes = %v, want only the forge's reads for a mesh-sdk pin (ADR 0484 clause 9)", m.Invokes)
// 1887/A5b: the checker reads nothing of the controller's plans-building (ADR 0483 superseded by ADR 0525 §7), runs no
// build module's step tool and holds no capacity: it puts check asks on the Go and Node build seats (ADR 0525 §1, §4).
// Its checker_failure reads a failed step's whole output with the build modules' log tools (the operator, 2026-10-30).
if want := []string{"gitea.gitea_commits", "gitea.gitea_compare", "gitea.gitea_get_file", "gitea.gitea_list_branches",
"mesh-build-go.build_go_log", "mesh-build-node.build_node_log"}; !reflect.DeepEqual(sorted(m.Invokes), want) {
t.Errorf("invokes = %v, want the forge's reads for a mesh-sdk pin (ADR 0484 clause 9) and the build logs' reads", m.Invokes)
}
if want := []string{"node-build-go", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) {
t.Errorf("uses = %v, want the Go and Node build seats", m.Uses)
@@ -126,7 +128,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
// The read tools (issue 1686) are served by the tools bundle, not by the checker's process.
// The try's tools (ADR 0501 §5; the end by hand, issue 734) beside them: mesh-controller's try calls them. A lost check
// is asked again by hand (checker_ask_again).
if want := []string{"checker_ask_again", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
if want := []string{"checker_ask_again", "checker_failure", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
+4 -1
View File
@@ -40,6 +40,7 @@
],
"tools": [
"checker_ask_again",
"checker_failure",
"checker_kept",
"checker_previews",
"checker_pull",
@@ -53,7 +54,9 @@
"gitea.gitea_commits",
"gitea.gitea_compare",
"gitea.gitea_get_file",
"gitea.gitea_list_branches"
"gitea.gitea_list_branches",
"mesh-build-go.build_go_log",
"mesh-build-node.build_node_log"
],
"own-secrets": {
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-grants
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030100935-c2ec8b6b2fde
github.com/nats-io/nats.go v1.54.0
)
+4
View File
@@ -1,5 +1,9 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 h1:Tt16HJk8JZRce1i3JzzDSNdCWOrv7dw8FQ/bckNQ5Ig=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030095816-530d8d3007dc h1:Wdjl2gomooHswJ5qpkwZPq/ureZgXVw6FMCsgzPJhPo=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030095816-530d8d3007dc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030100935-c2ec8b6b2fde h1:eKXSllHCS31GgMazfvZ7BeiLoXIUibnUmzxWVkj+Vcw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030100935-c2ec8b6b2fde/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=