Compare commits

..
Author SHA1 Message Date
jschoubben b55a735a97 mesh-delivery-checker: Pin mesh-sdk at 43788bb, the group's head: a check ask runs every level after a failure
mesh/check checking b55a735a97e0 on 511e4b741497 | novox/mesh-delivery#140 in group fix/check-every-failure
mesh/delivery-group group fix/check-every-failure checking: 2 of 2 member(s) ready
mesh/delivery closed: the pull request closed unmerged
Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
2026-11-02 16:33:12 +01:00
jschoubben 9a469222f8 mesh-delivery-checker: the line test holds ADR 0591's case of a first failure longer than the forge's limit
The first failure's words are shortened; "failed (N)" and "and N-1 more" stay whole within the limit.

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
2026-11-02 16:26:30 +01:00
jschoubben 8fdd92f532 mesh-delivery-checker: write a member's preview whenever its fetch merged, and do not wait on a composition that cannot come
Review of 650a1d0: a member that only failed its tests got no preview, so the indexer held no manifest
of it, the assigner's composition only said it waits, and the check waited for ever. The preview is now
written whenever every build module's fetch merged; a member with no preview of a merge (a conflict, a
fetch that did not answer) makes the composition not run, said with the members it names, unawaited.

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
2026-11-02 16:25:33 +01:00
jschoubben 650a1d0c58 mesh-delivery-checker: a check runs every member, level and the composition to its end and names every failure
The operator, 2026-11-02: "A failed check shows only its first fault". Group feat/1126-removal's
check failed four times, each verdict naming one fault, so three faults took three runs of 20-40 minutes.

The checker no longer returns at a member's first failure: every member is asked, every level's
failures and every member's refused step or conflict are joined in judging order, the manifests are
judged and the composition is read also when the tests failed, and a failed composition is added after
them. The status line says how many failed and names the first, cut so "; and N more" stays whole.
checker_verdicts and checker_pull answer the whole list (failed). Needs mesh-sdk's buildstep change
(every level runs after a failure) pinned at the group's head.

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
2026-11-02 16:21:23 +01:00
13 changed files with 281 additions and 81 deletions
@@ -189,7 +189,7 @@ func SeatTryTools(tr Trying) []stdio.Tool {
func Tools(r check.Reader) []stdio.Tool {
return []stdio.Tool{
{Name: check.VerdictsReadTool,
Description: "The verdicts the checker holds, each as stored under verdicts.<group>.<heads>: its state, what failed first, the tests run, kept and skipped per level, the time of each part, the trunk heads it judged against and the composition, with the full " +
Description: "The verdicts the checker holds, each as stored under verdicts.<group>.<heads>: its state, every failure it names one line each (every member's failed tests, refused steps and conflicts, the manifests' refusals and the composition), the tests run, kept and skipped per level, the time of each part, the trunk heads it judged against and the composition, with the full " +
"report of the status each pull request carries (the line the forge shows is cut at 140 characters, this is not). With a group, that group's only. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"group": map[string]any{"type": "string", "description": "the delivery group's branch name"}}},
@@ -199,7 +199,7 @@ func Tools(r check.Reader) []stdio.Tool {
return cutAnswer(map[string]any{"verdicts": vs}, cut), err
}},
{Name: check.PullReadTool,
Description: "The verdict that judges one pull request and the full report of its status mesh/check. Read-only.",
Description: "The verdict that judges one pull request, every failure it names one line each (all of them, not only the first the status line shows), and the full report of its status mesh/check. Read-only.",
Input: map[string]any{"type": "object", "required": []string{"repository", "number"}, "properties": map[string]any{
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
"number": map[string]any{"type": "integer", "description": "the pull request's number"}}},
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-checker
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102153155-43788bbfdef3
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102153155-43788bbfdef3 h1:K3Uw1b+IPAKd8NeGzJNqggQifnEYYv9IUxBFpunx05c=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102153155-43788bbfdef3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -88,7 +88,12 @@ func withComposition(v *delivery.Verdict, comp delivery.CompositionVerdict) {
if machine == "" {
machine = "the composition"
}
v.State, v.Failed = delivery.VerdictFail, delivery.FailedFirst{Machine: machine, Words: comp.Words}
f := delivery.FailedFirst{Machine: machine, Words: comp.Words}
if v.State == delivery.VerdictFail {
addFailure(v, f) // the tests failed too: the composition is named after them (the operator, 2026-11-02)
return
}
v.State, v.Failed = delivery.VerdictFail, f
}
}
@@ -98,9 +103,9 @@ func notJudged(v delivery.Verdict) bool {
}
// unanswered says whether a verdict failed only on its composition, and that only because nodes the change places did
// not answer its placement in time (delivery.CompositionUnanswered, ADR 0559 §5): its tests passed (a composition is read
// only into a passing verdict), so a later composition of the same heads is taken in on its event, with no new check
// (ADR 0548 §1: a part of the mesh not heard is judged again when it is heard).
// not answer its placement in time (delivery.CompositionUnanswered, ADR 0559 §5): its tests passed (the composition is
// its first failure, and a composition is named after every other failure), so a later composition of the same heads is
// taken in on its event, with no new check (ADR 0548 §1: a part of the mesh not heard is judged again when it is heard).
func unanswered(v delivery.Verdict) bool {
return v.State == delivery.VerdictFail && v.Composition != nil && v.Composition.State == delivery.CompositionFail &&
strings.Contains(v.Composition.Words, delivery.CompositionUnanswered) && v.Failed.Words == v.Composition.Words
@@ -141,7 +146,7 @@ func (ch *Checker) Composed(ctx context.Context, comp delivery.CompositionVerdic
case v.State == delivery.VerdictPass && notJudged(v):
case unanswered(v) && comp.Words != v.Composition.Words:
// The nodes answered since (or another failure stands now): the verdict is its tests' pass with this composition.
v.State, v.Failed = delivery.VerdictPass, delivery.FailedFirst{}
v.State, v.Failed, v.Failures = delivery.VerdictPass, delivery.FailedFirst{}, nil
default:
return
}
@@ -14,10 +14,17 @@ type fakeCompositions struct {
v delivery.CompositionVerdict
err error
asked []string
// waits answers only that the composition waits, as the assigner does for a member the indexer holds no manifest of:
// Await returns only when its context ends.
waits bool
}
func (f *fakeCompositions) Await(_ context.Context, key string) (delivery.CompositionVerdict, error) {
func (f *fakeCompositions) Await(ctx context.Context, key string) (delivery.CompositionVerdict, error) {
f.asked = append(f.asked, key)
if f.waits {
<-ctx.Done()
return delivery.CompositionVerdict{}, ctx.Err()
}
return f.v, f.err
}
@@ -23,7 +23,7 @@ func judgeLevel(t *testing.T, tests ...json.RawMessage) (*delivery.Verdict, *sta
t.Helper()
st := &state{j: &Examiner{}, c: &Case{}, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}}
v, _ := st.level(t.Context(), &target{pull: Pull{Repository: "novox/x"}}, delivery.LevelAnswer{Level: delivery.LevelIntegration, Tests: tests})
return v, st
return settled(joinFailures(v)), st
}
// Issue 1921: two failing tests in two packages are two entries of the verdict's Failures and two lines of a try's
@@ -58,7 +58,7 @@ func TestTwoFailingTestsInTwoPackagesAreTwoEntries(t *testing.T) {
}
// Issue 1921: past delivery.MaxFailures the list names the first fifty and how many more, the status line says how many
// failed, and each entry's line is cut to MaxFailureLine.
// failed and names the first, cut so "and N more" stays whole, and each entry's line is cut to MaxFailureLine.
func TestFailuresAreCappedWithHowManyMore(t *testing.T) {
var fs []Failure
for i := range 60 {
@@ -75,16 +75,20 @@ func TestFailuresAreCappedWithHowManyMore(t *testing.T) {
t.Errorf("failedCount = %d, want 60", n)
}
cs := &Case{Pulls: []Pull{{Repository: "novox/x", Head: "0123456789abcdef", Trunk: "fedcba9876543210"}}}
if line := Line(*v, cs, cs.Pulls[0], nil, nil, nil); !strings.HasPrefix(line, "failed (60 tests): integration: x/p0.TestT") {
if line := Line(*v, cs, cs.Pulls[0], nil, nil, nil); !strings.HasPrefix(line, "failed (60): integration: x/p0.TestT: yyy") ||
!strings.Contains(line, "…; and 59 more | 0123456789ab on fedcba987654") {
t.Errorf("line %q", line)
}
}
// A composition failing after them is counted in the last entry (the operator, 2026-11-02).
withComposition(v, delivery.CompositionVerdict{State: delivery.CompositionFail, Machine: "g14", Words: "route-proxy is defined twice"})
if n := failedCount(*v); n != 61 || v.Failed.Test != "integration: x/p0.TestT" {
t.Errorf("after the composition failedCount = %d, failed %+v", n, v.Failed)
}
// A verdict that names no list (a conflict, a refused step, a verdict written before issue 1921) is one failed line, as
// before.
func TestAVerdictWithoutFailuresIsOneLine(t *testing.T) {
v := delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "g14", Words: "files in conflict: go.mod\nmore"}}
if got := failedLines(v); len(got) != 1 || got[0] != "g14: files in conflict: go.mod" {
// A verdict that names no list (a conflict, a refused step, a verdict written before issue 1921) is one failed line, as
// before.
one := delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "g14", Words: "files in conflict: go.mod\nmore"}}
if got := failedLines(one); len(got) != 1 || got[0] != "g14: files in conflict: go.mod" {
t.Fatalf("%q", got)
}
}
@@ -10,11 +10,13 @@ import (
"os"
"path/filepath"
"reflect"
"slices"
"sort"
"strings"
"sync"
"testing"
"time"
"unicode/utf8"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/composer"
@@ -431,32 +433,87 @@ func TestANodePackageIsNeverKeptAndRunsEveryCheck(t *testing.T) { // 1288/AC4, 1
}
}
// 1289/AC1, AC2 (R4c, mesh-sdk buildrun at 0b12b62): the change's own tests run first, and when one fails the level ends
// there: no other package of the level and no later level runs, so the rest of the suite is not paid for.
func TestTheChangesOwnFailingTestEndsTheLevelThere(t *testing.T) {
b := newBench(t)
b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"}
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}}
failing := captured[Tested](t, "go-test-failing")
b.test = func(c stepCall) (any, error) {
if got := packagesOf(c); len(got) == 1 && got[0] == "m/internal/b" {
return failing, nil
// The operator, 2026-11-02 ("A failed check shows only its first fault"): a check runs every member, every level and the
// composition to its end and its verdict names every failure in the order judged, Failed keeping the first whole; the
// change's own tests still run first (issue 1289), and the rest of the level after them. A member that only failed its
// tests still gets its preview, so the composition is read and its failure named after the tests'. A member whose head
// does not merge has no preview of a merge, so the assigner's composition would only wait: the check does not wait for it
// and says why it was not run (review of 650a1d0).
func TestEveryFailureOfEveryMemberLevelAndTheCompositionIsNamed(t *testing.T) {
for _, conflicts := range []bool{false, true} {
b := newBench(t)
comps := &fakeCompositions{v: delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionFail, SDK: "x",
Machine: "g14", Words: "route-proxy is defined twice"}, waits: conflicts}
b.ex.Compositions = comps
b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"}
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}}
conflict := captured[Fetched](t, "go-fetch-conflict")
b.steps.handlers["build_go_fetch"] = func(c stepCall) (any, error) {
if conflicts && c.args["repository"] == repoSDK {
return conflict, nil
}
return b.fetch, nil
}
failing := captured[Tested](t, "go-test-failing")
b.test = func(c stepCall) (any, error) {
if got := packagesOf(c); c.args["level"] == "integration" || (len(got) == 1 && got[0] == "m/internal/b") {
r := failing
r.Level = c.args["level"].(string)
return r, nil
}
return captured[Tested](t, "go-test-slow"), nil
}
cs := caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))
done := make(chan delivery.Verdict, 1)
go func() { done <- b.judge(t, cs).Verdict }()
var v delivery.Verdict
select {
case v = <-done:
case <-time.After(5 * time.Second):
t.Fatalf("conflicts %v: the check waits on a composition that only waits", conflicts)
}
var got []string
for _, f := range v.Failures {
got = append(got, firstFailed(delivery.Verdict{Failed: f}))
}
tests := []string{"unit: fixture/failing/bad.TestThreeIsFour", "integration: fixture/failing/bad.TestThreeIsFour"}
want := append(append(slices.Clone(tests), tests...), "g14: route-proxy is defined twice") // both members fail their tests
if conflicts {
want = append([]string{"shanks: novox/mesh-sdk#12 does not merge into the trunk: files in conflict: a.go"}, tests...)
}
if v.State != delivery.VerdictFail || len(got) != len(want) {
t.Fatalf("conflicts %v: verdict %+v: failures\n%s\nwant\n%s", conflicts, v, strings.Join(got, "\n"), strings.Join(want, "\n"))
}
for i := range want {
if !strings.HasPrefix(got[i], want[i]) {
t.Errorf("conflicts %v: failure %d is %q, want %q", conflicts, i, got[i], want[i])
}
}
keys, _ := b.prev.Keys(context.Background())
switch {
case !conflicts && (len(keys) != 2 || v.Composition == nil || v.Composition.State != delivery.CompositionFail):
t.Errorf("tests alone failed: previews %v, composition %+v, want both previews and the composition read", keys, v.Composition)
case conflicts && (len(comps.asked) != 0 || v.Composition == nil || v.Composition.State != delivery.CompositionNotRun ||
!strings.Contains(v.Composition.Words, "novox/mesh-sdk#12")):
t.Errorf("a member did not merge: asked %v, composition %+v, want not run naming novox/mesh-sdk#12", comps.asked, v.Composition)
}
var unit []string
for _, c := range b.steps.tool("test") {
if c.args["level"] == "unit" && c.args["workspace"] != nil {
unit = append(unit, strings.Join(packagesOf(c), ","))
}
}
if conflicts && (strings.Join(unit, " | ") != "m/internal/b | m/internal/a" || len(b.steps.tool("digest")) != 2) {
t.Errorf("the unit level ran %v and %d digests, want the changed test's package, then the rest, at both levels", unit, len(b.steps.tool("digest")))
}
v.Group, v.Heads = "feat/x", "0123456789ab"
if _, err := delivery.EncodeVerdict(v); err != nil {
t.Errorf("conflicts %v: the verdict cannot be written: %v", conflicts, err)
}
if line := Line(v, cs, cs.Pulls[0], nil, nil, nil); !strings.HasPrefix(line, fmt.Sprintf("failed (%d): %s", len(want), want[0][:20])) ||
!strings.Contains(line, fmt.Sprintf("; and %d more | ", len(want)-1)) || utf8.RuneCountInString(line) > MaxLine {
t.Errorf("conflicts %v: line %q", conflicts, line)
}
return captured[Tested](t, "go-test-slow"), nil
}
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
if out.Verdict.State != delivery.VerdictFail || !strings.Contains(out.Verdict.Failed.Test, "TestThreeIsFour") {
t.Fatalf("verdict = %+v", out.Verdict)
}
tests := b.steps.tool("test")
if len(tests) != 1 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" {
t.Fatalf("test calls = %+v, want m/internal/b alone: its failure ends the level", tests)
}
if n := len(b.steps.tool("digest")); n != 1 {
t.Errorf("%d digest calls, want the unit level's alone (the check stopped)", n)
}
if out.Verdict.Composition == nil || out.Verdict.Composition.State != delivery.CompositionNotRun {
t.Errorf("composition = %+v", out.Verdict.Composition)
}
}
@@ -76,8 +76,9 @@ var askLanguage = map[string]string{buildrun.GoBuildModule: delivery.AskGo, buil
// Examiner judges a case through the build seats (ADR 0484 clauses 3, 4, 10, 13; ADR 0525 §1, §3): for each pull request
// and build module it puts one check ask whose job fetches the head merged onto its trunk head, then runs the unit level
// and the integration level, taking the kept verdicts of unchanged packages, running the change's own tests first and
// stopping at the first failure; it judges the step's answer as it judged the tools' answers.
// and the integration level, taking the kept verdicts of unchanged packages and running the change's own tests first;
// every level and member runs to its end also after a failure, so the verdict names every failure (the operator,
// 2026-11-02). It judges the step's answer as it judged the tools' answers.
type Examiner struct {
Asks Asks
// DeclineBound is how long a placed ask may go on being declined, from its first decline, before the check fails
@@ -231,6 +232,9 @@ type state struct {
judgeHead string
// predefinedHead is the catalogue member's head when its merge changes predefined.json: the list is read there.
predefinedHead string
// unpreviewed are the members (pullKey) with no preview of a merge: a head that does not merge, or a fetch that did not
// answer. The indexer holds no manifest of them, so the assigner's composition would only wait: it is not awaited.
unpreviewed []string
}
// Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot
@@ -238,23 +242,20 @@ type state struct {
func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) {
st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{},
beside: map[string]string{}}
if v := st.judgeAll(ctx); v != nil {
return st.finish(ctx, *v), nil
}
// Every manifest the change touches, judged by the indexer's judge (ADR 0578 §2): a refusal fails the check.
if v := st.judgeManifests(ctx); v != nil {
// Every member and level runs to its end, and every manifest the change touches is judged by the indexer's judge (ADR
// 0578 §2), also after a failure: the verdict names every failure, not only the first (the operator, 2026-11-02).
tests := st.judgeAll(ctx)
if v := settled(joinFailures(tests, st.judgeManifests(ctx))); v != nil {
return st.finish(ctx, *v), nil
}
return st.finish(ctx, delivery.Verdict{State: delivery.VerdictPass}), nil
}
// finish is the outcome of the verdict with the state of the builds the candidate's modules carry, and, for a check
// whose tests passed, the assigner's composition of it.
// finish is the outcome of the verdict with the state of the builds the candidate's modules carry and the assigner's
// composition of it, read also when the tests failed, so a failed composition is named beside them.
func (st *state) finish(ctx context.Context, v delivery.Verdict) Outcome {
out := st.outcome(v)
if out.Verdict.State == delivery.VerdictPass {
st.composed(ctx, &out.Verdict)
}
st.composed(ctx, &out.Verdict)
out.Builds = st.j.buildNotes(ctx, st.c)
return out
}
@@ -273,6 +274,12 @@ func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
if st.j.Compositions == nil || st.c.Try != nil {
return
}
if len(st.unpreviewed) > 0 {
// The assigner would only say it waits for a manifest the indexer will not have: the check does not wait for it.
v.Composition = &delivery.Composition{State: delivery.CompositionNotRun, Words: "no preview of a merge of " +
strings.Join(st.unpreviewed, ", ") + " (its head does not merge or its fetch did not answer), so the change cannot be composed"}
return
}
comp, err := st.j.Compositions.Await(ctx, st.c.Key)
switch {
case errors.Is(err, ErrAssignerNotRunning):
@@ -354,8 +361,9 @@ func (st *state) levelsOf() []string {
}
// judgeAll puts one check ask per pull request and build module the repository needs, the mesh-sdk member first, and
// judges each answer. A head that does not merge, a pin that is not allowed or a level that fails ends the check at once
// (ADR 0484 clause 13).
// judges each answer. A head that does not merge, a pin that is not allowed or a level that fails is a failure of that
// member; every other member is still judged, and the verdict names every failure (the operator, 2026-11-02), unjoined
// to its first lines (settled does that). Only a check whose context ended stops.
func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
// The mesh-sdk member first: whether its merge changes the TypeScript package's paths decides whether the other members'
// Node fetches take the group's package (ADR 0500 §5).
@@ -370,9 +378,17 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
return 0
})
var groupPackage map[string]any
var failures []*delivery.Verdict
for _, p := range pulls {
if ctx.Err() != nil {
if len(failures) == 0 { // an ask that ended with the context said it already
failures = append(failures, fail(delivery.CheckerModule, ctx.Err().Error()))
}
break
}
var merged []Fetched
for _, bm := range st.j.Modules.BuildModules(p.Repository) {
modules := st.j.Modules.BuildModules(p.Repository)
for _, bm := range modules {
t := &target{pull: p, module: bm}
if tc, none := delivery.AgreedToolchain(st.j.Toolchains.Entries(buildrun.LanguageOf[bm])); none == "" {
t.toolchain = tc.Version
@@ -385,7 +401,8 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
if bm == buildrun.GoBuildModule {
beside, err := st.j.besideFor(ctx, st.c, p)
if err != nil {
return fail(forgeMachine(err, bm), err.Error()+unheardWords(err))
failures = append(failures, fail(forgeMachine(err, bm), err.Error()+unheardWords(err)))
continue
}
if len(beside) > 0 {
args["beside"] = beside
@@ -399,18 +416,24 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
t.fetchArgs = args
st.targets = append(st.targets, t)
if v := st.ask(ctx, t, &merged); v != nil {
return v
failures = append(failures, v)
}
}
if st.c.Try == nil { // a try owes the indexer no preview: no pull request holds its head
// Every build module's fetch merged (each is in merged once): the preview is written, also when the member's tests
// failed, so the indexer holds its manifests and the assigner can compose the change (review of 650a1d0).
switch {
case st.c.Try != nil: // a try owes the indexer no preview: no pull request holds its head
case len(merged) == len(modules):
st.j.writePreview(ctx, p, merged)
default:
st.unpreviewed = append(st.unpreviewed, pullKey(p))
}
st.judgedByMember(p, merged)
if p.Repository == SDKRepository && len(st.c.Pulls) > 1 && st.j.changesPackage(p, merged) {
groupPackage = map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number)}
}
}
return nil
return joinFailures(failures...)
}
// DefaultDeclineBound is an Examiner's DeclineBound when it names none: longer than a check runs, so a node with the
@@ -508,6 +531,7 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
st.j.logf("%s#%d: the ask %s was declined on %s, which lacks %s; asked again", p.Repository, p.Number, a.ID, step.Node, strings.Join(d.Lacks, ", "))
a.ID, a.At = buildstep.NewID(), now.UTC()
}
var failures []*delivery.Verdict
// The fetch and the digests are what of the ask's time its levels' running did not take (Took.Clone).
took := time.Duration(0)
if !step.Started.IsZero() && step.Ended.After(step.Started) {
@@ -539,7 +563,7 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
if !placed { // the placed ask fetched the same merge: its preview was read from the first
*merged = append(*merged, t.fetched)
if v := st.checkBinariesOf(t); v != nil {
return v
failures = append(failures, v) // the levels still run: the verdict names every failure
}
}
ran := 0.0
@@ -555,13 +579,12 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
v, elapsed := st.level(ctx, t, la)
ran += elapsed
if v != nil {
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
return v
failures = append(failures, v) // the next level was run too (buildstep): its failures are named as well
}
}
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
if unmet := delivery.UnmetNeeds(answer.Levels); len(unmet) > 0 && !placed {
// Every level passed, its passes kept, and some packages were left to a node with what their tests need: the levels
// Every level ran, its passes kept, and some packages were left to a node with what their tests need: the levels
// from the first that left one are asked of such a node, which takes the kept passes and tests those (issue 890).
var again, packages []string
seen := map[string]bool{}
@@ -577,9 +600,9 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
}
}
st.j.logf("%s#%d: %s on %s left untested; asked of a node with %s", p.Repository, p.Number, unmetOf(answer.Levels), step.Node, strings.Join(unmet, ", "))
return st.askNeeding(ctx, t, merged, again, unmet, packages)
failures = append(failures, st.askNeeding(ctx, t, merged, again, unmet, packages))
}
return nil
return joinFailures(failures...)
}
// unmetNames names the unmet packages and their needs.
@@ -865,10 +888,58 @@ func levelFailure(t *target, level string, results ...Tested) *delivery.Verdict
all = append(all, delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: "the " + level + " level did not pass"})
}
v.Failed = all[0]
v.Failures = failureLines(all)
v.Failures = all // whole: settled cuts them to their first lines once every failure of the check is joined
return &v
}
// joinFailures is one failed verdict of every failure of vs, in the order given (a nil one passed), unsettled: Failed is
// the first whole, and Failures every failure whole, or none when the one failure is no level's (a conflict, a refused
// step: one line, as before issue 1921). Nil when none failed.
func joinFailures(vs ...*delivery.Verdict) *delivery.Verdict {
var out *delivery.Verdict
var all []delivery.FailedFirst
listed := false
for _, v := range vs {
if v == nil {
continue
}
if out == nil {
out = &delivery.Verdict{State: delivery.VerdictFail, Failed: v.Failed}
}
if len(v.Failures) > 0 {
all, listed = append(all, v.Failures...), true
continue
}
all = append(all, v.Failed)
}
if out != nil && (listed || len(all) > 1) {
out.Failures = all
}
return out
}
// settled is a joined verdict as it is written: each failure of its list cut to its first line, at most
// delivery.MaxFailures and then how many more (failureLines).
func settled(v *delivery.Verdict) *delivery.Verdict {
if v != nil && len(v.Failures) > 0 {
v.Failures = failureLines(v.Failures)
}
return v
}
// addFailure is a settled list of failures with one more at its end: the first failure is put first when the list is
// empty (a verdict of one failure keeps none), and past delivery.MaxFailures the last entry counts one more.
func addFailure(v *delivery.Verdict, f delivery.FailedFirst) {
if len(v.Failures) == 0 {
v.Failures = failureLines([]delivery.FailedFirst{v.Failed})
}
if n := len(v.Failures); n > delivery.MaxFailures {
v.Failures[n-1].Test = fmt.Sprintf("and %d more", failedCount(*v)-delivery.MaxFailures+1)
return
}
v.Failures = append(v.Failures, failureLines([]delivery.FailedFirst{f})...)
}
// failureLines is every failure with the first line of its words, cut to MaxFailureLine, at most delivery.MaxFailures
// and then one entry saying how many more failed.
func failureLines(all []delivery.FailedFirst) []delivery.FailedFirst {
@@ -65,7 +65,35 @@ func failedLines(v delivery.Verdict) []string {
return lines
}
// failedCount is how many tests a verdict names as failed, the ones after delivery.MaxFailures counted too.
// failedHead is the head of a failed verdict's line: what failed first, and when more did, how many failed and "and N
// more" after it (the operator, 2026-11-02), so a line cut to MaxLine still says that more failed. The words of the first
// are cut to fit room characters when room is not negative; the count and "and N more" are never cut.
func failedHead(v delivery.Verdict, room int) string {
first := firstFailed(v)
n := failedCount(v)
if n <= 1 {
return clipRunes("failed: "+first, room)
}
prefix, more := fmt.Sprintf("failed (%d): ", n), fmt.Sprintf("; and %d more", n-1)
if room >= 0 {
first = clipRunes(first, room-utf8.RuneCountInString(prefix+more))
}
return prefix + first + more
}
// clipRunes cuts s to at most n characters, the last an ellipsis; a negative n leaves it whole.
func clipRunes(s string, n int) string {
r := []rune(s)
if n < 0 || len(r) <= n {
return s
}
if n < 1 {
return ""
}
return string(r[:n-1]) + "…"
}
// failedCount is how many failures a verdict names, the ones after delivery.MaxFailures counted too.
func failedCount(v delivery.Verdict) int {
n := len(v.Failures)
if n > delivery.MaxFailures {
@@ -232,10 +260,7 @@ func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []s
pc.carried = "carried from " + w // the counts that follow say what the carry ran and kept
}
default:
pc.head = "failed: " + firstFailed(v)
if n := failedCount(v); n > 1 {
pc.head = fmt.Sprintf("failed (%d tests): %s", n, firstFailed(v))
}
pc.head = failedHead(v, -1)
pc.commits = commits
}
var counts []string
@@ -325,6 +350,11 @@ func Report(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []
// what follows, and is cut in its first part rather than lose it.
func Line(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string, judged map[string]string) string {
pc := parts(v, cs, p, builds, slowest, judged)
if v.State == delivery.VerdictFail {
// The head gives way first to the commits and what is never dropped: its count and "and N more" stay whole.
kept := joinSet(pc.commits, pc.carried, pc.warning, pc.judged)
pc.head = failedHead(v, MaxLine-utf8.RuneCountInString(" | "+kept))
}
for drop := 0; drop <= 4; drop++ {
if s := pc.join(drop); utf8.RuneCountInString(s) <= MaxLine {
return s
@@ -342,6 +372,12 @@ func Line(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []st
return string(head[:room]) + "…" + rest
}
}
if v.State == delivery.VerdictFail {
// A failure's head, sized above, and the commits fit: the counts and the composition are in the report.
if s := joinSet(pc.head, pc.commits, pc.carried, pc.warning, pc.judged); utf8.RuneCountInString(s) <= MaxLine {
return s
}
}
runes := []rune(pc.join(4))
return string(runes[:MaxLine-1]) + "…"
}
@@ -1,6 +1,7 @@
package check
import (
"fmt"
"strings"
"testing"
"unicode/utf8"
@@ -43,6 +44,7 @@ func TestAPassOpensWithTheVerdictAndSaysTheCommitsTheCountsTheTimesAndThatTheCom
func TestAFailureNamesTheTestTheConflictOrTheRefusedStepFirst(t *testing.T) { // 1300/AC2
cs, p := lineCase(t, pullOf(repoA, 7, head1))
long := delivery.FailedFirst{Test: "integration: " + strings.Repeat("pkg/", 50) + "TestSomething", Words: strings.Repeat("because ", 40)}
for name, c := range map[string]struct {
v delivery.Verdict
want string
@@ -50,8 +52,14 @@ func TestAFailureNamesTheTestTheConflictOrTheRefusedStepFirst(t *testing.T) { //
"test": {delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Test: "unit: m/a.TestX", Words: "want 4"}}, "failed: unit: m/a.TestX: want 4"},
"conflict": {delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "shanks", Words: "novox/mesh-delivery#7 does not merge into the trunk: files in conflict: go.mod"}}, "failed: shanks: novox/mesh-delivery#7 does not merge into the trunk: files in conflict: go.mod"},
"refused": {delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "mesh-build-go", Words: "no go build node qualifies: builders disagree"}}, "failed: mesh-build-go: no go build node qualifies"},
// ADR 0591: a first failure longer than the forge's limit is shortened; "failed (N)" and "and N-1 more" are never cut.
"many, the first too long": {delivery.Verdict{State: delivery.VerdictFail, Failed: long, Failures: []delivery.FailedFirst{long, {Machine: "g14", Words: "route-proxy is defined twice"},
{Test: "unit: m/b.TestY", Words: "boom"}}}, "failed (3): integration: pkg/pkg/"},
} {
line := Line(c.v, cs, p, nil, nil, nil)
if more := len(c.v.Failures) - 1; more > 0 && (!strings.Contains(line, fmt.Sprintf("…; and %d more | 111111111111", more)) || utf8.RuneCountInString(line) > MaxLine) {
t.Errorf("%s: line = %q (%d), want the first's words shortened, the count and \"and %d more\" whole, within %d", name, line, utf8.RuneCountInString(line), more, MaxLine)
}
if !strings.HasPrefix(line, c.want) || !strings.Contains(line, "111111111111 on aaaaaaaaaaaa") {
t.Errorf("%s: line = %q, want it to open with %q and name the commits", name, line, c.want)
}
@@ -52,11 +52,14 @@ type Reader struct {
Now func() time.Time
}
// VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
// candidate still holds those heads; without it the report is that of the verdict alone.
// VerdictRead is a verdict as stored with every failure it names, one line each, and the full report of the status of each
// pull request it judged (Report), when the candidate still holds those heads; without it the report is that of the verdict alone.
type VerdictRead struct {
Key string `json:"key"`
Verdict json.RawMessage `json:"verdict"`
Key string `json:"key"`
Verdict json.RawMessage `json:"verdict"`
// Failed is everything the verdict names as failed, one line each, in the order judged (failedLines): every member's
// failed tests, refused steps and conflicts, the manifests' refusals and the composition (the operator, 2026-11-02).
Failed []string `json:"failed,omitempty"`
Reports map[string]string `json:"reports,omitempty"`
Note string `json:"note,omitempty"`
}
@@ -80,6 +83,7 @@ func (r Reader) read(key string, raw json.RawMessage) (VerdictRead, error) {
if err != nil {
return out, fmt.Errorf("%s holds what is not a verdict: %w", key, err)
}
out.Failed = failedLines(v)
c, ok := r.candidate(v.Group)
if !ok || (c.HeadsHash != v.Heads && GateHeads(c) != v.Heads) {
out.Note = "the candidate no longer holds these heads"
@@ -67,6 +67,13 @@ func TestTheVerdictToolsAnswerTheStoredVerdictWithTheFullReportOfEachPullRequest
if only, _, _ := rd.Verdicts("feat/other"); len(only) != 0 {
t.Errorf("another group's verdicts = %+v", only)
}
// A failed verdict is answered with every failure it names, one line each (the operator, 2026-11-02).
raw, _ := delivery.EncodeVerdict(delivery.Verdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.VerdictFail,
Failed: delivery.FailedFirst{Test: "unit: m/a.TestA", Words: "a_test.go:3: boom\nmore"},
Failures: []delivery.FailedFirst{{Test: "unit: m/a.TestA", Words: "a_test.go:3: boom"}, {Machine: "g14", Words: "route-proxy is defined twice"}}})
if got, _ := rd.read(key, raw); strings.Join(got.Failed, " | ") != "unit: m/a.TestA: a_test.go:3: boom | g14: route-proxy is defined twice" {
t.Errorf("failed = %q", got.Failed)
}
}
// 1887/A10: an hq pull request is answered as any other, its report its verdict's (ADR 0527 §2), never a note that a script
@@ -204,7 +204,8 @@ func TestAFailingTryNamesWhatFailedAndAStepThatCannotRunIsAnError(t *testing.T)
raw, _, _, _ := kv.Get(ctx, id)
tr.Put(ctx, id, raw)
tr.Wait()
if got := readTry(t, kv, id); got.State != TryFail || len(got.Failed) != 1 || !strings.Contains(got.Failed[0], "TestThreeIsFour") {
if got := readTry(t, kv, id); got.State != TryFail || len(got.Failed) != 2 || !strings.Contains(got.Failed[0], "unit: fixture/failing/bad.TestThreeIsFour") ||
!strings.Contains(got.Failed[1], "integration: fixture/failing/bad.TestThreeIsFour") { // both levels ran (the operator, 2026-11-02)
t.Errorf("try = %+v", got)
}
// No build node is no error since R4g: the try's ask waits in the check stream until a node pulls it (ADR 0525 §4).