novox/mesh-delivery
19
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Move the level rules and the failure words into the checker's rules package (issue 1921)

Browse Source 
The checking program judges a check from its facts (milestone 47/B0,
ADR 0644, ADR 0656): the rules package holds Facts v1, the level rule,
the failure words and the verdict's outcome, and the running checker
records each level answer as a fact and calls the rules. A row of the
every-failure tests judges the gathered facts and finds the running
checker's verdict.

Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
This commit is contained in:
jschoubben
2026-11-06 22:12:47 +01:00
parent 2d201f781c
commit 03bb56481a
12 changed files with 628 additions and 445 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-delivery-checker/internal/check/builds.go
+2 -1
View File
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"maps"
"mesh-delivery-checker/rules"
"slices"
"sort"
"strings"
@@ -160,7 +161,7 @@ func (st *state) awaitBuilds(ctx context.Context, v *delivery.Verdict) []string
v.State, v.Failed = delivery.VerdictFail, f
continue
}
addFailure(v, f)
rules.AddFailure(v, f)
}
}
if len(late) > 0 && ctx.Err() == nil {
modules/mesh-delivery-checker/internal/check/checker.go
+2 -1
View File
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"maps"
"mesh-delivery-checker/rules"
"slices"
"sort"
"strings"
@@ -449,7 +450,7 @@ func (ch *Checker) untrunked(ctx context.Context, c delivery.Candidate, left []P
continue
}
o := delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "error",
Target: strings.TrimSuffix(ch.Link, "/") + "/" + key, Text: clipLine(words, MaxLine)}
Target: strings.TrimSuffix(ch.Link, "/") + "/" + key, Text: rules.ClipLine(words, MaxLine)}
data, err := delivery.EncodeCheckerOwed(o)
if err != nil {
ch.logf("%s: the status of %s cannot be written: %v", key, okey, err)
modules/mesh-delivery-checker/internal/check/composition.go
+2 -1
View File
@@ -1,6 +1,7 @@
package check
import (
"mesh-delivery-checker/rules"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -20,7 +21,7 @@ func withComposition(v *delivery.Verdict, comp delivery.CompositionVerdict) {
}
f := delivery.FailedFirst{Machine: machine, Words: comp.Words}
if v.State == delivery.VerdictFail {
addFailure(v, f) // the tests failed too: the composition is named after them (the operator, 2026-11-02)
rules.AddFailure(v, f) // the tests failed too: the composition is named after them (the operator, 2026-11-02)
return
}
v.State, v.Failed = delivery.VerdictFail, f
modules/mesh-delivery-checker/internal/check/every_failure_test.go
+18 -7
View File
@@ -3,6 +3,8 @@ package check
import (
"encoding/json"
"fmt"
"mesh-delivery-checker/rules"
"reflect"
"strings"
"testing"
@@ -21,9 +23,18 @@ func failingResult(t *testing.T, tests int, failed ...Failure) json.RawMessage {
func judgeLevel(t *testing.T, tests ...json.RawMessage) (*delivery.Verdict, *state) {
t.Helper()
st := &state{j: &Examiner{}, c: &Case{}, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}}
st := &state{j: &Examiner{}, c: &Case{}, levels: rules.NewTally(), nodes: map[string]bool{}}
v, _ := st.level(t.Context(), &target{pull: Pull{Repository: "novox/x"}}, delivery.LevelAnswer{Level: delivery.LevelIntegration, Tests: tests})
return settled(joinFailures(v)), st
v = rules.Settled(rules.JoinFailures(v))
// 1921 (B0, D1a): the facts the running checker gathered, judged by the rules, give the same failures and counts.
judged := rules.Judge(rules.Facts{V: rules.FactsVersion, Steps: st.facts})
switch {
case v == nil && judged.State != delivery.VerdictPass, v != nil && (judged.Failed != v.Failed || !reflect.DeepEqual(judged.Failures, v.Failures)):
t.Errorf("the rules judge the facts %+v, the running checker %+v", judged, v)
case judged.Levels[delivery.LevelIntegration] != (delivery.LevelResult{Run: st.levels[delivery.LevelIntegration].Run}):
t.Errorf("the rules count %+v, the running checker %+v", judged.Levels, *st.levels[delivery.LevelIntegration])
}
return v, st
}
// Issue 1921: two failing tests in two packages are two entries of the verdict's Failures and two lines of a try's
@@ -48,7 +59,7 @@ func TestTwoFailingTestsInTwoPackagesAreTwoEntries(t *testing.T) {
if len(v.Failures) != 2 || v.Failed.Test != "integration: x/a.TestA" || !strings.Contains(v.Failed.Words, "a_test.go:3: logged") {
t.Errorf("%s: verdict %+v", name, v)
}
if run := st.levels[delivery.LevelIntegration].run; run != 10 {
if run := st.levels[delivery.LevelIntegration].Run; run != 10 {
t.Errorf("%s: %d tests counted, want all 10", name, run)
}
if _, err := delivery.EncodeVerdict(delivery.Verdict{Group: "g", Heads: "0123456789ab", State: v.State, Failed: v.Failed, Failures: v.Failures}); err != nil {
@@ -58,7 +69,7 @@ func TestTwoFailingTestsInTwoPackagesAreTwoEntries(t *testing.T) {
}
// Issue 1921: past delivery.MaxFailures the list names the first fifty and how many more, the status line says how many
// failed and names the first, cut so "and N more" stays whole, and each entry's line is cut to MaxFailureLine.
// failed and names the first, cut so "and N more" stays whole, and each entry's line is cut to rules.MaxFailureLine.
func TestFailuresAreCappedWithHowManyMore(t *testing.T) {
var fs []Failure
for i := range 60 {
@@ -68,10 +79,10 @@ func TestFailuresAreCappedWithHowManyMore(t *testing.T) {
if len(v.Failures) != delivery.MaxFailures+1 || v.Failures[delivery.MaxFailures].Test != "and 10 more" {
t.Fatalf("%d entries, last %+v", len(v.Failures), v.Failures[len(v.Failures)-1])
}
if len(v.Failures[0].Words) > MaxFailureLine {
if len(v.Failures[0].Words) > rules.MaxFailureLine {
t.Errorf("an entry's line is %d bytes", len(v.Failures[0].Words))
}
if n := failedCount(*v); n != 60 {
if n := rules.FailedCount(*v); n != 60 {
t.Errorf("failedCount = %d, want 60", n)
}
cs := &Case{Pulls: []Pull{{Repository: "novox/x", Head: "0123456789abcdef", Trunk: "fedcba9876543210"}}}
@@ -81,7 +92,7 @@ func TestFailuresAreCappedWithHowManyMore(t *testing.T) {
}
// A composition failing after them is counted in the last entry (the operator, 2026-11-02).
withComposition(v, delivery.CompositionVerdict{State: delivery.CompositionFail, Machine: "g14", Words: "route-proxy is defined twice"})
if n := failedCount(*v); n != 61 || v.Failed.Test != "integration: x/p0.TestT" {
if n := rules.FailedCount(*v); n != 61 || v.Failed.Test != "integration: x/p0.TestT" {
t.Errorf("after the composition failedCount = %d, failed %+v", n, v.Failed)
}
modules/mesh-delivery-checker/internal/check/failure_words_test.go
+10 -9
View File
@@ -2,6 +2,7 @@ package check
import (
"encoding/json"
"mesh-delivery-checker/rules"
"strings"
"testing"
@@ -21,19 +22,19 @@ func fatalAfterLog() Failure {
}
func TestAFailedTestsVerdictKeepsItsWholeFailureMessage(t *testing.T) { // 1901/AC1
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{fatalAfterLog()}})
v := rules.LevelFailure("", "", "unit", Tested{Failed: []Failure{fatalAfterLog()}})
for _, want := range []string{"3 subscription(s) derive to a subject no emitter owns:", "a.b.c", "d.e.f", "g.h.i"} {
if !strings.Contains(v.Failed.Words, want) {
t.Errorf("words lack %q: %q", want, v.Failed.Words)
}
}
if len(v.Failed.Words) > MaxWords+200 {
t.Errorf("words are %d bytes, bound %d", len(v.Failed.Words), MaxWords)
if len(v.Failed.Words) > rules.MaxWords+200 {
t.Errorf("words are %d bytes, bound %d", len(v.Failed.Words), rules.MaxWords)
}
}
func TestTheStatusLineNamesTheFailureNotTheFirstLoggedLine(t *testing.T) { // 1901/AC2
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{fatalAfterLog()}})
v := rules.LevelFailure("", "", "unit", Tested{Failed: []Failure{fatalAfterLog()}})
got := firstFailed(*v)
if !strings.Contains(got, "3 subscription(s) derive to a subject no emitter owns:") || strings.Contains(got, "judged against") || strings.Contains(got, "\n") {
t.Errorf("firstFailed = %q", got)
@@ -41,7 +42,7 @@ func TestTheStatusLineNamesTheFailureNotTheFirstLoggedLine(t *testing.T) { // 19
// With no cause told apart (an older build module), the words still start at the first line, as before.
f := fatalAfterLog()
f.Cause = ""
v = levelFailure(&target{}, "unit", Tested{Failed: []Failure{f}})
v = rules.LevelFailure("", "", "unit", Tested{Failed: []Failure{f}})
if !strings.HasPrefix(afterHid(v.Failed.Words), "agreement_catalogue_test.go:130") {
t.Errorf("words = %q", v.Failed.Words)
}
@@ -49,8 +50,8 @@ func TestTheStatusLineNamesTheFailureNotTheFirstLoggedLine(t *testing.T) { // 19
func TestWordsLongerThanTheBoundKeepTheirLastLines(t *testing.T) { // 1901/AC1
f := Failure{Output: strings.Repeat("log line of some length\n", 500) + "the cause is here"}
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{f}})
if !strings.HasSuffix(v.Failed.Words, "the cause is here") || len(v.Failed.Words) > MaxWords+200 {
v := rules.LevelFailure("", "", "unit", Tested{Failed: []Failure{f}})
if !strings.HasSuffix(v.Failed.Words, "the cause is here") || len(v.Failed.Words) > rules.MaxWords+200 {
t.Errorf("%d bytes, ends %q", len(v.Failed.Words), v.Failed.Words[len(v.Failed.Words)-30:])
}
_ = delivery.VerdictFail
@@ -66,7 +67,7 @@ func TestAnOlderBuildModulesFailureIsAcceptedWithItsWholeOutput(t *testing.T) {
if err := json.Unmarshal([]byte(old), &f); err != nil || f.Cause != "" {
t.Fatalf("1901/AC3: err = %v, cause = %q", err, f.Cause)
}
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{f}})
v := rules.LevelFailure("", "", "unit", Tested{Failed: []Failure{f}})
if afterHid(v.Failed.Words) != "x_test.go:1: logged\nx_test.go:2: the old kept lines" {
t.Errorf("1901/AC3: words = %q", v.Failed.Words)
}
@@ -84,7 +85,7 @@ func TestAnOlderBuildModulesFailureIsAcceptedWithItsWholeOutput(t *testing.T) {
{"a real cause", "mesh-build-node", by, "one plus one is not three", "not ok 3 - fails", ""},
} {
t.Run(c.name, func(t *testing.T) {
v := levelFailure(&target{module: c.module}, "unit", Tested{By: c.by, Failed: []Failure{{Package: "modules/gitea", Test: "fails", Cause: c.cause, Output: c.output}}})
v := rules.LevelFailure("", c.module, "unit", Tested{By: c.by, Failed: []Failure{{Package: "modules/gitea", Test: "fails", Cause: c.cause, Output: c.output}}})
if v.State != delivery.VerdictFail {
t.Errorf("1901/AC8: the verdict is %q, want it failed", v.State)
}
modules/mesh-delivery-checker/internal/check/judge.go
+24 -350
View File
@@ -4,7 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"path"
"mesh-delivery-checker/rules"
"slices"
"sort"
"strings"
@@ -228,24 +228,17 @@ func (t *target) stepArgs(args map[string]any) map[string]any {
return args
}
// acc collects a level's counts across targets.
type acc struct {
run, kept, skipped int
elapsed, setup float64
lint float64
over bool
slowest []TestTime
}
// state is one judging.
type state struct {
j *Examiner
c *Case
targets []*target
levels map[string]*acc
nodes map[string]bool
clone time.Duration
slow []string
levels rules.Tally
// facts are the facts of the check met so far, in order: what the checking program judges (milestone 47/B0).
facts []rules.Step
nodes map[string]bool
clone time.Duration
slow []string
// beside are the siblings read at their trunk's head, by repository (ADR 0548 §6).
beside map[string]string
// judgeHead is the group's mesh-delivery member head when its merge changes the judge (ADR 0578 §2); empty judges
@@ -305,13 +298,13 @@ func (st *state) takeTryCandidate(ctx context.Context) {
// Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot
// be judged at all.
func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) {
st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{},
st := &state{j: j, c: c, levels: rules.NewTally(), nodes: map[string]bool{},
beside: map[string]string{}}
// Every member and level runs to its end, and every manifest the change touches is judged by the indexer's judge (ADR
// 0578 §2), also after a failure: the verdict names every failure, not only the first (the operator, 2026-11-02).
tests := st.judgeAll(ctx)
st.takeTryCandidate(ctx)
if v := settled(joinFailures(tests, st.judgeManifests(ctx), st.refuseCycle(ctx))); v != nil {
if v := rules.Settled(rules.JoinFailures(tests, st.judgeManifests(ctx), st.refuseCycle(ctx))); v != nil {
return st.finish(ctx, *v), nil
}
return st.finish(ctx, delivery.Verdict{State: delivery.VerdictPass}), nil
@@ -370,40 +363,8 @@ func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
// outcome fills the verdict's levels, times, machines and composition from what ran.
func (st *state) outcome(v delivery.Verdict) Outcome {
if v.State == delivery.VerdictPass || len(v.Levels) == 0 {
v.Levels = map[string]delivery.LevelResult{}
for level, a := range st.levels {
v.Levels[level] = delivery.LevelResult{Run: a.run, Kept: a.kept, Skipped: a.skipped}
}
}
ms := func(s float64) int64 { return int64(s * 1000) }
u, i := st.levels[delivery.LevelUnit], st.levels[delivery.LevelIntegration]
v.Took = delivery.Took{Clone: st.clone.Milliseconds(), Unit: ms(u.elapsed), Lint: ms(u.lint), IntegrationSetup: ms(i.setup), IntegrationRun: ms(i.elapsed)}
for node := range st.nodes {
v.Machines = append(v.Machines, node)
}
sort.Strings(v.Machines)
for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} {
if st.levels[level].over {
v.OverBound = append(v.OverBound, level)
}
}
// The composition is made from the engines' answers (ADR 0593 §7): until it is, a verdict says it was not run (ADR 0484 clause 6).
v.Composition = &delivery.Composition{State: delivery.CompositionNotRun}
var slow []string
for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} {
a := st.levels[level]
if !a.over {
continue
}
sort.SliceStable(a.slowest, func(x, y int) bool { return a.slowest[x].Elapsed > a.slowest[y].Elapsed })
for n, s := range a.slowest {
if n == 3 {
break
}
slow = append(slow, fmt.Sprintf("%s %s %.1fs", level, path.Base(s.Package)+"."+s.Test, s.Elapsed))
}
}
rules.Outcome(&v, st.levels, st.clone.Milliseconds(), st.nodes)
slow := rules.Slowest(st.levels)
out := Outcome{Verdict: v, Slowest: slow}
if len(st.beside) > 0 {
out.Beside = st.beside
@@ -506,7 +467,7 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
groupPackage = map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number)}
}
}
return joinFailures(failures...)
return rules.JoinFailures(failures...)
}
// DefaultDeclineBound is an Examiner's DeclineBound when it names none: longer than a check runs, so a node with the
@@ -675,7 +636,7 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
st.j.logf("%s#%d: %s on %s left untested; asked of a node with %s", p.Repository, p.Number, unmetOf(answer.Levels), step.Node, strings.Join(unmet, ", "))
failures = append(failures, st.askNeeding(ctx, t, merged, again, unmet, packages))
}
return joinFailures(failures...)
return rules.JoinFailures(failures...)
}
// unmetNames names the unmet packages and their needs.
@@ -703,62 +664,30 @@ func unmetOf(levels []delivery.LevelAnswer) string {
// one that did not pass, so the verdict names every failed test of the level (issue 1921). It answers the level's
// running time (its tests' and setup's).
func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer) (*delivery.Verdict, float64) {
level := la.Level
a := st.levels[level]
if a == nil {
return fail(t.node, fmt.Sprintf("the check of %s#%d answered the level %q, which is no level", t.pull.Repository, t.pull.Number, level)), 0
}
var digests map[string]string
if t.module == buildrun.GoBuildModule && la.Digest != nil {
// The packages come as a list, never keyed by a package path (issue 1930); an answer stored before reads too.
digests = la.Digest.Map()
if t.runner != "" {
t.judgedBy(la.Digest.JudgedBy) // the installed module judged it: the runner is not used again
}
if t.runner != "" && t.module == buildrun.GoBuildModule && la.Digest != nil {
t.judgedBy(la.Digest.JudgedBy) // the installed module judged it: the runner is not used again
}
if t.runner != "" && la.Judged != "" {
t.judgedBy(la.Judged) // the candidate judged the level: nothing is kept (ADR 0535 §4)
}
for _, k := range la.Kept {
a.kept += k.Tests
v, elapsed, results := st.levels.Level(t.pull.Repository, t.pull.Number, t.module, t.node, la)
st.facts = append(st.facts, rules.Step{Repository: t.pull.Repository, Number: t.pull.Number, Module: t.module, Node: t.node, Level: &la})
var digests map[string]string
if t.module == buildrun.GoBuildModule && la.Digest != nil {
// The packages come as a list, never keyed by a package path (issue 1930); an answer stored before reads too.
digests = la.Digest.Map()
}
elapsed := 0.0
var failedResults []Tested
for _, raw := range la.Tests {
var res Tested
if err := json.Unmarshal(raw, &res); err != nil {
return fail(t.node, fmt.Sprintf("the %s level of %s#%d answered what is not a result: %v", level, t.pull.Repository, t.pull.Number, err)), elapsed
}
for _, res := range results {
if t.runner != "" {
t.judgedBy(res.JudgedBy)
}
elapsed += res.Elapsed + res.Setup + res.Lint
if t.module == buildrun.NodeBuildModule {
if !res.NoTests {
a.run++ // a Node package is counted whole: its build module counts no tests (ADR 0484 clause 10)
}
} else {
a.run += res.Tests
a.skipped += res.Skipped
a.slowest = append(a.slowest, res.Slowest...)
}
a.elapsed += res.Elapsed
a.setup += res.Setup
a.lint += res.Lint
a.over = a.over || res.OverBound
if len(digests) > 0 {
if _, err := st.j.Kept.KeepPasses(ctx, t.pull.Repository, level, t.toolchain, res, digests, st.j.now()); err != nil {
if _, err := st.j.Kept.KeepPasses(ctx, t.pull.Repository, la.Level, t.toolchain, res, digests, st.j.now()); err != nil {
st.j.logf("%s#%d: a passing package could not be kept: %v", t.pull.Repository, t.pull.Number, err)
}
}
if !res.Passed {
failedResults = append(failedResults, res)
}
}
if len(failedResults) > 0 {
return levelFailure(t, level, failedResults...), elapsed
}
return nil, elapsed
return v, elapsed
}
// changesPackage says whether the merge of a pull request changes a file inside the named paths of one of its repository's
@@ -780,258 +709,3 @@ func (j *Examiner) changesPackage(p Pull, merged []Fetched) bool {
}
return false
}
// failedName names a failure: its package and test.
func failedName(f Failure) string {
if f.Test == "" {
return f.Package
}
return f.Package + "." + f.Test
}
// MaxWords is the most a failed test's words in a verdict hold (issue 1901). **The words keep the failure's last lines**,
// where a test says why it failed: the first line is often something it logged before (the group r4g verdict kept a
// "judged against" line and lost the t.Fatalf after it), and the 120 characters kept before cut a docker command's echo
// before docker's own error. Four kilobytes holds a long message and is small beside a verdict's other parts.
const MaxWords = 4096
// failureWords is a failure's words: its output's last lines up to MaxWords, with the line the build module told apart as
// the test's error first when the output does not start with it, so the first line of the words is the cause and the
// status line, which is cut short, names it. **A failure's words are never "PASS" or "ok"** (issue 1901, its third
// meeting: a package that failed after its tests passed was worded "PASS"): go test's summary lines are dropped, and with
// no cause told apart (an older build module) the cause is the output's first "panic:", "fatal error:" or "signal:"
// line, read before the output is cut, or the words say the tests passed and the program failed after them.
func failureWords(f Failure) string {
return failureWordsAfter(f, "")
}
// failureWordsAfter is failureWords with the line loud put first, within the same MaxWords: what the checker itself says of
// the failure (that the build module hid it), which the status line then names.
func failureWordsAfter(f Failure, loud string) string {
out, passed := dropSummary(strings.TrimSpace(f.Output))
cause := strings.TrimSpace(f.Cause)
if cause == "" {
cause = causeIn(out)
}
switch {
case cause == "" && out == "" && passed:
cause = "the tests passed, then the test program exited with a failure: an os.Exit or a panic in TestMain or after the tests"
case cause == "" && out == "":
// The build module on main sends a package that stopped (a timeout, an exit, a kill) as its "FAIL" line alone.
cause = "the package failed with no test to name and no panic, signal or build error in its output (an older build module sent no cause)"
}
if len(cause) > MaxCause {
cause = strings.ToValidUTF8(cause[:MaxCause], "") + "..."
}
// The words keep MaxWords exactly: the loud line, the cause and their newlines, the marker and the kept lines.
budget := MaxWords
if cause != "" {
budget -= len(cause) + 1
}
if loud != "" {
budget -= len(loud) + 1
}
if len(out) > budget {
const marker = "... "
tail := out[len(out)-(budget-len(marker)):]
// Start on a whole line, and a whole character.
if i := strings.IndexByte(tail, '\n'); i >= 0 {
tail = strings.TrimLeft(tail[i:], "\n")
}
out = marker + strings.ToValidUTF8(tail, "")
}
if cause != "" && !strings.HasPrefix(firstLine(out), cause) {
out = strings.TrimRight(cause+"\n"+out, "\n")
}
if loud != "" {
out = strings.TrimRight(loud+"\n"+out, "\n")
}
return out
}
// hidWords is the checker's loud line for a failure the build module sent hiding its cause (issue 1901, its thorough
// fix): with no cause, or a passing line as its cause, under the failure contract (buildstep.Hidden). It names the build
// module and the version it says it is (Tested.By, "<module> with mesh-sdk <version>"); one from before the contract
// says nothing of itself. "" when the failure keeps the contract. The check fails either way: only its words change.
func hidWords(module, by string, f Failure) string {
why := buildstep.Hidden(f)
if why == "" {
return ""
}
version := "a version before the failure contract"
if module == "" {
module = "an unnamed build module"
}
if m, v, ok := strings.Cut(by, " with "); ok && m != "" {
module, version = m, v
}
return fmt.Sprintf("the build module hid the failure (%s at %s): %s", module, version, why)
}
// MaxCause is the most a cause in a failure's words holds: one long line (the 1901 review: a cause of a kilobyte could
// push the words past MaxWords).
const MaxCause = 1024
// dropSummary drops go test's summary lines from a failure's output ("PASS", "FAIL", "ok <pkg>", "FAIL <pkg>"), and says
// whether a "PASS" was among them: the package's tests passed before it failed.
func dropSummary(out string) (string, bool) {
passed := false
lines := strings.Split(out, "\n")
kept := lines[:0]
for _, l := range lines {
// go's exact forms only (review round 1): "PASS" or "FAIL" alone, "ok \t<pkg>\t<time>", "FAIL\t<pkg>\t<time>".
switch t := strings.TrimRight(l, "\r"); {
case t == "PASS":
passed = true
case t == "FAIL", strings.HasPrefix(t, "ok \t"), strings.HasPrefix(t, "FAIL\t"):
default:
kept = append(kept, l)
}
}
return strings.TrimSpace(strings.Join(kept, "\n")), passed
}
// causeIn is a race the race detector reported, as its heading, its access and the frame that made it ("WARNING: DATA
// RACE: Write at 0x... by goroutine 9 in pkg.F.func1()"); else the first "panic:", "fatal error:" or "signal:" line of a
// failure's output, in that order; or "". **Only lines at column 0**, where go prints the runtime's: a test's indented log
// line holding the same words is not the cause (review round 1).
func causeIn(out string) string {
lines := strings.Split(out, "\n")
for i, l := range lines {
if strings.TrimRight(l, "\r") != "WARNING: DATA RACE" {
continue
}
words := "WARNING: DATA RACE"
if i+1 < len(lines) && strings.TrimSpace(lines[i+1]) != "" {
words += ": " + strings.TrimSuffix(strings.TrimSpace(lines[i+1]), ":")
if i+2 < len(lines) && strings.HasPrefix(lines[i+2], " ") {
words += " in " + strings.TrimSpace(lines[i+2])
}
}
return words
}
for _, prefix := range []string{"panic: ", "fatal error: ", "signal: "} {
for _, l := range lines {
if l = strings.TrimRight(l, "\r"); strings.HasPrefix(l, prefix) {
return l
}
}
}
return ""
}
// firstLine is the first non-empty line of a failure's words.
func firstLine(s string) string {
for _, l := range strings.Split(s, "\n") {
if l = strings.TrimSpace(l); l != "" {
return l
}
}
return ""
}
// MaxFailureLine is the most bytes of a failure's line in a verdict's Failures: the first line of its words, which is
// its cause where the build module told one apart (issue 1901). Failed keeps the first failure's words whole.
const MaxFailureLine = 300
// levelFailure turns a level that did not pass into a verdict: Failed names what failed first with its words whole,
// and Failures every failed test of the level's results with its first line, at most delivery.MaxFailures and an entry
// saying how many more (issue 1921).
func levelFailure(t *target, level string, results ...Tested) *delivery.Verdict {
v := delivery.Verdict{State: delivery.VerdictFail}
var all []delivery.FailedFirst
for _, res := range results {
if len(res.Failed) == 0 {
why := res.Why
if why == "" {
why = "the " + level + " level did not pass"
}
all = append(all, delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: why})
continue
}
for _, f := range res.Failed {
name := failedName(f)
if t.module == buildrun.NodeBuildModule && f.Package == "" {
name = res.Module
}
all = append(all, delivery.FailedFirst{Test: level + ": " + name, Words: failureWordsAfter(f, hidWords(t.module, res.By, f))})
}
}
if len(all) == 0 { // no result: a level said failed with nothing to name
all = append(all, delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: "the " + level + " level did not pass"})
}
v.Failed = all[0]
v.Failures = all // whole: settled cuts them to their first lines once every failure of the check is joined
return &v
}
// joinFailures is one failed verdict of every failure of vs, in the order given (a nil one passed), unsettled: Failed is
// the first whole, and Failures every failure whole, or none when the one failure is no level's (a conflict, a refused
// step: one line, as before issue 1921). Nil when none failed.
func joinFailures(vs ...*delivery.Verdict) *delivery.Verdict {
var out *delivery.Verdict
var all []delivery.FailedFirst
listed := false
for _, v := range vs {
if v == nil {
continue
}
if out == nil {
out = &delivery.Verdict{State: delivery.VerdictFail, Failed: v.Failed}
}
if len(v.Failures) > 0 {
all, listed = append(all, v.Failures...), true
continue
}
all = append(all, v.Failed)
}
if out != nil && (listed || len(all) > 1) {
out.Failures = all
}
return out
}
// settled is a joined verdict as it is written: each failure of its list cut to its first line, at most
// delivery.MaxFailures and then how many more (failureLines).
func settled(v *delivery.Verdict) *delivery.Verdict {
if v != nil && len(v.Failures) > 0 {
v.Failures = failureLines(v.Failures)
}
return v
}
// addFailure is a settled list of failures with one more at its end: the first failure is put first when the list is
// empty (a verdict of one failure keeps none), and past delivery.MaxFailures the last entry counts one more.
func addFailure(v *delivery.Verdict, f delivery.FailedFirst) {
if len(v.Failures) == 0 {
v.Failures = failureLines([]delivery.FailedFirst{v.Failed})
}
if n := len(v.Failures); n > delivery.MaxFailures {
v.Failures[n-1].Test = fmt.Sprintf("and %d more", failedCount(*v)-delivery.MaxFailures+1)
return
}
v.Failures = append(v.Failures, failureLines([]delivery.FailedFirst{f})...)
}
// failureLines is every failure with the first line of its words, cut to MaxFailureLine, at most delivery.MaxFailures
// and then one entry saying how many more failed.
func failureLines(all []delivery.FailedFirst) []delivery.FailedFirst {
n := min(len(all), delivery.MaxFailures)
out := make([]delivery.FailedFirst, 0, n+1)
for _, f := range all[:n] {
out = append(out, delivery.FailedFirst{Test: f.Test, Machine: f.Machine, Words: clipLine(firstLine(f.Words), MaxFailureLine)})
}
if more := len(all) - n; more > 0 {
out = append(out, delivery.FailedFirst{Test: fmt.Sprintf("and %d more", more)})
}
return out
}
// clipLine cuts a line to at most n bytes on a whole character, marking the cut.
func clipLine(s string, n int) string {
if len(s) <= n {
return s
}
s = strings.ToValidUTF8(s[:n-3], "")
return s + "..."
}
modules/mesh-delivery-checker/internal/check/line.go
+3 -14
View File
@@ -2,6 +2,7 @@ package check
import (
"fmt"
"mesh-delivery-checker/rules"
"slices"
"strings"
"unicode/utf8"
@@ -70,7 +71,7 @@ func failedLines(v delivery.Verdict) []string {
// are cut to fit room characters when room is not negative; the count and "and N more" are never cut.
func failedHead(v delivery.Verdict, room int) string {
first := firstFailed(v)
n := failedCount(v)
n := rules.FailedCount(v)
if n <= 1 {
return clipRunes("failed: "+first, room)
}
@@ -93,20 +94,8 @@ func clipRunes(s string, n int) string {
return string(r[:n-1]) + "…"
}
// failedCount is how many failures a verdict names, the ones after delivery.MaxFailures counted too.
func failedCount(v delivery.Verdict) int {
n := len(v.Failures)
if n > delivery.MaxFailures {
var more int
if _, err := fmt.Sscanf(v.Failures[n-1].Test, "and %d more", &more); err == nil {
return delivery.MaxFailures + more
}
}
return n
}
// firstWordsLine is the first line of a failure's words: the words are several lines (issue 1901) and the status line one.
func firstWordsLine(w string) string { return firstLine(w) }
func firstWordsLine(w string) string { return rules.FirstLine(w) }
// pieces are the parts of a line, from the most to the least needed.
type pieces struct {
modules/mesh-delivery-checker/internal/check/package_cause_test.go
+11 -10
View File
@@ -1,6 +1,7 @@
package check
import (
"mesh-delivery-checker/rules"
"strings"
"testing"
)
@@ -32,14 +33,14 @@ func TestAPackageFailureIsNeverWordedPass(t *testing.T) { // 1901/AC5
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
v := levelFailure(&target{}, "integration", Tested{Failed: []Failure{{Package: "github.com/novox/mesh-controller/cmd/mesh-controller", Output: c.output}}})
v := rules.LevelFailure("", "", "integration", Tested{Failed: []Failure{{Package: "github.com/novox/mesh-controller/cmd/mesh-controller", Output: c.output}}})
for _, l := range strings.Split(v.Failed.Words, "\n") {
if l = strings.TrimSpace(l); l == "PASS" || strings.HasPrefix(l, "ok ") || strings.HasPrefix(l, "ok\t") {
t.Errorf("1901/AC5: the words keep %q: %q", l, v.Failed.Words)
}
}
if !strings.HasPrefix(afterHid(v.Failed.Words), c.want) {
t.Errorf("1901/AC5: words after the loud line start %q, want %q", firstLine(afterHid(v.Failed.Words)), c.want)
t.Errorf("1901/AC5: words after the loud line start %q, want %q", rules.FirstLine(afterHid(v.Failed.Words)), c.want)
}
if got := firstFailed(*v); !strings.Contains(got, "the build module hid the failure") || strings.HasSuffix(got, ": PASS") {
t.Errorf("1901/AC5, AC8: status = %q", got)
@@ -51,9 +52,9 @@ func TestAPackageFailureIsNeverWordedPass(t *testing.T) { // 1901/AC5
// A cause the build module told apart is cut to a line's bound, so the words keep their bound (the 1901 review's note).
func TestALongCauseIsCutToALinesBound(t *testing.T) { // 1901/AC5
f := Failure{Package: "p", Test: "TestX", Cause: strings.Repeat("c", 5000), Output: "x_test.go:1: something"}
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{f}})
if len(v.Failed.Words) > MaxWords+200 {
t.Errorf("1901/AC5: words are %d bytes, bound %d", len(v.Failed.Words), MaxWords)
v := rules.LevelFailure("", "", "unit", Tested{Failed: []Failure{f}})
if len(v.Failed.Words) > rules.MaxWords+200 {
t.Errorf("1901/AC5: words are %d bytes, bound %d", len(v.Failed.Words), rules.MaxWords)
}
}
@@ -62,7 +63,7 @@ func TestALongCauseIsCutToALinesBound(t *testing.T) { // 1901/AC5
// capture of the same name (go-test-json-pkg-timeout, -exiting, -killed): the package's own output, its "FAIL" line.
func TestAPackageStoppedWithOnlyItsFailLineIsStillWorded(t *testing.T) { // 1901/AC5
for _, out := range []string{"FAIL\texample.com/timeout\t2.017s", "FAIL\texample.com/exiting\t0.006s", "FAIL\texample.com/killed\t0.006s"} {
v := levelFailure(&target{}, "integration", Tested{Failed: []Failure{{Package: "example.com/x", Output: out}}})
v := rules.LevelFailure("", "", "integration", Tested{Failed: []Failure{{Package: "example.com/x", Output: out}}})
if !strings.HasPrefix(afterHid(v.Failed.Words), "the package failed with no test to name and no panic, signal or build error in its output") {
t.Errorf("1901/AC5: %q is worded %q", out, v.Failed.Words)
}
@@ -73,19 +74,19 @@ func TestAPackageStoppedWithOnlyItsFailLineIsStillWorded(t *testing.T) { // 1901
// either (review round 1): go prints its summary in exact forms and the runtime's lines at column 0.
func TestOnlyGosExactFormsAreSummaryOrCause(t *testing.T) { // 1901/AC5
out := " x_test.go:3: ok so far\nok then: the seat is free\n panic: child retried and recovered\npanic: the real one\nFAIL\tp\t1.0s"
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{{Package: "p", Output: out}}})
v := rules.LevelFailure("", "", "unit", Tested{Failed: []Failure{{Package: "p", Output: out}}})
if !strings.HasPrefix(afterHid(v.Failed.Words), "panic: the real one") || !strings.Contains(v.Failed.Words, "ok then: the seat is free") {
t.Errorf("1901/AC5: words = %q", v.Failed.Words)
}
}
// The words keep MaxWords exactly, counting the cause, the marker and the newlines (review round 1).
// The words keep rules.MaxWords exactly, counting the cause, the marker and the newlines (review round 1).
func TestTheWordsKeepTheirBoundExactly(t *testing.T) { // 1901/AC5
for _, cause := range []string{"", "a cause", strings.Repeat("c", 3000)} {
for _, n := range []int{100, 170, 171, 172, 500} {
f := Failure{Package: "p", Test: "TestX", Cause: cause, Output: strings.Repeat("a log line of twenty\n", n) + "the end"}
if w := failureWords(f); len(w) > MaxWords {
t.Errorf("1901/AC5: cause %d bytes, %d lines: words are %d bytes, bound %d", len(cause), n, len(w), MaxWords)
if w := rules.FailureWords(f); len(w) > rules.MaxWords {
t.Errorf("1901/AC5: cause %d bytes, %d lines: words are %d bytes, bound %d", len(cause), n, len(w), rules.MaxWords)
}
}
}
modules/mesh-delivery-checker/internal/check/results.go
+8 -52
View File
@@ -4,8 +4,8 @@ import (
"encoding/json"
"strings"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-checker/rules"
)
// The shapes of what the build modules' steps answer, as their code writes them (mesh-catalog modules/mesh-build-go
@@ -60,57 +60,13 @@ type ProgramFile struct {
Commit string `json:"commit,omitempty"`
}
// GoPackage is one package of a Go test result.
type GoPackage struct {
Dir string `json:"dir"`
Package string `json:"package"`
Passed bool `json:"passed"`
Elapsed float64 `json:"elapsed"`
Tests int `json:"tests"`
NoTests bool `json:"no_tests,omitempty"`
// Skipped is how many of the package's tests skipped; an older build module names none.
Skipped int `json:"skipped,omitempty"`
}
// Failure is a test that failed, or a package that failed with no test to name: the build modules' failure contract
// (buildstep.Failure, issue 1901), its cause never empty and never a passing line from a build module that keeps it.
type Failure = buildstep.Failure
// TestTime is one test and the seconds it took.
type TestTime struct {
Package string `json:"package"`
Test string `json:"test"`
Elapsed float64 `json:"elapsed"`
}
// Tested is the answer of a test step of either build module (the Go one names packages and counts tests; the Node one
// names the package directory as Module and counts none).
type Tested struct {
Level string `json:"level"`
Module string `json:"module,omitempty"`
Passed bool `json:"passed"`
Why string `json:"why,omitempty"`
NoTests bool `json:"no_tests,omitempty"`
Tests int `json:"tests"`
Skipped int `json:"skipped"`
Packages []GoPackage `json:"packages,omitempty"`
Elapsed float64 `json:"elapsed"`
Setup float64 `json:"setup,omitempty"`
// Lint is the unit level's gofmt and go vet time, before its tests (mesh-build-go, ADR 0529 §1).
Lint float64 `json:"lint,omitempty"`
Bound float64 `json:"bound"`
SetupBound float64 `json:"setup_bound,omitempty"`
Slowest []TestTime `json:"slowest,omitempty"`
Failed []Failure `json:"failed,omitempty"`
OverBound bool `json:"over_bound"`
OverWhich string `json:"over_which,omitempty"`
// JudgedBy is the program that ran the step (ADR 0535 §5): "installed", or "candidate <merge commit>" under runner
// workspace; absent from a build module that does not take the runner.
JudgedBy string `json:"judged_by,omitempty"`
// By is the build module that answered and the SDK it was built with (buildstep.By); absent from a build module from
// before the failure contract.
By string `json:"by,omitempty"`
}
// The test step's answer is read by the judging rules (milestone 47/B0); these name their shapes here.
type (
Failure = rules.Failure
TestTime = rules.TestTime
Tested = rules.Tested
GoPackage = rules.GoPackage
)
// Digested is the answer of the Go build module's digest step: one digest per package that has tests at the level, by import
// path.
modules/mesh-delivery-checker/rules/facts.go
+209
View File
@@ -0,0 +1,209 @@
// Package rules is the checker's judging rules, the checking program's: given the facts of one check (what the build
// modules, the manifest validator and the composer answered, and what failed in the asking), the verdict (novox/hq ADR
// 0644, ADR 0656; design 47's bootstrap delivery). The running checker asks, waits, keeps and records; it gathers the
// facts and has them judged by the checking program built from the change's heads (cmd/mesh-checking-program), which
// imports this package, so a change to a rule is judged by its own rule.
//
// It is the checker module's own package (novox/hq ADR 0634 §1): no seat carries it and no other module imports it.
package rules
import (
"encoding/json"
"fmt"
"path"
"sort"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// FactsVersion is the version of Facts this program reads. A checking program built from a later change reads the facts
// a running checker of this version writes, so a version is added to, never changed; a program refuses one it does not
// read, naming it (DecodeFacts).
const FactsVersion = 1
// Facts are everything one check judges, in the order the running checker met it.
type Facts struct {
V int `json:"v"`
// Group and Heads name the check (delivery.Verdict's Group and Heads); Trunk is each repository's trunk head merged.
Group string `json:"group"`
Heads string `json:"heads"`
Trunk map[string]string `json:"trunk,omitempty"`
Members []Member `json:"members"`
// Steps are, in the order met, each level a build module answered and each failure the asking met.
Steps []Step `json:"steps,omitempty"`
// CloneMS is how much of the steps' time their levels' running did not take (delivery.Took.Clone).
CloneMS int64 `json:"clone-ms,omitempty"`
// Notes are the running checker's notes of the check, for a reader.
Notes []string `json:"notes,omitempty"`
}
// Member is one pull request of the check.
type Member struct {
Repository string `json:"repository"`
Number int `json:"number"`
Head string `json:"head"`
Trunk string `json:"trunk,omitempty"`
}
// Step is one fact of a member's check step: the node that ran it, and either one level its build module answered
// (Level, as the build module gave it) or a failure the asking met (Failed: a step not asked, refused or not answered,
// a conflict, a binary, worded by the running checker). A step with neither only says which node ran it.
type Step struct {
Repository string `json:"repository"`
Number int `json:"number"`
Module string `json:"module,omitempty"`
Node string `json:"node,omitempty"`
Level *delivery.LevelAnswer `json:"level,omitempty"`
Failed *delivery.FailedFirst `json:"failed,omitempty"`
}
// DecodeFacts reads facts, refusing a version this program does not read, naming it.
func DecodeFacts(data []byte) (Facts, error) {
var f Facts
if err := json.Unmarshal(data, &f); err != nil {
return Facts{}, fmt.Errorf("rules: the facts are not JSON of their shape: %w", err)
}
if f.V != FactsVersion {
return Facts{}, fmt.Errorf("rules: facts of version %d, this checking program reads version %d", f.V, FactsVersion)
}
return f, nil
}
// Counts are one level's counts across a check's members.
type Counts struct {
Run, Kept, Skipped int
Elapsed, Setup float64
Lint float64
Over bool
Slowest []TestTime
}
// Tally is a check's counts by level.
type Tally map[string]*Counts
// NewTally is a tally of both levels, at nothing.
func NewTally() Tally {
return Tally{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}
}
// Level judges one level of a member's check step as it was answered: the kept ones taken are counted, and every test
// result is read and counted, also after one that did not pass, so the verdict names every failed test of the level
// (issue 1921). It answers the level's failure (nil when it passed), its running time (its tests' and setup's) and the
// results read, for the running checker to keep the passes from.
func (t Tally) Level(repository string, number int, module, node string, la delivery.LevelAnswer) (*delivery.Verdict, float64, []Tested) {
a := t[la.Level]
if a == nil {
return failAt(node, fmt.Sprintf("the check of %s#%d answered the level %q, which is no level", repository, number, la.Level)), 0, nil
}
for _, k := range la.Kept {
a.Kept += k.Tests
}
elapsed := 0.0
var results, failed []Tested
for _, raw := range la.Tests {
var res Tested
if err := json.Unmarshal(raw, &res); err != nil {
return failAt(node, fmt.Sprintf("the %s level of %s#%d answered what is not a result: %v", la.Level, repository, number, err)), elapsed, results
}
results = append(results, res)
elapsed += res.Elapsed + res.Setup + res.Lint
if module == buildrun.NodeBuildModule {
if !res.NoTests {
a.Run++ // a Node package is counted whole: its build module counts no tests (ADR 0484 clause 10)
}
} else {
a.Run += res.Tests
a.Skipped += res.Skipped
a.Slowest = append(a.Slowest, res.Slowest...)
}
a.Elapsed += res.Elapsed
a.Setup += res.Setup
a.Lint += res.Lint
a.Over = a.Over || res.OverBound
if !res.Passed {
failed = append(failed, res)
}
}
if len(failed) > 0 {
return LevelFailure(repository, module, la.Level, failed...), elapsed, results
}
return nil, elapsed, results
}
// Judge is the verdict of the facts: every failure in the order met, joined and settled, or a pass; the levels'
// counts, times, machines and the levels over their bound; the composition not yet run (the running checker composes).
func Judge(f Facts) delivery.Verdict {
tally := NewTally()
var failures []*delivery.Verdict
nodes := map[string]bool{}
for _, s := range f.Steps {
if s.Node != "" {
nodes[s.Node] = true
}
switch {
case s.Failed != nil:
failures = append(failures, &delivery.Verdict{State: delivery.VerdictFail, Failed: *s.Failed})
case s.Level != nil:
if v, _, _ := tally.Level(s.Repository, s.Number, s.Module, s.Node, *s.Level); v != nil {
failures = append(failures, v)
}
}
}
v := delivery.Verdict{State: delivery.VerdictPass}
if joined := Settled(JoinFailures(failures...)); joined != nil {
v = *joined
}
v.Group, v.Heads, v.Trunk = f.Group, f.Heads, f.Trunk
Outcome(&v, tally, f.CloneMS, nodes)
return v
}
// Outcome fills a verdict's levels (on a pass, or when it names none), times, machines and levels over their bound from
// the tally, and says its composition is not yet run (ADR 0484 clause 6).
func Outcome(v *delivery.Verdict, tally Tally, cloneMS int64, nodes map[string]bool) {
if v.State == delivery.VerdictPass || len(v.Levels) == 0 {
v.Levels = map[string]delivery.LevelResult{}
for level, a := range tally {
v.Levels[level] = delivery.LevelResult{Run: a.Run, Kept: a.Kept, Skipped: a.Skipped}
}
}
ms := func(s float64) int64 { return int64(s * 1000) }
u, i := tally[delivery.LevelUnit], tally[delivery.LevelIntegration]
v.Took = delivery.Took{Clone: cloneMS, Unit: ms(u.Elapsed), Lint: ms(u.Lint), IntegrationSetup: ms(i.Setup), IntegrationRun: ms(i.Elapsed)}
v.Machines = nil
for node := range nodes {
v.Machines = append(v.Machines, node)
}
sort.Strings(v.Machines)
v.OverBound = nil
for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} {
if tally[level].Over {
v.OverBound = append(v.OverBound, level)
}
}
v.Composition = &delivery.Composition{State: delivery.CompositionNotRun}
}
// Slowest are, of each level over its bound, its three slowest tests, worded for the status line.
func Slowest(tally Tally) []string {
var slow []string
for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} {
a := tally[level]
if !a.Over {
continue
}
sort.SliceStable(a.Slowest, func(x, y int) bool { return a.Slowest[x].Elapsed > a.Slowest[y].Elapsed })
for n, s := range a.Slowest {
if n == 3 {
break
}
slow = append(slow, fmt.Sprintf("%s %s %.1fs", level, path.Base(s.Package)+"."+s.Test, s.Elapsed))
}
}
return slow
}
func failAt(machine, words string) *delivery.Verdict {
return &delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: machine, Words: words}}
}
modules/mesh-delivery-checker/rules/results.go
+59
View File
@@ -0,0 +1,59 @@
package rules
import "git.novox.be/novox/mesh-sdk/go/buildstep"
// The shapes of a test step's answer, as the build modules' code writes them (mesh-catalog modules/mesh-build-go
// cmd/mesh-build-go levels.go TestResult; modules/mesh-build-node cmd/mesh-build-node check.go TestResult). Only the
// fields the rules read are named; the rest is ignored.
// GoPackage is one package of a Go test result.
type GoPackage struct {
Dir string `json:"dir"`
Package string `json:"package"`
Passed bool `json:"passed"`
Elapsed float64 `json:"elapsed"`
Tests int `json:"tests"`
NoTests bool `json:"no_tests,omitempty"`
// Skipped is how many of the package's tests skipped; an older build module names none.
Skipped int `json:"skipped,omitempty"`
}
// Failure is a test that failed, or a package that failed with no test to name: the build modules' failure contract
// (buildstep.Failure, issue 1901), its cause never empty and never a passing line from a build module that keeps it.
type Failure = buildstep.Failure
// TestTime is one test and the seconds it took.
type TestTime struct {
Package string `json:"package"`
Test string `json:"test"`
Elapsed float64 `json:"elapsed"`
}
// Tested is the answer of a test step of either build module (the Go one names packages and counts tests; the Node one
// names the package directory as Module and counts none).
type Tested struct {
Level string `json:"level"`
Module string `json:"module,omitempty"`
Passed bool `json:"passed"`
Why string `json:"why,omitempty"`
NoTests bool `json:"no_tests,omitempty"`
Tests int `json:"tests"`
Skipped int `json:"skipped"`
Packages []GoPackage `json:"packages,omitempty"`
Elapsed float64 `json:"elapsed"`
Setup float64 `json:"setup,omitempty"`
// Lint is the unit level's gofmt and go vet time, before its tests (mesh-build-go, ADR 0529 §1).
Lint float64 `json:"lint,omitempty"`
Bound float64 `json:"bound"`
SetupBound float64 `json:"setup_bound,omitempty"`
Slowest []TestTime `json:"slowest,omitempty"`
Failed []Failure `json:"failed,omitempty"`
OverBound bool `json:"over_bound"`
OverWhich string `json:"over_which,omitempty"`
// JudgedBy is the program that ran the step (ADR 0535 §5): "installed", or "candidate <merge commit>" under runner
// workspace; absent from a build module that does not take the runner.
JudgedBy string `json:"judged_by,omitempty"`
// By is the build module that answered and the SDK it was built with (buildstep.By); absent from a build module from
// before the failure contract.
By string `json:"by,omitempty"`
}
modules/mesh-delivery-checker/rules/words.go
+280
View File
@@ -0,0 +1,280 @@
package rules
import (
"fmt"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The words rules: how a failed test, a failed package and a list of failures are worded in a verdict (issue 1901, issue
// 1921). Moved whole from the running checker's package (milestone 47/B0): they are the checking program's.
// FailedName names a failure: its package and test.
func FailedName(f Failure) string {
if f.Test == "" {
return f.Package
}
return f.Package + "." + f.Test
}
// MaxWords is the most a failed test's words in a verdict hold (issue 1901). **The words keep the failure's last lines**,
// where a test says why it failed: the first line is often something it logged before (the group r4g verdict kept a
// "judged against" line and lost the t.Fatalf after it), and the 120 characters kept before cut a docker command's echo
// before docker's own error. Four kilobytes holds a long message and is small beside a verdict's other parts.
const MaxWords = 4096
// FailureWords is a failure's words: its output's last lines up to MaxWords, with the line the build module told apart as
// the test's error first when the output does not start with it, so the first line of the words is the cause and the
// status line, which is cut short, names it. **A failure's words are never "PASS" or "ok"** (issue 1901, its third
// meeting: a package that failed after its tests passed was worded "PASS"): go test's summary lines are dropped, and with
// no cause told apart (an older build module) the cause is the output's first "panic:", "fatal error:" or "signal:"
// line, read before the output is cut, or the words say the tests passed and the program failed after them.
func FailureWords(f Failure) string {
return FailureWordsAfter(f, "")
}
// FailureWordsAfter is FailureWords with the line loud put first, within the same MaxWords: what the checker itself says of
// the failure (that the build module hid it), which the status line then names.
func FailureWordsAfter(f Failure, loud string) string {
out, passed := DropSummary(strings.TrimSpace(f.Output))
cause := strings.TrimSpace(f.Cause)
if cause == "" {
cause = CauseIn(out)
}
switch {
case cause == "" && out == "" && passed:
cause = "the tests passed, then the test program exited with a failure: an os.Exit or a panic in TestMain or after the tests"
case cause == "" && out == "":
// The build module on main sends a package that stopped (a timeout, an exit, a kill) as its "FAIL" line alone.
cause = "the package failed with no test to name and no panic, signal or build error in its output (an older build module sent no cause)"
}
if len(cause) > MaxCause {
cause = strings.ToValidUTF8(cause[:MaxCause], "") + "..."
}
// The words keep MaxWords exactly: the loud line, the cause and their newlines, the marker and the kept lines.
budget := MaxWords
if cause != "" {
budget -= len(cause) + 1
}
if loud != "" {
budget -= len(loud) + 1
}
if len(out) > budget {
const marker = "... "
tail := out[len(out)-(budget-len(marker)):]
// Start on a whole line, and a whole character.
if i := strings.IndexByte(tail, '\n'); i >= 0 {
tail = strings.TrimLeft(tail[i:], "\n")
}
out = marker + strings.ToValidUTF8(tail, "")
}
if cause != "" && !strings.HasPrefix(FirstLine(out), cause) {
out = strings.TrimRight(cause+"\n"+out, "\n")
}
if loud != "" {
out = strings.TrimRight(loud+"\n"+out, "\n")
}
return out
}
// HidWords is the checker's loud line for a failure the build module sent hiding its cause (issue 1901, its thorough
// fix): with no cause, or a passing line as its cause, under the failure contract (buildstep.Hidden). It names the build
// module and the version it says it is (Tested.By, "<module> with mesh-sdk <version>"); one from before the contract
// says nothing of itself. "" when the failure keeps the contract. The check fails either way: only its words change.
func HidWords(module, by string, f Failure) string {
why := buildstep.Hidden(f)
if why == "" {
return ""
}
version := "a version before the failure contract"
if module == "" {
module = "an unnamed build module"
}
if m, v, ok := strings.Cut(by, " with "); ok && m != "" {
module, version = m, v
}
return fmt.Sprintf("the build module hid the failure (%s at %s): %s", module, version, why)
}
// MaxCause is the most a cause in a failure's words holds: one long line (the 1901 review: a cause of a kilobyte could
// push the words past MaxWords).
const MaxCause = 1024
// DropSummary drops go test's summary lines from a failure's output ("PASS", "FAIL", "ok <pkg>", "FAIL <pkg>"), and says
// whether a "PASS" was among them: the package's tests passed before it failed.
func DropSummary(out string) (string, bool) {
passed := false
lines := strings.Split(out, "\n")
kept := lines[:0]
for _, l := range lines {
// go's exact forms only (review round 1): "PASS" or "FAIL" alone, "ok \t<pkg>\t<time>", "FAIL\t<pkg>\t<time>".
switch t := strings.TrimRight(l, "\r"); {
case t == "PASS":
passed = true
case t == "FAIL", strings.HasPrefix(t, "ok \t"), strings.HasPrefix(t, "FAIL\t"):
default:
kept = append(kept, l)
}
}
return strings.TrimSpace(strings.Join(kept, "\n")), passed
}
// CauseIn is a race the race detector reported, as its heading, its access and the frame that made it ("WARNING: DATA
// RACE: Write at 0x... by goroutine 9 in pkg.F.func1()"); else the first "panic:", "fatal error:" or "signal:" line of a
// failure's output, in that order; or "". **Only lines at column 0**, where go prints the runtime's: a test's indented log
// line holding the same words is not the cause (review round 1).
func CauseIn(out string) string {
lines := strings.Split(out, "\n")
for i, l := range lines {
if strings.TrimRight(l, "\r") != "WARNING: DATA RACE" {
continue
}
words := "WARNING: DATA RACE"
if i+1 < len(lines) && strings.TrimSpace(lines[i+1]) != "" {
words += ": " + strings.TrimSuffix(strings.TrimSpace(lines[i+1]), ":")
if i+2 < len(lines) && strings.HasPrefix(lines[i+2], " ") {
words += " in " + strings.TrimSpace(lines[i+2])
}
}
return words
}
for _, prefix := range []string{"panic: ", "fatal error: ", "signal: "} {
for _, l := range lines {
if l = strings.TrimRight(l, "\r"); strings.HasPrefix(l, prefix) {
return l
}
}
}
return ""
}
// FirstLine is the first non-empty line of a failure's words.
func FirstLine(s string) string {
for _, l := range strings.Split(s, "\n") {
if l = strings.TrimSpace(l); l != "" {
return l
}
}
return ""
}
// MaxFailureLine is the most bytes of a failure's line in a verdict's Failures: the first line of its words, which is
// its cause where the build module told one apart (issue 1901). Failed keeps the first failure's words whole.
const MaxFailureLine = 300
// LevelFailure turns a level that did not pass into a verdict: Failed names what failed first with its words whole,
// and Failures every failed test of the level's results with its first line, at most delivery.MaxFailures and an entry
// saying how many more (issue 1921).
func LevelFailure(repository, module, level string, results ...Tested) *delivery.Verdict {
v := delivery.Verdict{State: delivery.VerdictFail}
var all []delivery.FailedFirst
for _, res := range results {
if len(res.Failed) == 0 {
why := res.Why
if why == "" {
why = "the " + level + " level did not pass"
}
all = append(all, delivery.FailedFirst{Test: level + ": " + repository, Words: why})
continue
}
for _, f := range res.Failed {
name := FailedName(f)
if module == buildrun.NodeBuildModule && f.Package == "" {
name = res.Module
}
all = append(all, delivery.FailedFirst{Test: level + ": " + name, Words: FailureWordsAfter(f, HidWords(module, res.By, f))})
}
}
if len(all) == 0 { // no result: a level said failed with nothing to name
all = append(all, delivery.FailedFirst{Test: level + ": " + repository, Words: "the " + level + " level did not pass"})
}
v.Failed = all[0]
v.Failures = all // whole: Settled cuts them to their first lines once every failure of the check is joined
return &v
}
// JoinFailures is one failed verdict of every failure of vs, in the order given (a nil one passed), unsettled: Failed is
// the first whole, and Failures every failure whole, or none when the one failure is no level's (a conflict, a refused
// step: one line, as before issue 1921). Nil when none failed.
func JoinFailures(vs ...*delivery.Verdict) *delivery.Verdict {
var out *delivery.Verdict
var all []delivery.FailedFirst
listed := false
for _, v := range vs {
if v == nil {
continue
}
if out == nil {
out = &delivery.Verdict{State: delivery.VerdictFail, Failed: v.Failed}
}
if len(v.Failures) > 0 {
all, listed = append(all, v.Failures...), true
continue
}
all = append(all, v.Failed)
}
if out != nil && (listed || len(all) > 1) {
out.Failures = all
}
return out
}
// Settled is a joined verdict as it is written: each failure of its list cut to its first line, at most
// delivery.MaxFailures and then how many more (FailureLines).
func Settled(v *delivery.Verdict) *delivery.Verdict {
if v != nil && len(v.Failures) > 0 {
v.Failures = FailureLines(v.Failures)
}
return v
}
// AddFailure is a Settled list of failures with one more at its end: the first failure is put first when the list is
// empty (a verdict of one failure keeps none), and past delivery.MaxFailures the last entry counts one more.
func AddFailure(v *delivery.Verdict, f delivery.FailedFirst) {
if len(v.Failures) == 0 {
v.Failures = FailureLines([]delivery.FailedFirst{v.Failed})
}
if n := len(v.Failures); n > delivery.MaxFailures {
v.Failures[n-1].Test = fmt.Sprintf("and %d more", FailedCount(*v)-delivery.MaxFailures+1)
return
}
v.Failures = append(v.Failures, FailureLines([]delivery.FailedFirst{f})...)
}
// FailureLines is every failure with the first line of its words, cut to MaxFailureLine, at most delivery.MaxFailures
// and then one entry saying how many more failed.
func FailureLines(all []delivery.FailedFirst) []delivery.FailedFirst {
n := min(len(all), delivery.MaxFailures)
out := make([]delivery.FailedFirst, 0, n+1)
for _, f := range all[:n] {
out = append(out, delivery.FailedFirst{Test: f.Test, Machine: f.Machine, Words: ClipLine(FirstLine(f.Words), MaxFailureLine)})
}
if more := len(all) - n; more > 0 {
out = append(out, delivery.FailedFirst{Test: fmt.Sprintf("and %d more", more)})
}
return out
}
// ClipLine cuts a line to at most n bytes on a whole character, marking the cut.
func ClipLine(s string, n int) string {
if len(s) <= n {
return s
}
s = strings.ToValidUTF8(s[:n-3], "")
return s + "..."
}
// FailedCount is how many failures a verdict names, the ones after delivery.MaxFailures counted too.
func FailedCount(v delivery.Verdict) int {
n := len(v.Failures)
if n > delivery.MaxFailures {
var more int
if _, err := fmt.Sscanf(v.Failures[n-1].Test, "and %d more", &more); err == nil {
return delivery.MaxFailures + more
}
}
return n
}