mesh-delivery/modules/mesh-delivery-checker/internal/check/judge.go

package check

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"path"
	"slices"
	"sort"
	"strings"
	"time"

	"git.novox.be/novox/mesh-sdk/go/buildstep"
	"git.novox.be/novox/mesh-sdk/go/delivery"
	"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)

// goModule is the build module the Go repositories are judged by.
const goModule = buildrun.GoBuildModule

// RunnerWorkspace is the runner the Go build module's digest and test steps are asked with when the merge changes the build
// module itself: the installed module compiles the candidate's program from the merged tree and runs the step as it, shut
// off (ADR 0535 §1).
const RunnerWorkspace = "workspace"

// buildGoDir is the Go build module's directory in the catalogue: a merge changing a path under it is judged by its
// candidate (ADR 0535 §1). mesh-build-node's own directory is judged by the installed module until its part of issue 1898.
const buildGoDir = "modules/mesh-build-go/"

// changesBuildGo says whether a merge of the repository with these changed paths changes the Go build module.
func changesBuildGo(repository string, changed []string) bool {
	if repository != catalogueRepository {
		return false
	}
	for _, p := range changed {
		if strings.HasPrefix(p, buildGoDir) {
			return true
		}
	}
	return false
}

// The words of the line on the program that judged a change to the Go build module (ADR 0535 §2, §5).
const (
	judgedByOldInstalled = "judged by the installed mesh-build-go: it does not run a candidate yet"
	judgedByInstalled    = "judged by the installed mesh-build-go"
	judgedByCandidate    = "judged by the candidate's mesh-build-go at "
)

// verdictPtr is a verdict of failure a step of the check ends with.
type verdictPtr = *delivery.Verdict

func failed(machine, words string) verdictPtr { return fail(machine, words) }

// Asks puts a check's ask on its build seat's stream and waits for its step (buildrun.Asker; ADR 0525 §1, §6): the
// checker chooses no node, starts no step and holds none; whichever build node pulls the ask runs the check.
type Asks interface {
	Put(ctx context.Context, a delivery.StepAsk) error
	Wait(ctx context.Context, a delivery.StepAsk) (delivery.Step, error)
}

// ToolchainEntries are the nodes' toolchain entries of a language (buildrun.ToolchainSet).
type ToolchainEntries interface {
	Entries(language string) []delivery.NodeToolchain
}

// BuildModules says which build modules judge a repository: the Go one always, the Node one when the repository holds a
// TypeScript module.
type BuildModules interface {
	BuildModules(repository string) []string
}

// askLanguage is the build seat language of each build module the checker asks.
var askLanguage = map[string]string{buildrun.GoBuildModule: delivery.AskGo, buildrun.NodeBuildModule: delivery.AskNode}

// Examiner judges a case through the build seats (ADR 0484 clauses 3, 4, 10, 13; ADR 0525 §1, §3): for each pull request
// and build module it puts one check ask whose job fetches the head merged onto its trunk head, then runs the unit level
// and the integration level, taking the kept verdicts of unchanged packages, running the change's own tests first and
// stopping at the first failure; it judges the step's answer as it judged the tools' answers.
type Examiner struct {
	Asks Asks
	// Toolchains are the nodes' toolchains: a kept verdict names the toolchain the nodes agree on.
	Toolchains ToolchainEntries
	Kept       Kept
	// Previews is the previews bucket; nil writes none.
	Previews KV
	Modules  BuildModules
	// Runs says which commit the mesh runs of a module (the indexer's modules); nil knows none.
	Runs interface{ CommitOf(module string) string }
	// Forge reads the go.mod files of the Go members and says whether a mesh-sdk commit is on its trunk (ADR 0484 clause 9);
	// nil checks no pin.
	Forge Forge
	// Builds says how the build of a fingerprint the candidate carries stands (the builder's builds bucket); nil says nothing
	// of the builds.
	Builds BuildStates
	// Announce says a preview was written under the key (EventPreviewWritten); nil says nothing.
	Announce Announcer
	// Packages names, of a repository, the paths (repository-relative) its TypeScript package artifacts are built from, from
	// the indexer's modules (delivery.Module.ArtifactPaths, lead decision L8); nil serves no group package.
	Packages interface {
		PackagePaths(repository string) []string
	}
	// Trunks are the trunk heads of every repository, for the siblings a repository's tests read.
	Trunks interface{ Heads() map[string]string }
	// Compositions answers the assigner's composition check of a check by the check's key (delivery.CompositionKey),
	// waiting until the assigner wrote a verdict that does not wait, or ctx ends (ADR 0528 clause 5, issue 1297); nil
	// leaves the composition not run.
	Compositions CompositionReader
	Now          func() time.Time
	Logf         func(format string, args ...any)
}

// CompositionReader answers the assigner's composition of a check by the check's key, waiting until the assigner wrote
// a verdict that does not wait, or ctx ends.
type CompositionReader interface {
	Await(ctx context.Context, key string) (delivery.CompositionVerdict, error)
}

func (j *Examiner) now() time.Time {
	if j.Now != nil {
		return j.Now()
	}
	return time.Now()
}

func (j *Examiner) logf(format string, args ...any) {
	if j.Logf != nil {
		j.Logf(format, args...)
	}
}

// besideLines are the siblings as the digest step echoes them: `<owner/repo> <commit>`, sorted.
func besideLines(refs []BesideRef) []string {
	out := make([]string, 0, len(refs))
	for _, r := range refs {
		out = append(out, r.Repository+" "+r.Commit)
	}
	sort.Strings(out)
	return out
}

// target is a pull request on one build module: the node that pulled its ask, the workspace its fetch made and what it
// said.
type target struct {
	pull      Pull
	module    string
	node      string
	toolchain string
	fetched   Fetched
	// fetchArgs are the arguments of the fetch its check ask runs.
	fetchArgs map[string]any
	// priority are the Go packages (import paths) or the Node module directories whose test files the merge changes.
	changed []string
	// beside are the lines `<owner/repo> <commit>` of the siblings the workspace was given, sorted: its tests may read them,
	// so a verdict is kept or taken only under a digest that says it folded exactly these in (lead decision L5).
	beside []string
	// runner is RunnerWorkspace when the target's ask runs the candidate's own build module (ADR 0535 §1), empty otherwise.
	runner string
	// judged is what the line says of the program that judged a change to the build module; empty for any other change.
	judged string
}

// judgedBy takes what a step under runner workspace answered of the program that ran it (judged_by): the candidate, or the
// installed module (a module that does not take the runner says nothing, and is asked without it from now on: ADR 0535 §2).
// It says whether the candidate ran.
func (t *target) judgedBy(answer string) bool {
	switch {
	case strings.HasPrefix(answer, "candidate "):
		t.judged = judgedByCandidate + short12(strings.TrimPrefix(answer, "candidate "))
		return true
	case answer == "":
		t.judged, t.runner = judgedByOldInstalled, ""
	default:
		t.judged, t.runner = judgedByInstalled, ""
	}
	return false
}

// stepArgs are a step's arguments with the target's runner.
func (t *target) stepArgs(args map[string]any) map[string]any {
	if t.runner != "" {
		args["runner"] = t.runner
	}
	return args
}

// acc collects a level's counts across targets.
type acc struct {
	run, kept, skipped int
	elapsed, setup     float64
	lint               float64
	over               bool
	slowest            []TestTime
}

// state is one judging.
type state struct {
	j       *Examiner
	c       *Case
	targets []*target
	levels  map[string]*acc
	nodes   map[string]bool
	clone   time.Duration
	slow    []string
}

// Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot
// be judged at all.
func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) {
	st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}}
	if v := st.judgeAll(ctx); v != nil {
		return st.finish(ctx, *v), nil
	}
	return st.finish(ctx, delivery.Verdict{State: delivery.VerdictPass}), nil
}

// finish is the outcome of the verdict with the state of the builds the candidate's modules carry, and, for a check
// whose tests passed, the assigner's composition of it.
func (st *state) finish(ctx context.Context, v delivery.Verdict) Outcome {
	out := st.outcome(v)
	if out.Verdict.State == delivery.VerdictPass {
		st.composed(ctx, &out.Verdict)
	}
	out.Builds = st.j.buildNotes(ctx, st.c)
	return out
}

// composed reads the assigner's composition of the check into its verdict (ADR 0528 clause 5): a failed composition
// fails the check, naming the machine it failed at. A try is not composed (it has no candidate); a composition that
// cannot be had leaves it not run, said; with no assigner running it is not judged, said at once.
func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
	if st.j.Compositions == nil || st.c.Try != nil {
		return
	}
	comp, err := st.j.Compositions.Await(ctx, st.c.Key)
	switch {
	case errors.Is(err, ErrAssignerNotRunning):
		v.Composition = &delivery.Composition{State: delivery.CompositionNotRun, Words: delivery.AssignerNotRunning}
		return
	case err != nil:
		st.j.logf("%s: the composition could not be read, so it is said as not run: %v", st.c.Key, err)
		return
	}
	withComposition(v, comp)
}

// outcome fills the verdict's levels, times, machines and composition from what ran.
func (st *state) outcome(v delivery.Verdict) Outcome {
	if v.State == delivery.VerdictPass || len(v.Levels) == 0 {
		v.Levels = map[string]delivery.LevelResult{}
		for level, a := range st.levels {
			v.Levels[level] = delivery.LevelResult{Run: a.run, Kept: a.kept, Skipped: a.skipped}
		}
	}
	ms := func(s float64) int64 { return int64(s * 1000) }
	u, i := st.levels[delivery.LevelUnit], st.levels[delivery.LevelIntegration]
	v.Took = delivery.Took{Clone: st.clone.Milliseconds(), Unit: ms(u.elapsed), Lint: ms(u.lint), IntegrationSetup: ms(i.setup), IntegrationRun: ms(i.elapsed)}
	for node := range st.nodes {
		v.Machines = append(v.Machines, node)
	}
	sort.Strings(v.Machines)
	for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} {
		if st.levels[level].over {
			v.OverBound = append(v.OverBound, level)
		}
	}
	// The composition is the assigner's check (issue 1297): until it delivers a verdict says it was not run (ADR 0484 clause 6).
	v.Composition = &delivery.Composition{State: delivery.CompositionNotRun}
	var slow []string
	for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} {
		a := st.levels[level]
		if !a.over {
			continue
		}
		sort.SliceStable(a.slowest, func(x, y int) bool { return a.slowest[x].Elapsed > a.slowest[y].Elapsed })
		for n, s := range a.slowest {
			if n == 3 {
				break
			}
			slow = append(slow, fmt.Sprintf("%s %s %.1fs", level, path.Base(s.Package)+"."+s.Test, s.Elapsed))
		}
	}
	out := Outcome{Verdict: v, Slowest: slow}
	for _, t := range st.targets {
		if t.judged != "" {
			if out.Judged == nil {
				out.Judged = map[string]string{}
			}
			out.Judged[pullKey(Pull{Repository: t.pull.Repository, Number: t.pull.Number})] = t.judged
		}
	}
	return out
}

// fail is a verdict of failure at a machine.
func fail(machine, words string) *delivery.Verdict {
	v := failure(machine, words)
	return &v
}

// levels are the levels a case's asks run: both, or the one a try names.
func (st *state) levelsOf() []string {
	if st.c.Try != nil && st.c.Try.Level != "" {
		return []string{st.c.Try.Level} // a try of one level runs that level alone
	}
	return []string{delivery.LevelUnit, delivery.LevelIntegration}
}

// judgeAll puts one check ask per pull request and build module the repository needs, the mesh-sdk member first, and
// judges each answer. A head that does not merge, a pin that is not allowed or a level that fails ends the check at once
// (ADR 0484 clause 13).
func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
	// The mesh-sdk member first: whether its merge changes the TypeScript package's paths decides whether the other members'
	// Node fetches take the group's package (ADR 0500 §5).
	pulls := slices.Clone(st.c.Pulls)
	slices.SortStableFunc(pulls, func(a, b Pull) int {
		if (a.Repository == SDKRepository) != (b.Repository == SDKRepository) {
			if a.Repository == SDKRepository {
				return -1
			}
			return 1
		}
		return 0
	})
	var groupPackage map[string]any
	for _, p := range pulls {
		var merged []Fetched
		for _, bm := range st.j.Modules.BuildModules(p.Repository) {
			t := &target{pull: p, module: bm}
			if tc, none := delivery.AgreedToolchain(st.j.Toolchains.Entries(buildrun.LanguageOf[bm])); none == "" {
				t.toolchain = tc.Version
			}
			args := map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number), "base": p.Trunk}
			if bm == buildrun.NodeBuildModule && groupPackage != nil && p.Repository != SDKRepository {
				args["package"] = groupPackage
			}
			if bm == buildrun.GoBuildModule {
				beside, err := st.j.besideFor(ctx, st.c, p)
				if err != nil {
					return fail(forgeMachine(err, bm), err.Error()+unheardWords(err))
				}
				if len(beside) > 0 {
					args["beside"] = beside
					t.beside = besideLines(beside)
				}
				if p.Repository == catalogueRepository {
					t.runner = RunnerWorkspace // used by the build module when the merge changes it (ADR 0535 §1)
				}
			}
			t.fetchArgs = args
			st.targets = append(st.targets, t)
			if v := st.ask(ctx, t, &merged); v != nil {
				return v
			}
		}
		if st.c.Try == nil { // a try owes the indexer no preview: no pull request holds its head
			st.j.writePreview(ctx, p, merged)
		}
		if p.Repository == SDKRepository && len(st.c.Pulls) > 1 && st.j.changesPackage(p, merged) {
			groupPackage = map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number)}
		}
	}
	return nil
}

// ask puts the target's check ask, waits for the step of the node that pulled it and judges its answer: the fetch, the
// pins of its go.mod files, then each level's digest, kept verdicts and tests.
func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *delivery.Verdict {
	p := t.pull
	job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: st.levelsOf(), Kept: true}
	if t.runner != "" {
		job.Runner, job.RunnerWhen = t.runner, buildGoDir
	}
	if st.c.Try != nil && st.c.Try.Run != "" {
		job.Run, job.Kept = st.c.Try.Run, false // a try of some tests runs them alone: a partial run proves no package
	}
	kind := delivery.AskCheck
	if st.c.Kind == KindTrunk {
		kind = delivery.AskRecheck // a re-judge after a trunk move goes after every new head's check (ADR 0536 §4)
	}
	a := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: askLanguage[t.module], Asker: delivery.CheckerModule,
		At: st.j.now().UTC(), Check: job}
	st.c.Progress.Set(fmt.Sprintf("the %s ask %s of %s#%d", kind, a.ID, p.Repository, p.Number), "")
	if err := st.j.Asks.Put(ctx, a); err != nil {
		return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err))
	}
	step, err := st.j.Asks.Wait(ctx, a)
	if err != nil {
		if ctx.Err() != nil {
			return fail(t.module, ctx.Err().Error())
		}
		return fail(t.module, fmt.Sprintf("the check of %s#%d: %v", p.Repository, p.Number, err))
	}
	// The fetch and the digests are what of the ask's time its levels' running did not take (Took.Clone).
	took := time.Duration(0)
	if !step.Started.IsZero() && step.Ended.After(step.Started) {
		took = step.Ended.Sub(step.Started)
	}
	t.node = step.Node
	st.nodes[step.Node] = true
	st.c.Progress.Set(fmt.Sprintf("judge the check of %s#%d", p.Repository, p.Number), step.Node)
	if step.State != delivery.StepPassed {
		return fail(step.Node, fmt.Sprintf("the check of %s#%d %s: %s%s", p.Repository, p.Number, step.State, step.Reason, pinHint(st.c, step.Reason)))
	}
	var answer delivery.CheckAnswer
	if err := json.Unmarshal(step.Result, &answer); err != nil {
		return fail(step.Node, fmt.Sprintf("the check of %s#%d answered what is not a check: %s", p.Repository, p.Number, step.Result))
	}
	if err := json.Unmarshal(answer.Fetch, &t.fetched); err != nil || t.fetched.Workspace == "" {
		return fail(step.Node, fmt.Sprintf("the fetch of %s#%d answered no workspace: %s", p.Repository, p.Number, answer.Fetch))
	}
	if len(t.fetched.Conflicts) > 0 {
		if st.c.Try == nil {
			st.j.writePreview(ctx, p, append(*merged, t.fetched))
		}
		return fail(step.Node, fmt.Sprintf("%s#%d does not merge into the trunk: files in conflict: %s", p.Repository, p.Number, strings.Join(t.fetched.Conflicts, ", ")))
	}
	t.changed = t.fetched.Changed
	if !changesBuildGo(p.Repository, t.changed) {
		t.runner = "" // the build module used no runner: the merge does not change it
	}
	*merged = append(*merged, t.fetched)
	if v := st.checkPinsOf(ctx, t); v != nil {
		return v
	}
	ran := 0.0
	for _, la := range answer.Levels {
		v, elapsed := st.level(ctx, t, la)
		ran += elapsed
		if v != nil {
			st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
			return v
		}
	}
	st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
	return nil
}

// level judges one level of a check ask's answer as the tools' answers were judged: the digest keys the kept verdicts
// of the packages that passed, the kept ones taken are counted, and the first test result that did not pass is the
// verdict. It answers the level's running time (its tests' and setup's).
func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer) (*delivery.Verdict, float64) {
	level := la.Level
	a := st.levels[level]
	if a == nil {
		return fail(t.node, fmt.Sprintf("the check of %s#%d answered the level %q, which is no level", t.pull.Repository, t.pull.Number, level)), 0
	}
	var digests map[string]string
	if t.module == buildrun.GoBuildModule && la.Digest != nil {
		// The packages come as a list, never keyed by a package path (issue 1930); an answer stored before reads too.
		digests = la.Digest.Map()
		if t.runner != "" {
			t.judgedBy(la.Digest.JudgedBy) // the installed module judged it: the runner is not used again
		}
	}
	if t.runner != "" && la.Judged != "" {
		t.judgedBy(la.Judged) // the candidate judged the level: nothing is kept (ADR 0535 §4)
	}
	for _, k := range la.Kept {
		a.kept += k.Tests
	}
	elapsed := 0.0
	for _, raw := range la.Tests {
		var res Tested
		if err := json.Unmarshal(raw, &res); err != nil {
			return fail(t.node, fmt.Sprintf("the %s level of %s#%d answered what is not a result: %v", level, t.pull.Repository, t.pull.Number, err)), elapsed
		}
		if t.runner != "" {
			t.judgedBy(res.JudgedBy)
		}
		elapsed += res.Elapsed + res.Setup + res.Lint
		if t.module == buildrun.NodeBuildModule {
			if !res.NoTests {
				a.run++ // a Node package is counted whole: its build module counts no tests (ADR 0484 clause 10)
			}
		} else {
			a.run += res.Tests
			a.skipped += res.Skipped
			a.slowest = append(a.slowest, res.Slowest...)
		}
		a.elapsed += res.Elapsed
		a.setup += res.Setup
		a.lint += res.Lint
		a.over = a.over || res.OverBound
		if len(digests) > 0 {
			if _, err := st.j.Kept.KeepPasses(ctx, t.pull.Repository, level, t.toolchain, res, digests, st.j.now()); err != nil {
				st.j.logf("%s#%d: a passing package could not be kept: %v", t.pull.Repository, t.pull.Number, err)
			}
		}
		if !res.Passed {
			return levelFailure(t, level, res), elapsed
		}
	}
	return nil, elapsed
}

// changesPackage says whether the merge of a pull request changes a file inside the named paths of one of its repository's
// TypeScript packages: a change of the manifest alone, or of files outside those paths, does not (ADR 0500 §5).
func (j *Examiner) changesPackage(p Pull, merged []Fetched) bool {
	if j.Packages == nil {
		return false
	}
	paths := j.Packages.PackagePaths(p.Repository)
	for _, f := range merged {
		for _, changed := range f.Changed {
			for _, named := range paths {
				named = strings.TrimSuffix(strings.TrimPrefix(named, "./"), "/")
				if changed == named || strings.HasPrefix(changed, named+"/") {
					return true
				}
			}
		}
	}
	return false
}

// failedName names a failure: its package and test.
func failedName(f Failure) string {
	if f.Test == "" {
		return f.Package
	}
	return f.Package + "." + f.Test
}

// MaxWords is the most a failed test's words in a verdict hold (issue 1901). **The words keep the failure's last lines**,
// where a test says why it failed: the first line is often something it logged before (the group r4g verdict kept a
// "judged against" line and lost the t.Fatalf after it), and the 120 characters kept before cut a docker command's echo
// before docker's own error. Four kilobytes holds a long message and is small beside a verdict's other parts.
const MaxWords = 4096

// failureWords is a failure's words: its output's last lines up to MaxWords, with the line the build module told apart as
// the test's error first when the output does not start with it, so the first line of the words is the cause and the
// status line, which is cut short, names it. **A failure's words are never "PASS" or "ok"** (issue 1901, its third
// meeting: a package that failed after its tests passed was worded "PASS"): go test's summary lines are dropped, and with
// no cause told apart (an older build module) the cause is the output's first "panic:", "fatal error:" or "signal:"
// line, read before the output is cut, or the words say the tests passed and the program failed after them.
func failureWords(f Failure) string {
	out, passed := dropSummary(strings.TrimSpace(f.Output))
	cause := strings.TrimSpace(f.Cause)
	if cause == "" {
		cause = causeIn(out)
	}
	switch {
	case cause == "" && out == "" && passed:
		cause = "the tests passed, then the test program exited with a failure: an os.Exit or a panic in TestMain or after the tests"
	case cause == "" && out == "":
		// The build module on main sends a package that stopped (a timeout, an exit, a kill) as its "FAIL" line alone.
		cause = "the package failed with no test to name and no panic, signal or build error in its output (an older build module sent no cause)"
	}
	if len(cause) > MaxCause {
		cause = strings.ToValidUTF8(cause[:MaxCause], "") + "..."
	}
	// The words keep MaxWords exactly: the cause and its newline, the marker and the kept lines (review round 1).
	budget := MaxWords
	if cause != "" {
		budget -= len(cause) + 1
	}
	if len(out) > budget {
		const marker = "... "
		tail := out[len(out)-(budget-len(marker)):]
		// Start on a whole line, and a whole character.
		if i := strings.IndexByte(tail, '\n'); i >= 0 {
			tail = strings.TrimLeft(tail[i:], "\n")
		}
		out = marker + strings.ToValidUTF8(tail, "")
	}
	if cause != "" && !strings.HasPrefix(firstLine(out), cause) {
		out = strings.TrimRight(cause+"\n"+out, "\n")
	}
	return out
}

// MaxCause is the most a cause in a failure's words holds: one long line (the 1901 review: a cause of a kilobyte could
// push the words past MaxWords).
const MaxCause = 1024

// dropSummary drops go test's summary lines from a failure's output ("PASS", "FAIL", "ok <pkg>", "FAIL <pkg>"), and says
// whether a "PASS" was among them: the package's tests passed before it failed.
func dropSummary(out string) (string, bool) {
	passed := false
	lines := strings.Split(out, "\n")
	kept := lines[:0]
	for _, l := range lines {
		// go's exact forms only (review round 1): "PASS" or "FAIL" alone, "ok  \t<pkg>\t<time>", "FAIL\t<pkg>\t<time>".
		switch t := strings.TrimRight(l, "\r"); {
		case t == "PASS":
			passed = true
		case t == "FAIL", strings.HasPrefix(t, "ok  \t"), strings.HasPrefix(t, "FAIL\t"):
		default:
			kept = append(kept, l)
		}
	}
	return strings.TrimSpace(strings.Join(kept, "\n")), passed
}

// causeIn is a race the race detector reported, as its heading, its access and the frame that made it ("WARNING: DATA
// RACE: Write at 0x... by goroutine 9 in pkg.F.func1()"); else the first "panic:", "fatal error:" or "signal:" line of a
// failure's output, in that order; or "". **Only lines at column 0**, where go prints the runtime's: a test's indented log
// line holding the same words is not the cause (review round 1).
func causeIn(out string) string {
	lines := strings.Split(out, "\n")
	for i, l := range lines {
		if strings.TrimRight(l, "\r") != "WARNING: DATA RACE" {
			continue
		}
		words := "WARNING: DATA RACE"
		if i+1 < len(lines) && strings.TrimSpace(lines[i+1]) != "" {
			words += ": " + strings.TrimSuffix(strings.TrimSpace(lines[i+1]), ":")
			if i+2 < len(lines) && strings.HasPrefix(lines[i+2], "  ") {
				words += " in " + strings.TrimSpace(lines[i+2])
			}
		}
		return words
	}
	for _, prefix := range []string{"panic: ", "fatal error: ", "signal: "} {
		for _, l := range lines {
			if l = strings.TrimRight(l, "\r"); strings.HasPrefix(l, prefix) {
				return l
			}
		}
	}
	return ""
}

// firstLine is the first non-empty line of a failure's words.
func firstLine(s string) string {
	for _, l := range strings.Split(s, "\n") {
		if l = strings.TrimSpace(l); l != "" {
			return l
		}
	}
	return ""
}

// failure turns a level that did not pass into a verdict naming what failed first.
func levelFailure(t *target, level string, res Tested) *delivery.Verdict {
	v := delivery.Verdict{State: delivery.VerdictFail}
	switch {
	case len(res.Failed) > 0:
		f := res.Failed[0]
		name := failedName(f)
		if t.module == buildrun.NodeBuildModule && f.Package == "" {
			name = res.Module
		}
		v.Failed = delivery.FailedFirst{Test: level + ": " + name, Words: failureWords(f)}
	default:
		why := res.Why
		if why == "" {
			why = "the " + level + " level did not pass"
		}
		v.Failed = delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: why}
	}
	return &v
}