mesh-delivery/modules/mesh-delivery-checker/internal/check/judge.go

package check

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"path"
	"slices"
	"sort"
	"strings"
	"time"

	"git.novox.be/novox/mesh-sdk/go/buildstep"
	"git.novox.be/novox/mesh-sdk/go/delivery"
	"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)

// goModule is the build module the Go repositories are judged by.
const goModule = buildrun.GoBuildModule

// RunnerWorkspace is the runner the Go build module's digest and test steps are asked with when the merge changes the build
// module itself: the installed module compiles the candidate's program from the merged tree and runs the step as it, shut
// off (ADR 0535 §1).
const RunnerWorkspace = "workspace"

// buildGoDir is the Go build module's directory in the catalogue: a merge changing a path under it is judged by its
// candidate (ADR 0535 §1). mesh-build-node's own directory is judged by the installed module until its part of issue 1898.
const buildGoDir = "modules/mesh-build-go/"

// changesBuildGo says whether a merge of the repository with these changed paths changes the Go build module.
func changesBuildGo(repository string, changed []string) bool {
	if repository != catalogueRepository {
		return false
	}
	for _, p := range changed {
		if strings.HasPrefix(p, buildGoDir) {
			return true
		}
	}
	return false
}

// The words of the line on the program that judged a change to the Go build module (ADR 0535 §2, §5).
const (
	judgedByOldInstalled = "judged by the installed mesh-build-go: it does not run a candidate yet"
	judgedByInstalled    = "judged by the installed mesh-build-go"
	judgedByCandidate    = "judged by the candidate's mesh-build-go at "
)

// verdictPtr is a verdict of failure a step of the check ends with.
type verdictPtr = *delivery.Verdict

func failed(machine, words string) verdictPtr { return fail(machine, words) }

// Asks puts a check's ask on its build seat's stream and waits for its step (buildrun.Asker; ADR 0525 §1, §6): the
// checker chooses no node, starts no step and holds none; whichever build node pulls the ask runs the check.
type Asks interface {
	Put(ctx context.Context, a delivery.StepAsk) error
	Wait(ctx context.Context, a delivery.StepAsk) (delivery.Step, error)
}

// ToolchainEntries are the nodes' toolchain entries of a language (buildrun.ToolchainSet).
type ToolchainEntries interface {
	Entries(language string) []delivery.NodeToolchain
}

// BuildModules says which build modules judge a repository: the Go one always, the Node one when the repository holds a
// TypeScript module.
type BuildModules interface {
	BuildModules(repository string) []string
}

// askLanguage is the build seat language of each build module the checker asks.
var askLanguage = map[string]string{buildrun.GoBuildModule: delivery.AskGo, buildrun.NodeBuildModule: delivery.AskNode}

// Examiner judges a case through the build seats (ADR 0484 clauses 3, 4, 10, 13; ADR 0525 §1, §3): for each pull request
// and build module it puts one check ask whose job fetches the head merged onto its trunk head, then runs the unit level
// and the integration level, taking the kept verdicts of unchanged packages, running the change's own tests first and
// stopping at the first failure; it judges the step's answer as it judged the tools' answers.
type Examiner struct {
	Asks Asks
	// DeclineBound is how long a placed ask may go on being declined, from its first decline, before the check fails
	// saying no node with its need took it (issue 890, ADR 0552); DefaultDeclineBound when zero.
	DeclineBound time.Duration
	// Toolchains are the nodes' toolchains: a kept verdict names the toolchain the nodes agree on.
	Toolchains ToolchainEntries
	Kept       Kept
	// Previews is the previews bucket; nil writes none.
	Previews KV
	Modules  BuildModules
	// Runs says which commit the mesh runs of a module (the indexer's modules); nil knows none.
	Runs interface{ CommitOf(module string) string }
	// Forge reads the forge's files for the siblings a repository's tests read; nil reads none.
	Forge Forge
	// Published names the latest published version of every mesh package a module of the mesh uses, by package, from the
	// indexer's modules (novox/hq ADR 0573 §2): a check builds against those, or against a member's head for a package
	// its group publishes (§4), whatever go.mod names. Nil names none, and go.mod's versions are built.
	Published interface {
		LatestPackages() delivery.PackageVersions
	}
	// Builds says how the build of a fingerprint the candidate carries stands (the builder's builds bucket); nil says nothing
	// of the builds.
	Builds BuildStates
	// Announce says a preview was written under the key (EventPreviewWritten); nil says nothing.
	Announce Announcer
	// Packages names, of a repository, the paths (repository-relative) its TypeScript package artifacts are built from, from
	// the indexer's modules (delivery.Module.ArtifactPaths, lead decision L8); nil serves no group package.
	Packages interface {
		PackagePaths(repository string) []string
	}
	// Trunks are the trunk heads of every repository, for the siblings a repository's tests read.
	Trunks interface{ Heads() map[string]string }
	// Compositions answers the assigner's composition check of a check by the check's key (delivery.CompositionKey),
	// waiting until the assigner wrote a verdict that does not wait, or ctx ends (ADR 0528 clause 5, issue 1297); nil
	// leaves the composition not run.
	Compositions CompositionReader
	// Nodes names every node with a record, read from each `<node>.node` of the node-engine's have (novox/hq ADR 0567
	// §12): every check ask carries them (CheckJob.NodeNames) for the names check, so no repository lists them. A check
	// whose names cannot be read, or that reads none, is not asked and fails saying so; nil hands no names.
	Nodes interface {
		Names(ctx context.Context) ([]string, error)
	}
	Now  func() time.Time
	Logf func(format string, args ...any)
}

// CompositionReader answers the assigner's composition of a check by the check's key, waiting until the assigner wrote
// a verdict that does not wait, or ctx ends.
type CompositionReader interface {
	Await(ctx context.Context, key string) (delivery.CompositionVerdict, error)
}

func (j *Examiner) now() time.Time {
	if j.Now != nil {
		return j.Now()
	}
	return time.Now()
}

func (j *Examiner) logf(format string, args ...any) {
	if j.Logf != nil {
		j.Logf(format, args...)
	}
}

// besideLines are the siblings as the digest step echoes them: `<owner/repo> <commit>`, sorted.
func besideLines(refs []BesideRef) []string {
	out := make([]string, 0, len(refs))
	for _, r := range refs {
		out = append(out, r.Repository+" "+r.Commit)
	}
	sort.Strings(out)
	return out
}

// target is a pull request on one build module: the node that pulled its ask, the workspace its fetch made and what it
// said.
type target struct {
	pull      Pull
	module    string
	node      string
	toolchain string
	fetched   Fetched
	// fetchArgs are the arguments of the fetch its check ask runs.
	fetchArgs map[string]any
	// priority are the Go packages (import paths) or the Node module directories whose test files the merge changes.
	changed []string
	// beside are the lines `<owner/repo> <commit>` of the siblings the workspace was given, sorted: its tests may read them,
	// so a verdict is kept or taken only under a digest that says it folded exactly these in (lead decision L5).
	beside []string
	// runner is RunnerWorkspace when the target's ask runs the candidate's own build module (ADR 0535 §1), empty otherwise.
	runner string
	// judged is what the line says of the program that judged a change to the build module; empty for any other change.
	judged string
}

// judgedBy takes what a step under runner workspace answered of the program that ran it (judged_by): the candidate, or the
// installed module (a module that does not take the runner says nothing, and is asked without it from now on: ADR 0535 §2).
// It says whether the candidate ran.
func (t *target) judgedBy(answer string) bool {
	switch {
	case strings.HasPrefix(answer, "candidate "):
		t.judged = judgedByCandidate + short12(strings.TrimPrefix(answer, "candidate "))
		return true
	case answer == "":
		t.judged, t.runner = judgedByOldInstalled, ""
	default:
		t.judged, t.runner = judgedByInstalled, ""
	}
	return false
}

// stepArgs are a step's arguments with the target's runner.
func (t *target) stepArgs(args map[string]any) map[string]any {
	if t.runner != "" {
		args["runner"] = t.runner
	}
	return args
}

// acc collects a level's counts across targets.
type acc struct {
	run, kept, skipped int
	elapsed, setup     float64
	lint               float64
	over               bool
	slowest            []TestTime
}

// state is one judging.
type state struct {
	j       *Examiner
	c       *Case
	targets []*target
	levels  map[string]*acc
	nodes   map[string]bool
	clone   time.Duration
	slow    []string
	// beside are the siblings read at their trunk's head, by repository (ADR 0548 §6).
	beside map[string]string
}

// Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot
// be judged at all.
func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) {
	st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{},
		beside: map[string]string{}}
	if v := st.judgeAll(ctx); v != nil {
		return st.finish(ctx, *v), nil
	}
	return st.finish(ctx, delivery.Verdict{State: delivery.VerdictPass}), nil
}

// finish is the outcome of the verdict with the state of the builds the candidate's modules carry, and, for a check
// whose tests passed, the assigner's composition of it.
func (st *state) finish(ctx context.Context, v delivery.Verdict) Outcome {
	out := st.outcome(v)
	if out.Verdict.State == delivery.VerdictPass {
		st.composed(ctx, &out.Verdict)
	}
	out.Builds = st.j.buildNotes(ctx, st.c)
	return out
}

// composed reads the assigner's composition of the check into its verdict (ADR 0528 clause 5): a failed composition
// fails the check, naming the machine it failed at. A try is not composed (it has no candidate); a composition that
// cannot be had leaves it not run, said; with no assigner running it is not judged, said at once.
func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
	if st.c.Composition != nil && st.c.Try == nil {
		// A carry whose candidate's fingerprints did not change keeps the composition it passed (ADR 0548 §5).
		kept := *st.c.Composition
		v.Composition = &kept
		st.c.note("the composition is kept: no fingerprint of a module it composed changed")
		return
	}
	if st.j.Compositions == nil || st.c.Try != nil {
		return
	}
	comp, err := st.j.Compositions.Await(ctx, st.c.Key)
	switch {
	case errors.Is(err, ErrAssignerNotRunning):
		v.Composition = &delivery.Composition{State: delivery.CompositionNotRun, Words: delivery.AssignerNotRunning}
		return
	case err != nil:
		st.j.logf("%s: the composition could not be read, so it is said as not run: %v", st.c.Key, err)
		return
	}
	withComposition(v, comp)
	if v.Composition != nil && v.Composition.State != delivery.CompositionNotRun {
		v.Composition.FPSet = st.c.FPSet // a carry keeps it while the fingerprints stay the same (ADR 0548 §5)
	}
}

// outcome fills the verdict's levels, times, machines and composition from what ran.
func (st *state) outcome(v delivery.Verdict) Outcome {
	if v.State == delivery.VerdictPass || len(v.Levels) == 0 {
		v.Levels = map[string]delivery.LevelResult{}
		for level, a := range st.levels {
			v.Levels[level] = delivery.LevelResult{Run: a.run, Kept: a.kept, Skipped: a.skipped}
		}
	}
	ms := func(s float64) int64 { return int64(s * 1000) }
	u, i := st.levels[delivery.LevelUnit], st.levels[delivery.LevelIntegration]
	v.Took = delivery.Took{Clone: st.clone.Milliseconds(), Unit: ms(u.elapsed), Lint: ms(u.lint), IntegrationSetup: ms(i.setup), IntegrationRun: ms(i.elapsed)}
	for node := range st.nodes {
		v.Machines = append(v.Machines, node)
	}
	sort.Strings(v.Machines)
	for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} {
		if st.levels[level].over {
			v.OverBound = append(v.OverBound, level)
		}
	}
	// The composition is the assigner's check (issue 1297): until it delivers a verdict says it was not run (ADR 0484 clause 6).
	v.Composition = &delivery.Composition{State: delivery.CompositionNotRun}
	var slow []string
	for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} {
		a := st.levels[level]
		if !a.over {
			continue
		}
		sort.SliceStable(a.slowest, func(x, y int) bool { return a.slowest[x].Elapsed > a.slowest[y].Elapsed })
		for n, s := range a.slowest {
			if n == 3 {
				break
			}
			slow = append(slow, fmt.Sprintf("%s %s %.1fs", level, path.Base(s.Package)+"."+s.Test, s.Elapsed))
		}
	}
	out := Outcome{Verdict: v, Slowest: slow}
	if len(st.beside) > 0 {
		out.Beside = st.beside
	}
	for _, t := range st.targets {
		if t.judged != "" {
			if out.Judged == nil {
				out.Judged = map[string]string{}
			}
			out.Judged[pullKey(Pull{Repository: t.pull.Repository, Number: t.pull.Number})] = t.judged
		}
	}
	return out
}

// fail is a verdict of failure at a machine.
func fail(machine, words string) *delivery.Verdict {
	v := failure(machine, words)
	return &v
}

// levels are the levels a case's asks run: both, or the one a try names.
func (st *state) levelsOf() []string {
	if st.c.Try != nil && st.c.Try.Level != "" {
		return []string{st.c.Try.Level} // a try of one level runs that level alone
	}
	return []string{delivery.LevelUnit, delivery.LevelIntegration}
}

// judgeAll puts one check ask per pull request and build module the repository needs, the mesh-sdk member first, and
// judges each answer. A head that does not merge, a pin that is not allowed or a level that fails ends the check at once
// (ADR 0484 clause 13).
func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
	// The mesh-sdk member first: whether its merge changes the TypeScript package's paths decides whether the other members'
	// Node fetches take the group's package (ADR 0500 §5).
	pulls := slices.Clone(st.c.Pulls)
	slices.SortStableFunc(pulls, func(a, b Pull) int {
		if (a.Repository == SDKRepository) != (b.Repository == SDKRepository) {
			if a.Repository == SDKRepository {
				return -1
			}
			return 1
		}
		return 0
	})
	var groupPackage map[string]any
	for _, p := range pulls {
		var merged []Fetched
		for _, bm := range st.j.Modules.BuildModules(p.Repository) {
			t := &target{pull: p, module: bm}
			if tc, none := delivery.AgreedToolchain(st.j.Toolchains.Entries(buildrun.LanguageOf[bm])); none == "" {
				t.toolchain = tc.Version
			}
			args := map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number), "base": p.Trunk}
			if bm == buildrun.NodeBuildModule && groupPackage != nil && p.Repository != SDKRepository {
				args["package"] = groupPackage
			}
			if bm == buildrun.GoBuildModule {
				beside, err := st.j.besideFor(ctx, st.c, p)
				if err != nil {
					return fail(forgeMachine(err, bm), err.Error()+unheardWords(err))
				}
				if len(beside) > 0 {
					args["beside"] = beside
					t.beside = besideLines(beside)
					besideAtTrunk(st.c, beside, st.beside)
				}
				if p.Repository == catalogueRepository {
					t.runner = RunnerWorkspace // used by the build module when the merge changes it (ADR 0535 §1)
				}
			}
			t.fetchArgs = args
			st.targets = append(st.targets, t)
			if v := st.ask(ctx, t, &merged); v != nil {
				return v
			}
		}
		if st.c.Try == nil { // a try owes the indexer no preview: no pull request holds its head
			st.j.writePreview(ctx, p, merged)
		}
		if p.Repository == SDKRepository && len(st.c.Pulls) > 1 && st.j.changesPackage(p, merged) {
			groupPackage = map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number)}
		}
	}
	return nil
}

// DefaultDeclineBound is an Examiner's DeclineBound when it names none: longer than a check runs, so a node with the
// need that is busy is waited for, and short enough that a mesh where no node has it says so.
const DefaultDeclineBound = 30 * time.Minute

// ask puts the target's check ask, waits for the step of the node that pulled it and judges its answer: the fetch, then
// each level's digest, kept verdicts and tests. When the answer leaves packages unmet,
// their tests needing what that node lacks (issue 890), the passes are kept and the levels from the first such one are
// asked again of a node that has it (placed).
func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *delivery.Verdict {
	return st.askNeeding(ctx, t, merged, st.levelsOf(), nil, nil)
}

// askNeeding is ask with the levels to run, what the node must have and the packages it tests: none for the first ask
// of a target; for the placed one the unmet packages' needs and those packages alone, so no package the first ask judged
// is run or counted again (ADR 0552 §4).
func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, levels, needs, packages []string) *delivery.Verdict {
	p := t.pull
	placed := len(needs) > 0
	job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: levels, Kept: true, Packages: packages}
	if st.j.Nodes != nil {
		names, err := st.j.Nodes.Names(ctx)
		if err == nil && len(names) == 0 {
			err = fmt.Errorf("no node record is on the bus")
		}
		if err != nil {
			return fail(t.module, fmt.Sprintf("the check of %s#%d is not asked: the nodes' names the names check needs cannot be read: %v", p.Repository, p.Number, err))
		}
		job.NodeNames = names
	}
	if t.runner != "" {
		job.Runner, job.RunnerWhen = t.runner, buildGoDir
	}
	if st.c.Try != nil && st.c.Try.Run != "" {
		job.Run, job.Kept = st.c.Try.Run, false // a try of some tests runs them alone: a partial run proves no package
	}
	kind := delivery.AskCheck
	if st.c.Kind == KindTrunk {
		kind = delivery.AskRecheck // a re-judge after a trunk move goes after every new head's check (ADR 0536 §4)
	}
	a := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: askLanguage[t.module], Asker: delivery.CheckerModule,
		At: st.j.now().UTC(), Check: job, Needs: needs}
	if a.Language == delivery.AskGo {
		a.Packages = st.packagesOf(p.Repository)
	}
	var step delivery.Step
	var firstDecline time.Time
	for {
		what := fmt.Sprintf("the %s ask %s of %s#%d", kind, a.ID, p.Repository, p.Number)
		if placed {
			what += " for a node with " + strings.Join(needs, ", ")
		}
		st.c.Progress.Set(what, "")
		st.c.Asked(a.ID)
		if err := st.j.Asks.Put(ctx, a); err != nil {
			return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err))
		}
		waiting := ctx
		if watch := st.c.Watch; watch != nil {
			id := a.ID
			watch(id, nil) // put: a try names it at once, not only when it ends (issue 1890)
			waiting = buildrun.WithStepSeen(ctx, func(s delivery.Step) { watch(id, &s) })
		}
		var err error
		step, err = st.j.Asks.Wait(waiting, a)
		if err != nil {
			if ctx.Err() != nil {
				return fail(t.module, ctx.Err().Error())
			}
			return fail(t.module, fmt.Sprintf("the check of %s#%d: %v", p.Repository, p.Number, err))
		}
		d, declined := delivery.DeclinedOf(step)
		if !declined {
			break
		}
		// A node that lacks the need dropped the ask (issue 890): it is put again, for a node that has it to take; the
		// node that declined looks at the stream again only after its next pull's wait.
		now := st.j.now()
		if firstDecline.IsZero() {
			firstDecline = now
		}
		bound := st.j.DeclineBound
		if bound <= 0 {
			bound = DefaultDeclineBound
		}
		if now.Sub(firstDecline) > bound {
			return fail(step.Node, fmt.Sprintf("the check of %s#%d needs a node with %s, and none took it in %s: every node that pulled it lacks it (last %s)",
				p.Repository, p.Number, strings.Join(needs, ", "), bound, step.Node))
		}
		st.j.logf("%s#%d: the ask %s was declined on %s, which lacks %s; asked again", p.Repository, p.Number, a.ID, step.Node, strings.Join(d.Lacks, ", "))
		a.ID, a.At = buildstep.NewID(), now.UTC()
	}
	// The fetch and the digests are what of the ask's time its levels' running did not take (Took.Clone).
	took := time.Duration(0)
	if !step.Started.IsZero() && step.Ended.After(step.Started) {
		took = step.Ended.Sub(step.Started)
	}
	t.node = step.Node
	st.nodes[step.Node] = true
	st.c.Progress.Set(fmt.Sprintf("judge the check of %s#%d", p.Repository, p.Number), step.Node)
	if step.State != delivery.StepPassed {
		return fail(step.Node, fmt.Sprintf("the check of %s#%d %s: %s", p.Repository, p.Number, step.State, step.Reason))
	}
	var answer delivery.CheckAnswer
	if err := json.Unmarshal(step.Result, &answer); err != nil {
		return fail(step.Node, fmt.Sprintf("the check of %s#%d answered what is not a check: %s", p.Repository, p.Number, step.Result))
	}
	if err := json.Unmarshal(answer.Fetch, &t.fetched); err != nil || t.fetched.Workspace == "" {
		return fail(step.Node, fmt.Sprintf("the fetch of %s#%d answered no workspace: %s", p.Repository, p.Number, answer.Fetch))
	}
	if len(t.fetched.Conflicts) > 0 {
		if st.c.Try == nil {
			st.j.writePreview(ctx, p, append(*merged, t.fetched))
		}
		return fail(step.Node, fmt.Sprintf("%s#%d %s%s", p.Repository, p.Number, ConflictWords, strings.Join(t.fetched.Conflicts, ", ")))
	}
	t.changed = t.fetched.Changed
	if !changesBuildGo(p.Repository, t.changed) {
		t.runner = "" // the build module used no runner: the merge does not change it
	}
	if !placed { // the placed ask fetched the same merge: its preview was read from the first
		*merged = append(*merged, t.fetched)
		if v := st.checkBinariesOf(t); v != nil {
			return v
		}
	}
	ran := 0.0
	for _, la := range answer.Levels {
		if placed {
			if len(la.Unmet) > 0 {
				st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
				return fail(step.Node, fmt.Sprintf("the %s level of %s#%d left %s untested on %s, which was asked for a node with %s",
					la.Level, p.Repository, p.Number, unmetNames(la.Unmet), step.Node, strings.Join(needs, ", ")))
			}
			la.Kept = nil // it names its packages and takes no other's kept verdict; this is only a guard against an older module
		}
		v, elapsed := st.level(ctx, t, la)
		ran += elapsed
		if v != nil {
			st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
			return v
		}
	}
	st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
	if unmet := delivery.UnmetNeeds(answer.Levels); len(unmet) > 0 && !placed {
		// Every level passed, its passes kept, and some packages were left to a node with what their tests need: the levels
		// from the first that left one are asked of such a node, which takes the kept passes and tests those (issue 890).
		var again, packages []string
		seen := map[string]bool{}
		for _, la := range answer.Levels {
			if len(again) > 0 || len(la.Unmet) > 0 {
				again = append(again, la.Level)
			}
			for _, u := range la.Unmet {
				if !seen[u.Package] {
					seen[u.Package] = true
					packages = append(packages, u.Package)
				}
			}
		}
		st.j.logf("%s#%d: %s on %s left untested; asked of a node with %s", p.Repository, p.Number, unmetOf(answer.Levels), step.Node, strings.Join(unmet, ", "))
		return st.askNeeding(ctx, t, merged, again, unmet, packages)
	}
	return nil
}

// unmetNames names the unmet packages and their needs.
func unmetNames(unmet []delivery.PackageNeeds) string {
	var out []string
	for _, u := range unmet {
		out = append(out, u.Package+" (needs "+strings.Join(u.Needs, ", ")+")")
	}
	return strings.Join(out, ", ")
}

// unmetOf names the unmet packages of every level.
func unmetOf(levels []delivery.LevelAnswer) string {
	var out []string
	for _, la := range levels {
		if len(la.Unmet) > 0 {
			out = append(out, "at "+la.Level+" "+unmetNames(la.Unmet))
		}
	}
	return strings.Join(out, "; ")
}

// level judges one level of a check ask's answer as the tools' answers were judged: the digest keys the kept verdicts
// of the packages that passed, the kept ones taken are counted, and every test result is read and counted, also after
// one that did not pass, so the verdict names every failed test of the level (issue 1921). It answers the level's
// running time (its tests' and setup's).
func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer) (*delivery.Verdict, float64) {
	level := la.Level
	a := st.levels[level]
	if a == nil {
		return fail(t.node, fmt.Sprintf("the check of %s#%d answered the level %q, which is no level", t.pull.Repository, t.pull.Number, level)), 0
	}
	var digests map[string]string
	if t.module == buildrun.GoBuildModule && la.Digest != nil {
		// The packages come as a list, never keyed by a package path (issue 1930); an answer stored before reads too.
		digests = la.Digest.Map()
		if t.runner != "" {
			t.judgedBy(la.Digest.JudgedBy) // the installed module judged it: the runner is not used again
		}
	}
	if t.runner != "" && la.Judged != "" {
		t.judgedBy(la.Judged) // the candidate judged the level: nothing is kept (ADR 0535 §4)
	}
	for _, k := range la.Kept {
		a.kept += k.Tests
	}
	elapsed := 0.0
	var failedResults []Tested
	for _, raw := range la.Tests {
		var res Tested
		if err := json.Unmarshal(raw, &res); err != nil {
			return fail(t.node, fmt.Sprintf("the %s level of %s#%d answered what is not a result: %v", level, t.pull.Repository, t.pull.Number, err)), elapsed
		}
		if t.runner != "" {
			t.judgedBy(res.JudgedBy)
		}
		elapsed += res.Elapsed + res.Setup + res.Lint
		if t.module == buildrun.NodeBuildModule {
			if !res.NoTests {
				a.run++ // a Node package is counted whole: its build module counts no tests (ADR 0484 clause 10)
			}
		} else {
			a.run += res.Tests
			a.skipped += res.Skipped
			a.slowest = append(a.slowest, res.Slowest...)
		}
		a.elapsed += res.Elapsed
		a.setup += res.Setup
		a.lint += res.Lint
		a.over = a.over || res.OverBound
		if len(digests) > 0 {
			if _, err := st.j.Kept.KeepPasses(ctx, t.pull.Repository, level, t.toolchain, res, digests, st.j.now()); err != nil {
				st.j.logf("%s#%d: a passing package could not be kept: %v", t.pull.Repository, t.pull.Number, err)
			}
		}
		if !res.Passed {
			failedResults = append(failedResults, res)
		}
	}
	if len(failedResults) > 0 {
		return levelFailure(t, level, failedResults...), elapsed
	}
	return nil, elapsed
}

// changesPackage says whether the merge of a pull request changes a file inside the named paths of one of its repository's
// TypeScript packages: a change of the manifest alone, or of files outside those paths, does not (ADR 0500 §5).
func (j *Examiner) changesPackage(p Pull, merged []Fetched) bool {
	if j.Packages == nil {
		return false
	}
	paths := j.Packages.PackagePaths(p.Repository)
	for _, f := range merged {
		for _, changed := range f.Changed {
			for _, named := range paths {
				named = strings.TrimSuffix(strings.TrimPrefix(named, "./"), "/")
				if changed == named || strings.HasPrefix(changed, named+"/") {
					return true
				}
			}
		}
	}
	return false
}

// failedName names a failure: its package and test.
func failedName(f Failure) string {
	if f.Test == "" {
		return f.Package
	}
	return f.Package + "." + f.Test
}

// MaxWords is the most a failed test's words in a verdict hold (issue 1901). **The words keep the failure's last lines**,
// where a test says why it failed: the first line is often something it logged before (the group r4g verdict kept a
// "judged against" line and lost the t.Fatalf after it), and the 120 characters kept before cut a docker command's echo
// before docker's own error. Four kilobytes holds a long message and is small beside a verdict's other parts.
const MaxWords = 4096

// failureWords is a failure's words: its output's last lines up to MaxWords, with the line the build module told apart as
// the test's error first when the output does not start with it, so the first line of the words is the cause and the
// status line, which is cut short, names it. **A failure's words are never "PASS" or "ok"** (issue 1901, its third
// meeting: a package that failed after its tests passed was worded "PASS"): go test's summary lines are dropped, and with
// no cause told apart (an older build module) the cause is the output's first "panic:", "fatal error:" or "signal:"
// line, read before the output is cut, or the words say the tests passed and the program failed after them.
func failureWords(f Failure) string {
	return failureWordsAfter(f, "")
}

// failureWordsAfter is failureWords with the line loud put first, within the same MaxWords: what the checker itself says of
// the failure (that the build module hid it), which the status line then names.
func failureWordsAfter(f Failure, loud string) string {
	out, passed := dropSummary(strings.TrimSpace(f.Output))
	cause := strings.TrimSpace(f.Cause)
	if cause == "" {
		cause = causeIn(out)
	}
	switch {
	case cause == "" && out == "" && passed:
		cause = "the tests passed, then the test program exited with a failure: an os.Exit or a panic in TestMain or after the tests"
	case cause == "" && out == "":
		// The build module on main sends a package that stopped (a timeout, an exit, a kill) as its "FAIL" line alone.
		cause = "the package failed with no test to name and no panic, signal or build error in its output (an older build module sent no cause)"
	}
	if len(cause) > MaxCause {
		cause = strings.ToValidUTF8(cause[:MaxCause], "") + "..."
	}
	// The words keep MaxWords exactly: the loud line, the cause and their newlines, the marker and the kept lines.
	budget := MaxWords
	if cause != "" {
		budget -= len(cause) + 1
	}
	if loud != "" {
		budget -= len(loud) + 1
	}
	if len(out) > budget {
		const marker = "... "
		tail := out[len(out)-(budget-len(marker)):]
		// Start on a whole line, and a whole character.
		if i := strings.IndexByte(tail, '\n'); i >= 0 {
			tail = strings.TrimLeft(tail[i:], "\n")
		}
		out = marker + strings.ToValidUTF8(tail, "")
	}
	if cause != "" && !strings.HasPrefix(firstLine(out), cause) {
		out = strings.TrimRight(cause+"\n"+out, "\n")
	}
	if loud != "" {
		out = strings.TrimRight(loud+"\n"+out, "\n")
	}
	return out
}

// hidWords is the checker's loud line for a failure the build module sent hiding its cause (issue 1901, its thorough
// fix): with no cause, or a passing line as its cause, under the failure contract (buildstep.Hidden). It names the build
// module and the version it says it is (Tested.By, "<module> with mesh-sdk <version>"); one from before the contract
// says nothing of itself. "" when the failure keeps the contract. The check fails either way: only its words change.
func hidWords(module, by string, f Failure) string {
	why := buildstep.Hidden(f)
	if why == "" {
		return ""
	}
	version := "a version before the failure contract"
	if module == "" {
		module = "an unnamed build module"
	}
	if m, v, ok := strings.Cut(by, " with "); ok && m != "" {
		module, version = m, v
	}
	return fmt.Sprintf("the build module hid the failure (%s at %s): %s", module, version, why)
}

// MaxCause is the most a cause in a failure's words holds: one long line (the 1901 review: a cause of a kilobyte could
// push the words past MaxWords).
const MaxCause = 1024

// dropSummary drops go test's summary lines from a failure's output ("PASS", "FAIL", "ok <pkg>", "FAIL <pkg>"), and says
// whether a "PASS" was among them: the package's tests passed before it failed.
func dropSummary(out string) (string, bool) {
	passed := false
	lines := strings.Split(out, "\n")
	kept := lines[:0]
	for _, l := range lines {
		// go's exact forms only (review round 1): "PASS" or "FAIL" alone, "ok  \t<pkg>\t<time>", "FAIL\t<pkg>\t<time>".
		switch t := strings.TrimRight(l, "\r"); {
		case t == "PASS":
			passed = true
		case t == "FAIL", strings.HasPrefix(t, "ok  \t"), strings.HasPrefix(t, "FAIL\t"):
		default:
			kept = append(kept, l)
		}
	}
	return strings.TrimSpace(strings.Join(kept, "\n")), passed
}

// causeIn is a race the race detector reported, as its heading, its access and the frame that made it ("WARNING: DATA
// RACE: Write at 0x... by goroutine 9 in pkg.F.func1()"); else the first "panic:", "fatal error:" or "signal:" line of a
// failure's output, in that order; or "". **Only lines at column 0**, where go prints the runtime's: a test's indented log
// line holding the same words is not the cause (review round 1).
func causeIn(out string) string {
	lines := strings.Split(out, "\n")
	for i, l := range lines {
		if strings.TrimRight(l, "\r") != "WARNING: DATA RACE" {
			continue
		}
		words := "WARNING: DATA RACE"
		if i+1 < len(lines) && strings.TrimSpace(lines[i+1]) != "" {
			words += ": " + strings.TrimSuffix(strings.TrimSpace(lines[i+1]), ":")
			if i+2 < len(lines) && strings.HasPrefix(lines[i+2], "  ") {
				words += " in " + strings.TrimSpace(lines[i+2])
			}
		}
		return words
	}
	for _, prefix := range []string{"panic: ", "fatal error: ", "signal: "} {
		for _, l := range lines {
			if l = strings.TrimRight(l, "\r"); strings.HasPrefix(l, prefix) {
				return l
			}
		}
	}
	return ""
}

// firstLine is the first non-empty line of a failure's words.
func firstLine(s string) string {
	for _, l := range strings.Split(s, "\n") {
		if l = strings.TrimSpace(l); l != "" {
			return l
		}
	}
	return ""
}

// MaxFailureLine is the most bytes of a failure's line in a verdict's Failures: the first line of its words, which is
// its cause where the build module told one apart (issue 1901). Failed keeps the first failure's words whole.
const MaxFailureLine = 300

// levelFailure turns a level that did not pass into a verdict: Failed names what failed first with its words whole,
// and Failures every failed test of the level's results with its first line, at most delivery.MaxFailures and an entry
// saying how many more (issue 1921).
func levelFailure(t *target, level string, results ...Tested) *delivery.Verdict {
	v := delivery.Verdict{State: delivery.VerdictFail}
	var all []delivery.FailedFirst
	for _, res := range results {
		if len(res.Failed) == 0 {
			why := res.Why
			if why == "" {
				why = "the " + level + " level did not pass"
			}
			all = append(all, delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: why})
			continue
		}
		for _, f := range res.Failed {
			name := failedName(f)
			if t.module == buildrun.NodeBuildModule && f.Package == "" {
				name = res.Module
			}
			all = append(all, delivery.FailedFirst{Test: level + ": " + name, Words: failureWordsAfter(f, hidWords(t.module, res.By, f))})
		}
	}
	if len(all) == 0 { // no result: a level said failed with nothing to name
		all = append(all, delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: "the " + level + " level did not pass"})
	}
	v.Failed = all[0]
	v.Failures = failureLines(all)
	return &v
}

// failureLines is every failure with the first line of its words, cut to MaxFailureLine, at most delivery.MaxFailures
// and then one entry saying how many more failed.
func failureLines(all []delivery.FailedFirst) []delivery.FailedFirst {
	n := min(len(all), delivery.MaxFailures)
	out := make([]delivery.FailedFirst, 0, n+1)
	for _, f := range all[:n] {
		out = append(out, delivery.FailedFirst{Test: f.Test, Machine: f.Machine, Words: clipLine(firstLine(f.Words), MaxFailureLine)})
	}
	if more := len(all) - n; more > 0 {
		out = append(out, delivery.FailedFirst{Test: fmt.Sprintf("and %d more", more)})
	}
	return out
}

// clipLine cuts a line to at most n bytes on a whole character, marking the cut.
func clipLine(s string, n int) string {
	if len(s) <= n {
		return s
	}
	s = strings.ToValidUTF8(s[:n-3], "")
	return s + "..."
}