867 lines
35 KiB
Go
867 lines
35 KiB
Go
package check
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"path"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
|
)
|
|
|
|
// goModule is the build module the Go repositories are judged by.
|
|
const goModule = buildrun.GoBuildModule
|
|
|
|
// RunnerWorkspace is the runner the Go build module's digest and test steps are asked with when the merge changes the build
|
|
// module itself: the installed module compiles the candidate's program from the merged tree and runs the step as it, shut
|
|
// off (ADR 0535 §1).
|
|
const RunnerWorkspace = "workspace"
|
|
|
|
// buildGoDir is the Go build module's directory in the catalogue: a merge changing a path under it is judged by its
|
|
// candidate (ADR 0535 §1). mesh-build-node's own directory is judged by the installed module until its part of issue 1898.
|
|
const buildGoDir = "modules/mesh-build-go/"
|
|
|
|
// changesBuildGo says whether a merge of the repository with these changed paths changes the Go build module.
|
|
func changesBuildGo(repository string, changed []string) bool {
|
|
if repository != catalogueRepository {
|
|
return false
|
|
}
|
|
for _, p := range changed {
|
|
if strings.HasPrefix(p, buildGoDir) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// The words of the line on the program that judged a change to the Go build module (ADR 0535 §2, §5).
|
|
const (
|
|
judgedByOldInstalled = "judged by the installed mesh-build-go: it does not run a candidate yet"
|
|
judgedByInstalled = "judged by the installed mesh-build-go"
|
|
judgedByCandidate = "judged by the candidate's mesh-build-go at "
|
|
)
|
|
|
|
// verdictPtr is a verdict of failure a step of the check ends with.
|
|
type verdictPtr = *delivery.Verdict
|
|
|
|
func failed(machine, words string) verdictPtr { return fail(machine, words) }
|
|
|
|
// Asks puts a check's ask on its build seat's stream and waits for its step (buildrun.Asker; ADR 0525 §1, §6): the
|
|
// checker chooses no node, starts no step and holds none; whichever build node pulls the ask runs the check.
|
|
type Asks interface {
|
|
Put(ctx context.Context, a delivery.StepAsk) error
|
|
Wait(ctx context.Context, a delivery.StepAsk) (delivery.Step, error)
|
|
}
|
|
|
|
// ToolchainEntries are the nodes' toolchain entries of a language (buildrun.ToolchainSet).
|
|
type ToolchainEntries interface {
|
|
Entries(language string) []delivery.NodeToolchain
|
|
}
|
|
|
|
// BuildModules says which build modules judge a repository: the Go one always, the Node one when the repository holds a
|
|
// TypeScript module.
|
|
type BuildModules interface {
|
|
BuildModules(repository string) []string
|
|
}
|
|
|
|
// askLanguage is the build seat language of each build module the checker asks.
|
|
var askLanguage = map[string]string{buildrun.GoBuildModule: delivery.AskGo, buildrun.NodeBuildModule: delivery.AskNode}
|
|
|
|
// Examiner judges a case through the build seats (ADR 0484 clauses 3, 4, 10, 13; ADR 0525 §1, §3): for each pull request
|
|
// and build module it puts one check ask whose job fetches the head merged onto its trunk head, then runs the unit level
|
|
// and the integration level, taking the kept verdicts of unchanged packages, running the change's own tests first and
|
|
// stopping at the first failure; it judges the step's answer as it judged the tools' answers.
|
|
type Examiner struct {
|
|
Asks Asks
|
|
// DeclineBound is how long a placed ask may go on being declined, from its first decline, before the check fails
|
|
// saying no node with its need took it (issue 890, ADR 0552); DefaultDeclineBound when zero.
|
|
DeclineBound time.Duration
|
|
// Toolchains are the nodes' toolchains: a kept verdict names the toolchain the nodes agree on.
|
|
Toolchains ToolchainEntries
|
|
Kept Kept
|
|
// Previews is the previews bucket; nil writes none.
|
|
Previews KV
|
|
Modules BuildModules
|
|
// Runs says which commit the mesh runs of a module (the indexer's modules); nil knows none.
|
|
Runs interface{ CommitOf(module string) string }
|
|
// Forge reads the go.mod files of the Go members and says whether a mesh-sdk commit is on its trunk (ADR 0484 clause 9);
|
|
// nil checks no pin.
|
|
Forge Forge
|
|
// Builds says how the build of a fingerprint the candidate carries stands (the builder's builds bucket); nil says nothing
|
|
// of the builds.
|
|
Builds BuildStates
|
|
// Announce says a preview was written under the key (EventPreviewWritten); nil says nothing.
|
|
Announce Announcer
|
|
// Packages names, of a repository, the paths (repository-relative) its TypeScript package artifacts are built from, from
|
|
// the indexer's modules (delivery.Module.ArtifactPaths, lead decision L8); nil serves no group package.
|
|
Packages interface {
|
|
PackagePaths(repository string) []string
|
|
}
|
|
// Trunks are the trunk heads of every repository, for the siblings a repository's tests read.
|
|
Trunks interface{ Heads() map[string]string }
|
|
// Compositions answers the assigner's composition check of a check by the check's key (delivery.CompositionKey),
|
|
// waiting until the assigner wrote a verdict that does not wait, or ctx ends (ADR 0528 clause 5, issue 1297); nil
|
|
// leaves the composition not run.
|
|
Compositions CompositionReader
|
|
// Nodes names every node with a record, read from each `<node>.node` of the node-engine's have (novox/hq ADR 0567
|
|
// §12): every check ask carries them (CheckJob.NodeNames) for the names check, so no repository lists them. A check
|
|
// whose names cannot be read, or that reads none, is not asked and fails saying so; nil hands no names.
|
|
Nodes interface {
|
|
Names(ctx context.Context) ([]string, error)
|
|
}
|
|
Now func() time.Time
|
|
Logf func(format string, args ...any)
|
|
}
|
|
|
|
// CompositionReader answers the assigner's composition of a check by the check's key, waiting until the assigner wrote
|
|
// a verdict that does not wait, or ctx ends.
|
|
type CompositionReader interface {
|
|
Await(ctx context.Context, key string) (delivery.CompositionVerdict, error)
|
|
}
|
|
|
|
func (j *Examiner) now() time.Time {
|
|
if j.Now != nil {
|
|
return j.Now()
|
|
}
|
|
return time.Now()
|
|
}
|
|
|
|
func (j *Examiner) logf(format string, args ...any) {
|
|
if j.Logf != nil {
|
|
j.Logf(format, args...)
|
|
}
|
|
}
|
|
|
|
// besideLines are the siblings as the digest step echoes them: `<owner/repo> <commit>`, sorted.
|
|
func besideLines(refs []BesideRef) []string {
|
|
out := make([]string, 0, len(refs))
|
|
for _, r := range refs {
|
|
out = append(out, r.Repository+" "+r.Commit)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// target is a pull request on one build module: the node that pulled its ask, the workspace its fetch made and what it
|
|
// said.
|
|
type target struct {
|
|
pull Pull
|
|
module string
|
|
node string
|
|
toolchain string
|
|
fetched Fetched
|
|
// fetchArgs are the arguments of the fetch its check ask runs.
|
|
fetchArgs map[string]any
|
|
// priority are the Go packages (import paths) or the Node module directories whose test files the merge changes.
|
|
changed []string
|
|
// beside are the lines `<owner/repo> <commit>` of the siblings the workspace was given, sorted: its tests may read them,
|
|
// so a verdict is kept or taken only under a digest that says it folded exactly these in (lead decision L5).
|
|
beside []string
|
|
// runner is RunnerWorkspace when the target's ask runs the candidate's own build module (ADR 0535 §1), empty otherwise.
|
|
runner string
|
|
// judged is what the line says of the program that judged a change to the build module; empty for any other change.
|
|
judged string
|
|
}
|
|
|
|
// judgedBy takes what a step under runner workspace answered of the program that ran it (judged_by): the candidate, or the
|
|
// installed module (a module that does not take the runner says nothing, and is asked without it from now on: ADR 0535 §2).
|
|
// It says whether the candidate ran.
|
|
func (t *target) judgedBy(answer string) bool {
|
|
switch {
|
|
case strings.HasPrefix(answer, "candidate "):
|
|
t.judged = judgedByCandidate + short12(strings.TrimPrefix(answer, "candidate "))
|
|
return true
|
|
case answer == "":
|
|
t.judged, t.runner = judgedByOldInstalled, ""
|
|
default:
|
|
t.judged, t.runner = judgedByInstalled, ""
|
|
}
|
|
return false
|
|
}
|
|
|
|
// stepArgs are a step's arguments with the target's runner.
|
|
func (t *target) stepArgs(args map[string]any) map[string]any {
|
|
if t.runner != "" {
|
|
args["runner"] = t.runner
|
|
}
|
|
return args
|
|
}
|
|
|
|
// acc collects a level's counts across targets.
|
|
type acc struct {
|
|
run, kept, skipped int
|
|
elapsed, setup float64
|
|
lint float64
|
|
over bool
|
|
slowest []TestTime
|
|
}
|
|
|
|
// state is one judging.
|
|
type state struct {
|
|
j *Examiner
|
|
c *Case
|
|
targets []*target
|
|
levels map[string]*acc
|
|
nodes map[string]bool
|
|
clone time.Duration
|
|
slow []string
|
|
// beside are the siblings read at their trunk's head, by repository (ADR 0548 §6).
|
|
beside map[string]string
|
|
}
|
|
|
|
// Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot
|
|
// be judged at all.
|
|
func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) {
|
|
st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{},
|
|
beside: map[string]string{}}
|
|
if v := st.judgeAll(ctx); v != nil {
|
|
return st.finish(ctx, *v), nil
|
|
}
|
|
return st.finish(ctx, delivery.Verdict{State: delivery.VerdictPass}), nil
|
|
}
|
|
|
|
// finish is the outcome of the verdict with the state of the builds the candidate's modules carry, and, for a check
|
|
// whose tests passed, the assigner's composition of it.
|
|
func (st *state) finish(ctx context.Context, v delivery.Verdict) Outcome {
|
|
out := st.outcome(v)
|
|
if out.Verdict.State == delivery.VerdictPass {
|
|
st.composed(ctx, &out.Verdict)
|
|
}
|
|
out.Builds = st.j.buildNotes(ctx, st.c)
|
|
return out
|
|
}
|
|
|
|
// composed reads the assigner's composition of the check into its verdict (ADR 0528 clause 5): a failed composition
|
|
// fails the check, naming the machine it failed at. A try is not composed (it has no candidate); a composition that
|
|
// cannot be had leaves it not run, said; with no assigner running it is not judged, said at once.
|
|
func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
|
|
if st.c.Composition != nil && st.c.Try == nil {
|
|
// A carry whose candidate's fingerprints did not change keeps the composition it passed (ADR 0548 §5).
|
|
kept := *st.c.Composition
|
|
v.Composition = &kept
|
|
st.c.note("the composition is kept: no fingerprint of a module it composed changed")
|
|
return
|
|
}
|
|
if st.j.Compositions == nil || st.c.Try != nil {
|
|
return
|
|
}
|
|
comp, err := st.j.Compositions.Await(ctx, st.c.Key)
|
|
switch {
|
|
case errors.Is(err, ErrAssignerNotRunning):
|
|
v.Composition = &delivery.Composition{State: delivery.CompositionNotRun, Words: delivery.AssignerNotRunning}
|
|
return
|
|
case err != nil:
|
|
st.j.logf("%s: the composition could not be read, so it is said as not run: %v", st.c.Key, err)
|
|
return
|
|
}
|
|
withComposition(v, comp)
|
|
if v.Composition != nil && v.Composition.State != delivery.CompositionNotRun {
|
|
v.Composition.FPSet = st.c.FPSet // a carry keeps it while the fingerprints stay the same (ADR 0548 §5)
|
|
}
|
|
}
|
|
|
|
// outcome fills the verdict's levels, times, machines and composition from what ran.
|
|
func (st *state) outcome(v delivery.Verdict) Outcome {
|
|
if v.State == delivery.VerdictPass || len(v.Levels) == 0 {
|
|
v.Levels = map[string]delivery.LevelResult{}
|
|
for level, a := range st.levels {
|
|
v.Levels[level] = delivery.LevelResult{Run: a.run, Kept: a.kept, Skipped: a.skipped}
|
|
}
|
|
}
|
|
ms := func(s float64) int64 { return int64(s * 1000) }
|
|
u, i := st.levels[delivery.LevelUnit], st.levels[delivery.LevelIntegration]
|
|
v.Took = delivery.Took{Clone: st.clone.Milliseconds(), Unit: ms(u.elapsed), Lint: ms(u.lint), IntegrationSetup: ms(i.setup), IntegrationRun: ms(i.elapsed)}
|
|
for node := range st.nodes {
|
|
v.Machines = append(v.Machines, node)
|
|
}
|
|
sort.Strings(v.Machines)
|
|
for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} {
|
|
if st.levels[level].over {
|
|
v.OverBound = append(v.OverBound, level)
|
|
}
|
|
}
|
|
// The composition is the assigner's check (issue 1297): until it delivers a verdict says it was not run (ADR 0484 clause 6).
|
|
v.Composition = &delivery.Composition{State: delivery.CompositionNotRun}
|
|
var slow []string
|
|
for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} {
|
|
a := st.levels[level]
|
|
if !a.over {
|
|
continue
|
|
}
|
|
sort.SliceStable(a.slowest, func(x, y int) bool { return a.slowest[x].Elapsed > a.slowest[y].Elapsed })
|
|
for n, s := range a.slowest {
|
|
if n == 3 {
|
|
break
|
|
}
|
|
slow = append(slow, fmt.Sprintf("%s %s %.1fs", level, path.Base(s.Package)+"."+s.Test, s.Elapsed))
|
|
}
|
|
}
|
|
out := Outcome{Verdict: v, Slowest: slow}
|
|
if len(st.beside) > 0 {
|
|
out.Beside = st.beside
|
|
}
|
|
for _, t := range st.targets {
|
|
if t.judged != "" {
|
|
if out.Judged == nil {
|
|
out.Judged = map[string]string{}
|
|
}
|
|
out.Judged[pullKey(Pull{Repository: t.pull.Repository, Number: t.pull.Number})] = t.judged
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// fail is a verdict of failure at a machine.
|
|
func fail(machine, words string) *delivery.Verdict {
|
|
v := failure(machine, words)
|
|
return &v
|
|
}
|
|
|
|
// levels are the levels a case's asks run: both, or the one a try names.
|
|
func (st *state) levelsOf() []string {
|
|
if st.c.Try != nil && st.c.Try.Level != "" {
|
|
return []string{st.c.Try.Level} // a try of one level runs that level alone
|
|
}
|
|
return []string{delivery.LevelUnit, delivery.LevelIntegration}
|
|
}
|
|
|
|
// judgeAll puts one check ask per pull request and build module the repository needs, the mesh-sdk member first, and
|
|
// judges each answer. A head that does not merge, a pin that is not allowed or a level that fails ends the check at once
|
|
// (ADR 0484 clause 13).
|
|
func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
|
|
// The mesh-sdk member first: whether its merge changes the TypeScript package's paths decides whether the other members'
|
|
// Node fetches take the group's package (ADR 0500 §5).
|
|
pulls := slices.Clone(st.c.Pulls)
|
|
slices.SortStableFunc(pulls, func(a, b Pull) int {
|
|
if (a.Repository == SDKRepository) != (b.Repository == SDKRepository) {
|
|
if a.Repository == SDKRepository {
|
|
return -1
|
|
}
|
|
return 1
|
|
}
|
|
return 0
|
|
})
|
|
var groupPackage map[string]any
|
|
for _, p := range pulls {
|
|
var merged []Fetched
|
|
for _, bm := range st.j.Modules.BuildModules(p.Repository) {
|
|
t := &target{pull: p, module: bm}
|
|
if tc, none := delivery.AgreedToolchain(st.j.Toolchains.Entries(buildrun.LanguageOf[bm])); none == "" {
|
|
t.toolchain = tc.Version
|
|
}
|
|
args := map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number), "base": p.Trunk}
|
|
if bm == buildrun.NodeBuildModule && groupPackage != nil && p.Repository != SDKRepository {
|
|
args["package"] = groupPackage
|
|
}
|
|
if bm == buildrun.GoBuildModule {
|
|
beside, err := st.j.besideFor(ctx, st.c, p)
|
|
if err != nil {
|
|
return fail(forgeMachine(err, bm), err.Error()+unheardWords(err))
|
|
}
|
|
if len(beside) > 0 {
|
|
args["beside"] = beside
|
|
t.beside = besideLines(beside)
|
|
besideAtTrunk(st.c, beside, st.beside)
|
|
}
|
|
if p.Repository == catalogueRepository {
|
|
t.runner = RunnerWorkspace // used by the build module when the merge changes it (ADR 0535 §1)
|
|
}
|
|
}
|
|
t.fetchArgs = args
|
|
st.targets = append(st.targets, t)
|
|
if v := st.ask(ctx, t, &merged); v != nil {
|
|
return v
|
|
}
|
|
}
|
|
if st.c.Try == nil { // a try owes the indexer no preview: no pull request holds its head
|
|
st.j.writePreview(ctx, p, merged)
|
|
}
|
|
if p.Repository == SDKRepository && len(st.c.Pulls) > 1 && st.j.changesPackage(p, merged) {
|
|
groupPackage = map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number)}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// DefaultDeclineBound is an Examiner's DeclineBound when it names none: longer than a check runs, so a node with the
|
|
// need that is busy is waited for, and short enough that a mesh where no node has it says so.
|
|
const DefaultDeclineBound = 30 * time.Minute
|
|
|
|
// ask puts the target's check ask, waits for the step of the node that pulled it and judges its answer: the fetch, the
|
|
// pins of its go.mod files, then each level's digest, kept verdicts and tests. When the answer leaves packages unmet,
|
|
// their tests needing what that node lacks (issue 890), the passes are kept and the levels from the first such one are
|
|
// asked again of a node that has it (placed).
|
|
func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *delivery.Verdict {
|
|
return st.askNeeding(ctx, t, merged, st.levelsOf(), nil, nil)
|
|
}
|
|
|
|
// askNeeding is ask with the levels to run, what the node must have and the packages it tests: none for the first ask
|
|
// of a target; for the placed one the unmet packages' needs and those packages alone, so no package the first ask judged
|
|
// is run or counted again (ADR 0552 §4).
|
|
func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, levels, needs, packages []string) *delivery.Verdict {
|
|
p := t.pull
|
|
placed := len(needs) > 0
|
|
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: levels, Kept: true, Packages: packages}
|
|
if st.j.Nodes != nil {
|
|
names, err := st.j.Nodes.Names(ctx)
|
|
if err == nil && len(names) == 0 {
|
|
err = fmt.Errorf("no node record is on the bus")
|
|
}
|
|
if err != nil {
|
|
return fail(t.module, fmt.Sprintf("the check of %s#%d is not asked: the nodes' names the names check needs cannot be read: %v", p.Repository, p.Number, err))
|
|
}
|
|
job.NodeNames = names
|
|
}
|
|
if t.runner != "" {
|
|
job.Runner, job.RunnerWhen = t.runner, buildGoDir
|
|
}
|
|
if st.c.Try != nil && st.c.Try.Run != "" {
|
|
job.Run, job.Kept = st.c.Try.Run, false // a try of some tests runs them alone: a partial run proves no package
|
|
}
|
|
kind := delivery.AskCheck
|
|
if st.c.Kind == KindTrunk {
|
|
kind = delivery.AskRecheck // a re-judge after a trunk move goes after every new head's check (ADR 0536 §4)
|
|
}
|
|
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: askLanguage[t.module], Asker: delivery.CheckerModule,
|
|
At: st.j.now().UTC(), Check: job, Needs: needs}
|
|
var step delivery.Step
|
|
var firstDecline time.Time
|
|
for {
|
|
what := fmt.Sprintf("the %s ask %s of %s#%d", kind, a.ID, p.Repository, p.Number)
|
|
if placed {
|
|
what += " for a node with " + strings.Join(needs, ", ")
|
|
}
|
|
st.c.Progress.Set(what, "")
|
|
st.c.Asked(a.ID)
|
|
if err := st.j.Asks.Put(ctx, a); err != nil {
|
|
return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err))
|
|
}
|
|
waiting := ctx
|
|
if watch := st.c.Watch; watch != nil {
|
|
id := a.ID
|
|
watch(id, nil) // put: a try names it at once, not only when it ends (issue 1890)
|
|
waiting = buildrun.WithStepSeen(ctx, func(s delivery.Step) { watch(id, &s) })
|
|
}
|
|
var err error
|
|
step, err = st.j.Asks.Wait(waiting, a)
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
return fail(t.module, ctx.Err().Error())
|
|
}
|
|
return fail(t.module, fmt.Sprintf("the check of %s#%d: %v", p.Repository, p.Number, err))
|
|
}
|
|
d, declined := delivery.DeclinedOf(step)
|
|
if !declined {
|
|
break
|
|
}
|
|
// A node that lacks the need dropped the ask (issue 890): it is put again, for a node that has it to take; the
|
|
// node that declined looks at the stream again only after its next pull's wait.
|
|
now := st.j.now()
|
|
if firstDecline.IsZero() {
|
|
firstDecline = now
|
|
}
|
|
bound := st.j.DeclineBound
|
|
if bound <= 0 {
|
|
bound = DefaultDeclineBound
|
|
}
|
|
if now.Sub(firstDecline) > bound {
|
|
return fail(step.Node, fmt.Sprintf("the check of %s#%d needs a node with %s, and none took it in %s: every node that pulled it lacks it (last %s)",
|
|
p.Repository, p.Number, strings.Join(needs, ", "), bound, step.Node))
|
|
}
|
|
st.j.logf("%s#%d: the ask %s was declined on %s, which lacks %s; asked again", p.Repository, p.Number, a.ID, step.Node, strings.Join(d.Lacks, ", "))
|
|
a.ID, a.At = buildstep.NewID(), now.UTC()
|
|
}
|
|
// The fetch and the digests are what of the ask's time its levels' running did not take (Took.Clone).
|
|
took := time.Duration(0)
|
|
if !step.Started.IsZero() && step.Ended.After(step.Started) {
|
|
took = step.Ended.Sub(step.Started)
|
|
}
|
|
t.node = step.Node
|
|
st.nodes[step.Node] = true
|
|
st.c.Progress.Set(fmt.Sprintf("judge the check of %s#%d", p.Repository, p.Number), step.Node)
|
|
if step.State != delivery.StepPassed {
|
|
return fail(step.Node, fmt.Sprintf("the check of %s#%d %s: %s%s", p.Repository, p.Number, step.State, step.Reason, pinHint(st.c, step.Reason)))
|
|
}
|
|
var answer delivery.CheckAnswer
|
|
if err := json.Unmarshal(step.Result, &answer); err != nil {
|
|
return fail(step.Node, fmt.Sprintf("the check of %s#%d answered what is not a check: %s", p.Repository, p.Number, step.Result))
|
|
}
|
|
if err := json.Unmarshal(answer.Fetch, &t.fetched); err != nil || t.fetched.Workspace == "" {
|
|
return fail(step.Node, fmt.Sprintf("the fetch of %s#%d answered no workspace: %s", p.Repository, p.Number, answer.Fetch))
|
|
}
|
|
if len(t.fetched.Conflicts) > 0 {
|
|
if st.c.Try == nil {
|
|
st.j.writePreview(ctx, p, append(*merged, t.fetched))
|
|
}
|
|
return fail(step.Node, fmt.Sprintf("%s#%d %s%s", p.Repository, p.Number, ConflictWords, strings.Join(t.fetched.Conflicts, ", ")))
|
|
}
|
|
t.changed = t.fetched.Changed
|
|
if !changesBuildGo(p.Repository, t.changed) {
|
|
t.runner = "" // the build module used no runner: the merge does not change it
|
|
}
|
|
if !placed { // the placed ask fetched the same merge: its preview and pins were read from the first
|
|
*merged = append(*merged, t.fetched)
|
|
if v := st.checkPinsOf(ctx, t); v != nil {
|
|
return v
|
|
}
|
|
if v := st.checkBinariesOf(t); v != nil {
|
|
return v
|
|
}
|
|
}
|
|
ran := 0.0
|
|
for _, la := range answer.Levels {
|
|
if placed {
|
|
if len(la.Unmet) > 0 {
|
|
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
|
|
return fail(step.Node, fmt.Sprintf("the %s level of %s#%d left %s untested on %s, which was asked for a node with %s",
|
|
la.Level, p.Repository, p.Number, unmetNames(la.Unmet), step.Node, strings.Join(needs, ", ")))
|
|
}
|
|
la.Kept = nil // it names its packages and takes no other's kept verdict; this is only a guard against an older module
|
|
}
|
|
v, elapsed := st.level(ctx, t, la)
|
|
ran += elapsed
|
|
if v != nil {
|
|
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
|
|
return v
|
|
}
|
|
}
|
|
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
|
|
if unmet := delivery.UnmetNeeds(answer.Levels); len(unmet) > 0 && !placed {
|
|
// Every level passed, its passes kept, and some packages were left to a node with what their tests need: the levels
|
|
// from the first that left one are asked of such a node, which takes the kept passes and tests those (issue 890).
|
|
var again, packages []string
|
|
seen := map[string]bool{}
|
|
for _, la := range answer.Levels {
|
|
if len(again) > 0 || len(la.Unmet) > 0 {
|
|
again = append(again, la.Level)
|
|
}
|
|
for _, u := range la.Unmet {
|
|
if !seen[u.Package] {
|
|
seen[u.Package] = true
|
|
packages = append(packages, u.Package)
|
|
}
|
|
}
|
|
}
|
|
st.j.logf("%s#%d: %s on %s left untested; asked of a node with %s", p.Repository, p.Number, unmetOf(answer.Levels), step.Node, strings.Join(unmet, ", "))
|
|
return st.askNeeding(ctx, t, merged, again, unmet, packages)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// unmetNames names the unmet packages and their needs.
|
|
func unmetNames(unmet []delivery.PackageNeeds) string {
|
|
var out []string
|
|
for _, u := range unmet {
|
|
out = append(out, u.Package+" (needs "+strings.Join(u.Needs, ", ")+")")
|
|
}
|
|
return strings.Join(out, ", ")
|
|
}
|
|
|
|
// unmetOf names the unmet packages of every level.
|
|
func unmetOf(levels []delivery.LevelAnswer) string {
|
|
var out []string
|
|
for _, la := range levels {
|
|
if len(la.Unmet) > 0 {
|
|
out = append(out, "at "+la.Level+" "+unmetNames(la.Unmet))
|
|
}
|
|
}
|
|
return strings.Join(out, "; ")
|
|
}
|
|
|
|
// level judges one level of a check ask's answer as the tools' answers were judged: the digest keys the kept verdicts
|
|
// of the packages that passed, the kept ones taken are counted, and every test result is read and counted, also after
|
|
// one that did not pass, so the verdict names every failed test of the level (issue 1921). It answers the level's
|
|
// running time (its tests' and setup's).
|
|
func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer) (*delivery.Verdict, float64) {
|
|
level := la.Level
|
|
a := st.levels[level]
|
|
if a == nil {
|
|
return fail(t.node, fmt.Sprintf("the check of %s#%d answered the level %q, which is no level", t.pull.Repository, t.pull.Number, level)), 0
|
|
}
|
|
var digests map[string]string
|
|
if t.module == buildrun.GoBuildModule && la.Digest != nil {
|
|
// The packages come as a list, never keyed by a package path (issue 1930); an answer stored before reads too.
|
|
digests = la.Digest.Map()
|
|
if t.runner != "" {
|
|
t.judgedBy(la.Digest.JudgedBy) // the installed module judged it: the runner is not used again
|
|
}
|
|
}
|
|
if t.runner != "" && la.Judged != "" {
|
|
t.judgedBy(la.Judged) // the candidate judged the level: nothing is kept (ADR 0535 §4)
|
|
}
|
|
for _, k := range la.Kept {
|
|
a.kept += k.Tests
|
|
}
|
|
elapsed := 0.0
|
|
var failedResults []Tested
|
|
for _, raw := range la.Tests {
|
|
var res Tested
|
|
if err := json.Unmarshal(raw, &res); err != nil {
|
|
return fail(t.node, fmt.Sprintf("the %s level of %s#%d answered what is not a result: %v", level, t.pull.Repository, t.pull.Number, err)), elapsed
|
|
}
|
|
if t.runner != "" {
|
|
t.judgedBy(res.JudgedBy)
|
|
}
|
|
elapsed += res.Elapsed + res.Setup + res.Lint
|
|
if t.module == buildrun.NodeBuildModule {
|
|
if !res.NoTests {
|
|
a.run++ // a Node package is counted whole: its build module counts no tests (ADR 0484 clause 10)
|
|
}
|
|
} else {
|
|
a.run += res.Tests
|
|
a.skipped += res.Skipped
|
|
a.slowest = append(a.slowest, res.Slowest...)
|
|
}
|
|
a.elapsed += res.Elapsed
|
|
a.setup += res.Setup
|
|
a.lint += res.Lint
|
|
a.over = a.over || res.OverBound
|
|
if len(digests) > 0 {
|
|
if _, err := st.j.Kept.KeepPasses(ctx, t.pull.Repository, level, t.toolchain, res, digests, st.j.now()); err != nil {
|
|
st.j.logf("%s#%d: a passing package could not be kept: %v", t.pull.Repository, t.pull.Number, err)
|
|
}
|
|
}
|
|
if !res.Passed {
|
|
failedResults = append(failedResults, res)
|
|
}
|
|
}
|
|
if len(failedResults) > 0 {
|
|
return levelFailure(t, level, failedResults...), elapsed
|
|
}
|
|
return nil, elapsed
|
|
}
|
|
|
|
// changesPackage says whether the merge of a pull request changes a file inside the named paths of one of its repository's
|
|
// TypeScript packages: a change of the manifest alone, or of files outside those paths, does not (ADR 0500 §5).
|
|
func (j *Examiner) changesPackage(p Pull, merged []Fetched) bool {
|
|
if j.Packages == nil {
|
|
return false
|
|
}
|
|
paths := j.Packages.PackagePaths(p.Repository)
|
|
for _, f := range merged {
|
|
for _, changed := range f.Changed {
|
|
for _, named := range paths {
|
|
named = strings.TrimSuffix(strings.TrimPrefix(named, "./"), "/")
|
|
if changed == named || strings.HasPrefix(changed, named+"/") {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// failedName names a failure: its package and test.
|
|
func failedName(f Failure) string {
|
|
if f.Test == "" {
|
|
return f.Package
|
|
}
|
|
return f.Package + "." + f.Test
|
|
}
|
|
|
|
// MaxWords is the most a failed test's words in a verdict hold (issue 1901). **The words keep the failure's last lines**,
|
|
// where a test says why it failed: the first line is often something it logged before (the group r4g verdict kept a
|
|
// "judged against" line and lost the t.Fatalf after it), and the 120 characters kept before cut a docker command's echo
|
|
// before docker's own error. Four kilobytes holds a long message and is small beside a verdict's other parts.
|
|
const MaxWords = 4096
|
|
|
|
// failureWords is a failure's words: its output's last lines up to MaxWords, with the line the build module told apart as
|
|
// the test's error first when the output does not start with it, so the first line of the words is the cause and the
|
|
// status line, which is cut short, names it. **A failure's words are never "PASS" or "ok"** (issue 1901, its third
|
|
// meeting: a package that failed after its tests passed was worded "PASS"): go test's summary lines are dropped, and with
|
|
// no cause told apart (an older build module) the cause is the output's first "panic:", "fatal error:" or "signal:"
|
|
// line, read before the output is cut, or the words say the tests passed and the program failed after them.
|
|
func failureWords(f Failure) string {
|
|
return failureWordsAfter(f, "")
|
|
}
|
|
|
|
// failureWordsAfter is failureWords with the line loud put first, within the same MaxWords: what the checker itself says of
|
|
// the failure (that the build module hid it), which the status line then names.
|
|
func failureWordsAfter(f Failure, loud string) string {
|
|
out, passed := dropSummary(strings.TrimSpace(f.Output))
|
|
cause := strings.TrimSpace(f.Cause)
|
|
if cause == "" {
|
|
cause = causeIn(out)
|
|
}
|
|
switch {
|
|
case cause == "" && out == "" && passed:
|
|
cause = "the tests passed, then the test program exited with a failure: an os.Exit or a panic in TestMain or after the tests"
|
|
case cause == "" && out == "":
|
|
// The build module on main sends a package that stopped (a timeout, an exit, a kill) as its "FAIL" line alone.
|
|
cause = "the package failed with no test to name and no panic, signal or build error in its output (an older build module sent no cause)"
|
|
}
|
|
if len(cause) > MaxCause {
|
|
cause = strings.ToValidUTF8(cause[:MaxCause], "") + "..."
|
|
}
|
|
// The words keep MaxWords exactly: the loud line, the cause and their newlines, the marker and the kept lines.
|
|
budget := MaxWords
|
|
if cause != "" {
|
|
budget -= len(cause) + 1
|
|
}
|
|
if loud != "" {
|
|
budget -= len(loud) + 1
|
|
}
|
|
if len(out) > budget {
|
|
const marker = "... "
|
|
tail := out[len(out)-(budget-len(marker)):]
|
|
// Start on a whole line, and a whole character.
|
|
if i := strings.IndexByte(tail, '\n'); i >= 0 {
|
|
tail = strings.TrimLeft(tail[i:], "\n")
|
|
}
|
|
out = marker + strings.ToValidUTF8(tail, "")
|
|
}
|
|
if cause != "" && !strings.HasPrefix(firstLine(out), cause) {
|
|
out = strings.TrimRight(cause+"\n"+out, "\n")
|
|
}
|
|
if loud != "" {
|
|
out = strings.TrimRight(loud+"\n"+out, "\n")
|
|
}
|
|
return out
|
|
}
|
|
|
|
// hidWords is the checker's loud line for a failure the build module sent hiding its cause (issue 1901, its thorough
|
|
// fix): with no cause, or a passing line as its cause, under the failure contract (buildstep.Hidden). It names the build
|
|
// module and the version it says it is (Tested.By, "<module> with mesh-sdk <version>"); one from before the contract
|
|
// says nothing of itself. "" when the failure keeps the contract. The check fails either way: only its words change.
|
|
func hidWords(module, by string, f Failure) string {
|
|
why := buildstep.Hidden(f)
|
|
if why == "" {
|
|
return ""
|
|
}
|
|
version := "a version before the failure contract"
|
|
if module == "" {
|
|
module = "an unnamed build module"
|
|
}
|
|
if m, v, ok := strings.Cut(by, " with "); ok && m != "" {
|
|
module, version = m, v
|
|
}
|
|
return fmt.Sprintf("the build module hid the failure (%s at %s): %s", module, version, why)
|
|
}
|
|
|
|
// MaxCause is the most a cause in a failure's words holds: one long line (the 1901 review: a cause of a kilobyte could
|
|
// push the words past MaxWords).
|
|
const MaxCause = 1024
|
|
|
|
// dropSummary drops go test's summary lines from a failure's output ("PASS", "FAIL", "ok <pkg>", "FAIL <pkg>"), and says
|
|
// whether a "PASS" was among them: the package's tests passed before it failed.
|
|
func dropSummary(out string) (string, bool) {
|
|
passed := false
|
|
lines := strings.Split(out, "\n")
|
|
kept := lines[:0]
|
|
for _, l := range lines {
|
|
// go's exact forms only (review round 1): "PASS" or "FAIL" alone, "ok \t<pkg>\t<time>", "FAIL\t<pkg>\t<time>".
|
|
switch t := strings.TrimRight(l, "\r"); {
|
|
case t == "PASS":
|
|
passed = true
|
|
case t == "FAIL", strings.HasPrefix(t, "ok \t"), strings.HasPrefix(t, "FAIL\t"):
|
|
default:
|
|
kept = append(kept, l)
|
|
}
|
|
}
|
|
return strings.TrimSpace(strings.Join(kept, "\n")), passed
|
|
}
|
|
|
|
// causeIn is a race the race detector reported, as its heading, its access and the frame that made it ("WARNING: DATA
|
|
// RACE: Write at 0x... by goroutine 9 in pkg.F.func1()"); else the first "panic:", "fatal error:" or "signal:" line of a
|
|
// failure's output, in that order; or "". **Only lines at column 0**, where go prints the runtime's: a test's indented log
|
|
// line holding the same words is not the cause (review round 1).
|
|
func causeIn(out string) string {
|
|
lines := strings.Split(out, "\n")
|
|
for i, l := range lines {
|
|
if strings.TrimRight(l, "\r") != "WARNING: DATA RACE" {
|
|
continue
|
|
}
|
|
words := "WARNING: DATA RACE"
|
|
if i+1 < len(lines) && strings.TrimSpace(lines[i+1]) != "" {
|
|
words += ": " + strings.TrimSuffix(strings.TrimSpace(lines[i+1]), ":")
|
|
if i+2 < len(lines) && strings.HasPrefix(lines[i+2], " ") {
|
|
words += " in " + strings.TrimSpace(lines[i+2])
|
|
}
|
|
}
|
|
return words
|
|
}
|
|
for _, prefix := range []string{"panic: ", "fatal error: ", "signal: "} {
|
|
for _, l := range lines {
|
|
if l = strings.TrimRight(l, "\r"); strings.HasPrefix(l, prefix) {
|
|
return l
|
|
}
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// firstLine is the first non-empty line of a failure's words.
|
|
func firstLine(s string) string {
|
|
for _, l := range strings.Split(s, "\n") {
|
|
if l = strings.TrimSpace(l); l != "" {
|
|
return l
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// MaxFailureLine is the most bytes of a failure's line in a verdict's Failures: the first line of its words, which is
|
|
// its cause where the build module told one apart (issue 1901). Failed keeps the first failure's words whole.
|
|
const MaxFailureLine = 300
|
|
|
|
// levelFailure turns a level that did not pass into a verdict: Failed names what failed first with its words whole,
|
|
// and Failures every failed test of the level's results with its first line, at most delivery.MaxFailures and an entry
|
|
// saying how many more (issue 1921).
|
|
func levelFailure(t *target, level string, results ...Tested) *delivery.Verdict {
|
|
v := delivery.Verdict{State: delivery.VerdictFail}
|
|
var all []delivery.FailedFirst
|
|
for _, res := range results {
|
|
if len(res.Failed) == 0 {
|
|
why := res.Why
|
|
if why == "" {
|
|
why = "the " + level + " level did not pass"
|
|
}
|
|
all = append(all, delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: why})
|
|
continue
|
|
}
|
|
for _, f := range res.Failed {
|
|
name := failedName(f)
|
|
if t.module == buildrun.NodeBuildModule && f.Package == "" {
|
|
name = res.Module
|
|
}
|
|
all = append(all, delivery.FailedFirst{Test: level + ": " + name, Words: failureWordsAfter(f, hidWords(t.module, res.By, f))})
|
|
}
|
|
}
|
|
if len(all) == 0 { // no result: a level said failed with nothing to name
|
|
all = append(all, delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: "the " + level + " level did not pass"})
|
|
}
|
|
v.Failed = all[0]
|
|
v.Failures = failureLines(all)
|
|
return &v
|
|
}
|
|
|
|
// failureLines is every failure with the first line of its words, cut to MaxFailureLine, at most delivery.MaxFailures
|
|
// and then one entry saying how many more failed.
|
|
func failureLines(all []delivery.FailedFirst) []delivery.FailedFirst {
|
|
n := min(len(all), delivery.MaxFailures)
|
|
out := make([]delivery.FailedFirst, 0, n+1)
|
|
for _, f := range all[:n] {
|
|
out = append(out, delivery.FailedFirst{Test: f.Test, Machine: f.Machine, Words: clipLine(firstLine(f.Words), MaxFailureLine)})
|
|
}
|
|
if more := len(all) - n; more > 0 {
|
|
out = append(out, delivery.FailedFirst{Test: fmt.Sprintf("and %d more", more)})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// clipLine cuts a line to at most n bytes on a whole character, marking the cut.
|
|
func clipLine(s string, n int) string {
|
|
if len(s) <= n {
|
|
return s
|
|
}
|
|
s = strings.ToValidUTF8(s[:n-3], "")
|
|
return s + "..."
|
|
}
|