BM2 (sdk): a test that cannot run says NOT RUN; a starved setup runs again
ReadGoTest answers the skips whose words open "NOT RUN:" as not-run with their reasons; a pipeline test step carries them into its end, the level passing. A level whose every failure is a starved setup (testbus.SetupStarved) ends its run again by the again-delays; its third run ends the step could not run "the bus did not answer in setup within <bound> under load", never failed. A step's last run that meets a fetch or publish fault now says its end failed with the why it opens with, so the runner hears it. Builds: design 47 H2 §800, §799; H-A-294, H-A-367 (3), H-A-368, H-A-399 (1)
This commit is contained in:
@@ -439,6 +439,10 @@ func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) {
|
||||
kept []string
|
||||
passing bool // the error is a transport fault
|
||||
noTests bool // the level holds no test, as the test body says it
|
||||
test map[string]any
|
||||
run int // which run of the ask this is (the step's Run), 0 outside a puller
|
||||
notRun []sdkstep.NotRun
|
||||
saidOpen string // the end's said opens with these words
|
||||
}{
|
||||
{name: "unit: the kept package carried, the other tested and kept", ask: stepOf(sdkstep.StepUnit, sdkstep.KeptResult{Package: "x/a", Digest: "d-a", Level: "unit"}),
|
||||
digests: map[string]string{"x/a": "d-a", "x/b": "d-b"}, tests: map[string]bool{"x/b": true}, outcome: sdkstep.OutcomePassed, counts: 1, carried: []string{"x/a"}, kept: []string{"x/b"}},
|
||||
@@ -453,6 +457,21 @@ func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) {
|
||||
{name: "the source cannot be fetched: could not run", ask: stepOf(sdkstep.StepUnit), fetchErr: Refuse("no commit %s", runCommit), outcome: sdkstep.OutcomeCouldNotRun},
|
||||
{name: "a transport fault: unsaid, given again", ask: stepOf(sdkstep.StepUnit), fetchErr: &sdkstep.Passing{Err: errors.New("no responders")}, passing: true},
|
||||
{name: "the end cannot be said: given again", ask: stepOf(sdkstep.StepLint), sayErr: errors.New("bus down"), passing: true},
|
||||
// Design 47 H2 §800: the tests that did not run are carried into the end, the level passing.
|
||||
{name: "not run: passed, listed", ask: stepOf(sdkstep.StepUnit), digests: map[string]string{"x/b": "d-b"}, outcome: sdkstep.OutcomePassed, counts: 2, kept: []string{"x/b"},
|
||||
test: map[string]any{"passed": true, "tests": 2, "not-run": []sdkstep.NotRun{{Test: "x/b TestBwrap", Reason: "no bwrap"}}},
|
||||
notRun: []sdkstep.NotRun{{Test: "x/b TestBwrap", Reason: "no bwrap"}}},
|
||||
// §800, H-A-368: a starved setup runs again; its third run ends could not run, never failed.
|
||||
{name: "a starved setup: given again", ask: stepOf(sdkstep.StepIntegration), digests: map[string]string{"x/b": "d-b"}, run: 2, passing: true,
|
||||
test: map[string]any{"passed": false, "tests": 1, "bound": 600, "failed": []sdkstep.Failure{{Package: "x/b", Test: "TestT", Cause: "mesh: setup starved: a bus for this test did not come up within 5m0s"}}}},
|
||||
{name: "a starved setup's third run: could not run", ask: stepOf(sdkstep.StepIntegration), digests: map[string]string{"x/b": "d-b"}, run: 3, outcome: sdkstep.OutcomeCouldNotRun,
|
||||
test: map[string]any{"passed": false, "tests": 1, "bound": 600, "failed": []sdkstep.Failure{{Package: "x/b", Test: "TestT", Cause: "mesh: setup starved: a bus for this test did not come up within 5m0s"}}},
|
||||
saidOpen: "the bus did not answer in setup within 5m0s under load"},
|
||||
{name: "a starved setup beside an assertion: failed", ask: stepOf(sdkstep.StepIntegration), digests: map[string]string{"x/b": "d-b"}, run: 1, outcome: sdkstep.OutcomeFailed, counts: 2,
|
||||
test: map[string]any{"passed": false, "tests": 2, "why": "2 failed", "failed": []sdkstep.Failure{{Package: "x/b", Test: "TestT", Cause: "mesh: setup starved: words"}, {Package: "x/b", Test: "TestU", Cause: "got 1, want 2"}}}},
|
||||
// §799: a fetch met away on the third run ends the step failed with its why.
|
||||
{name: "a fetch fault's third run: failed, fetch:", ask: stepOf(sdkstep.StepUnit), run: 3, outcome: sdkstep.OutcomeFailed,
|
||||
fetchErr: &sdkstep.Passing{Err: errors.New("fetch: git fetch: status 502")}, saidOpen: "fetch: git fetch"},
|
||||
} {
|
||||
r := &recorder{fetch: map[string]any{"workspace": "/w"}, tests: row.tests, digests: row.digests}
|
||||
s := r.steps()
|
||||
@@ -465,6 +484,13 @@ func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) {
|
||||
return map[string]any{"passed": true, "tests": 0, "no_tests": true}, nil
|
||||
}
|
||||
}
|
||||
if row.test != nil {
|
||||
s.Test = func(context.Context, map[string]any) (any, error) { return row.test, nil }
|
||||
}
|
||||
ctx := context.Background()
|
||||
if row.run > 0 {
|
||||
ctx = context.WithValue(ctx, phaseKey{}, &phases{step: sdkstep.Step{Run: row.run}})
|
||||
}
|
||||
var got said
|
||||
s.SayStep = func(e sdkstep.StepEnded) error {
|
||||
if row.sayErr != nil {
|
||||
@@ -473,7 +499,7 @@ func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) {
|
||||
got.ends = append(got.ends, e)
|
||||
return nil
|
||||
}
|
||||
out, err := RunAsk(context.Background(), s, row.ask)
|
||||
out, err := RunAsk(ctx, s, row.ask)
|
||||
if row.passing {
|
||||
if !sdkstep.IsPassing(err) || len(got.ends) != 0 {
|
||||
t.Errorf("%s: %v, said %+v; want a transport fault, nothing said", row.name, err, got.ends)
|
||||
@@ -510,6 +536,9 @@ func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) {
|
||||
if _, err := sdkstep.EncodeStepEnded(e); err != nil {
|
||||
t.Errorf("%s: the end does not encode: %v", row.name, err)
|
||||
}
|
||||
if !slices.Equal(e.NotRun, row.notRun) || !strings.HasPrefix(e.Said, row.saidOpen) {
|
||||
t.Errorf("%s: not run %+v said %q, want %+v and said opening %q", row.name, e.NotRun, e.Said, row.notRun, row.saidOpen)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -166,9 +167,14 @@ func TestTheFailureContract(t *testing.T) {
|
||||
t.Errorf("1901/AC4: failures = %+v, want the timeout's panic as the cause", got)
|
||||
}
|
||||
res := steprun.ReadGoTest(`{"Action":"run","Package":"m/a","Test":"TestA"}
|
||||
{"Action":"output","Package":"m/a","Test":"TestA","Output":" a_test.go:41: NOT RUN: no bwrap installed where this check runs\n"}
|
||||
{"Action":"output","Package":"m/a","Test":"TestA","Output":"--- SKIP: TestA (0.00s)\n"}
|
||||
{"Action":"skip","Package":"m/a","Test":"TestA","Elapsed":0}
|
||||
{"Action":"run","Package":"m/b","Test":"TestB"}
|
||||
{"Action":"pass","Package":"m/b","Test":"TestB","Elapsed":0}
|
||||
{"Action":"run","Package":"m/b","Test":"TestC"}
|
||||
{"Action":"output","Package":"m/b","Test":"TestC","Output":" b_test.go:7: short mode\n"}
|
||||
{"Action":"skip","Package":"m/b","Test":"TestC","Elapsed":0}
|
||||
{"Action":"pass","Package":"m/a","Elapsed":0}
|
||||
{"Action":"pass","Package":"m/b","Elapsed":0}
|
||||
`, ".")
|
||||
@@ -176,8 +182,12 @@ func TestTheFailureContract(t *testing.T) {
|
||||
for _, p := range res.Packages {
|
||||
got[p.Package] = p.Skipped
|
||||
}
|
||||
if res.Skipped != 1 || got["m/a"] != 1 || got["m/b"] != 0 {
|
||||
t.Errorf("skipped = %d, per package %v, want 1 with m/a 1 and m/b 0", res.Skipped, got)
|
||||
if res.Skipped != 2 || got["m/a"] != 1 || got["m/b"] != 1 {
|
||||
t.Errorf("skipped = %d, per package %v, want 2 with m/a 1 and m/b 1", res.Skipped, got)
|
||||
}
|
||||
// Design 47 H2 §800: a skip whose words open "NOT RUN:" is a test not run, with its reason; another skip is none.
|
||||
if want := []sdkstep.NotRun{{Test: "m/a TestA", Reason: "no bwrap installed where this check runs"}}; !slices.Equal(res.NotRun, want) {
|
||||
t.Errorf("not run = %+v, want %+v", res.NotRun, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -41,6 +41,9 @@ type GoTest struct {
|
||||
Skipped int
|
||||
Times []TestTime
|
||||
Failed []sdkstep.Failure
|
||||
// NotRun are the skipped tests whose words open "NOT RUN:", each with its reason (design 47 H2 §800, H-A-294): a test
|
||||
// lacking tooling its step does not give, counted apart from passed and failed. A skip with other words is none.
|
||||
NotRun []sdkstep.NotRun
|
||||
// Elapsed is the slowest package's running time in seconds.
|
||||
Elapsed float64
|
||||
// Noise is the lines that were no event: what go said when it could not run the tests at all.
|
||||
@@ -163,6 +166,9 @@ func (r *goReader) event(ev goEvent) {
|
||||
if ev.Action == "skip" {
|
||||
r.res.Skipped++
|
||||
p.Skipped++
|
||||
if why, ok := notRunWords(r.testOut[key]); ok {
|
||||
r.res.NotRun = append(r.res.NotRun, sdkstep.NotRun{Test: ev.Package + " " + ev.Test, Reason: why})
|
||||
}
|
||||
}
|
||||
r.res.Times = append(r.res.Times, TestTime{Package: ev.Package, Test: ev.Test, Elapsed: ev.Elapsed})
|
||||
if ev.Action == "fail" {
|
||||
@@ -179,6 +185,19 @@ func (r *goReader) event(ev goEvent) {
|
||||
}
|
||||
}
|
||||
|
||||
// notRunOpens opens the words of a test that says it did not run (design 47 H2 §800): `t.Skip("NOT RUN: <reason>")`.
|
||||
const notRunOpens = "NOT RUN:"
|
||||
|
||||
// notRunWords is the reason a skipped test's output gives after "NOT RUN:", the first such line's, and whether it said one.
|
||||
func notRunWords(lines []string) (string, bool) {
|
||||
for _, l := range lines {
|
||||
if why, ok := strings.CutPrefix(goErrorWords(l), notRunOpens); ok {
|
||||
return sdkstep.CutLine(strings.TrimSpace(why)), true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// subtestFailed says a failure of test under pkg was already reported for one of its subtests (they end first), each with
|
||||
// a cause of its own, and the test reported no error and no runtime cause (a subtest's panic) of its own.
|
||||
func (r *goReader) subtestFailed(pkg, test string) bool {
|
||||
|
||||
@@ -72,3 +72,14 @@ func (r *phases) phase() string {
|
||||
defer r.mu.Unlock()
|
||||
return r.step.Phase
|
||||
}
|
||||
|
||||
// runOf is the run of its ask that ctx belongs to (the step's Run, the bus's count of runs that started), 0 outside an
|
||||
// ask's run.
|
||||
func runOf(ctx context.Context) int {
|
||||
if r, ok := ctx.Value(phaseKey{}).(*phases); ok && r != nil {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.step.Run
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -6,8 +6,10 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// **A pipeline step, run and said** (novox/hq ADR 0666 §7, §8, §17, §21; design 47 *Pipeline steps and the policy*,
|
||||
@@ -41,7 +43,16 @@ func (s AskSteps) runStep(ctx context.Context, ask sdkstep.StepAsk) (any, error)
|
||||
return nil, Refuse("%v", err)
|
||||
}
|
||||
if err := s.stepOutcome(ctx, ask, &end); err != nil {
|
||||
return nil, err
|
||||
if !sdkstep.IsPassing(err) || runOf(ctx) < sdkstep.AskRuns {
|
||||
return nil, err
|
||||
}
|
||||
// Its last run met a transport fault (design 47 H2 §799, §800): the step ends here and its end is said, a starved
|
||||
// setup could-not-run, never failed, a fetch or a publish met away failed with the why it opens with.
|
||||
end.Outcome, end.Said = sdkstep.OutcomeFailed, err.Error()
|
||||
if errors.Is(err, errSetupStarved) {
|
||||
end.Outcome = sdkstep.OutcomeCouldNotRun
|
||||
}
|
||||
end.Tests, end.Carried, end.Kept, end.NotRun = nil, nil, nil, nil
|
||||
}
|
||||
if err := s.SayStep(end); err != nil {
|
||||
return nil, &sdkstep.Passing{Err: fmt.Errorf("the end of the step %s could not be said: %w", ask.ID, err)}
|
||||
@@ -147,25 +158,42 @@ func (s AskSteps) testStep(ctx context.Context, ask sdkstep.StepAsk, end *sdkste
|
||||
}
|
||||
tests, passed, ran := 0, true, false
|
||||
var why []string
|
||||
var failed []sdkstep.Failure
|
||||
bound := 0.0
|
||||
for _, t := range la.Tests {
|
||||
var tr struct {
|
||||
Passed bool `json:"passed"`
|
||||
Tests int `json:"tests"`
|
||||
NoTests bool `json:"no_tests"`
|
||||
Why string `json:"why"`
|
||||
// NotRun are the tests that did not run, with their reasons (design 47 H2 §800): the part's not-run, read by
|
||||
// the build module's reader (ReadGoTest's NotRun).
|
||||
NotRun []sdkstep.NotRun `json:"not-run"`
|
||||
Failed []sdkstep.Failure `json:"failed"`
|
||||
// Bound is the level's bound in seconds, the setup bound being half of it (H-A-368).
|
||||
Bound float64 `json:"bound"`
|
||||
}
|
||||
if err := json.Unmarshal(t, &tr); err != nil {
|
||||
return fmt.Errorf("the %s level answered what is no test result: %w", level, err)
|
||||
}
|
||||
tests += tr.Tests
|
||||
ran = ran || (!tr.NoTests && tr.Tests > 0) || !tr.Passed
|
||||
end.NotRun = append(end.NotRun, tr.NotRun...)
|
||||
bound = max(bound, tr.Bound)
|
||||
if !tr.Passed {
|
||||
passed = false
|
||||
failed = append(failed, tr.Failed...)
|
||||
if tr.Why != "" {
|
||||
why = append(why, tr.Why)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !passed && starvedOnly(failed) {
|
||||
// Design 47 H2 §800, H-A-368: a setup whose wait on the bus passed its bound runs again by §799's delays, never
|
||||
// failed; the third such run ends could-not-run (runStep).
|
||||
end.NotRun = nil
|
||||
return &sdkstep.Passing{Err: fmt.Errorf("%w %s under load (%s)", errSetupStarved, setupBound(bound), firstOf(failed[0].Cause))}
|
||||
}
|
||||
for _, k := range la.Kept {
|
||||
end.Carried = append(end.Carried, keptOf(run, k.Package, k.Digest))
|
||||
}
|
||||
@@ -198,6 +226,29 @@ func (s AskSteps) testStep(ctx context.Context, ask sdkstep.StepAsk, end *sdkste
|
||||
return nil
|
||||
}
|
||||
|
||||
// errSetupStarved opens the why of a level whose every failure is a starved setup (design 47 H2 §800): its words are the
|
||||
// could-not-run end's at the third run.
|
||||
var errSetupStarved = errors.New("the bus did not answer in setup within")
|
||||
|
||||
// starvedOnly says a level failed only by starved setups: every failure's words open stdio.SetupStarved (H-A-368). An
|
||||
// assertion that fails is never `again` (§800), so one such failure among them makes the level failed.
|
||||
func starvedOnly(failed []sdkstep.Failure) bool {
|
||||
for _, f := range failed {
|
||||
if !strings.HasPrefix(strings.TrimSpace(f.Cause), stdio.SetupStarved) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(failed) > 0
|
||||
}
|
||||
|
||||
// setupBound is the setup bound of a level bound in seconds: half of it, 30 s when the level names none (H-A-368).
|
||||
func setupBound(levelSeconds float64) time.Duration {
|
||||
if levelSeconds <= 0 {
|
||||
return 30 * time.Second
|
||||
}
|
||||
return time.Duration(levelSeconds * float64(time.Second) / 2).Round(time.Second)
|
||||
}
|
||||
|
||||
func keptOf(run sdkstep.StepRun, pkg, digest string) sdkstep.KeptResult {
|
||||
return sdkstep.KeptResult{Package: pkg, Digest: digest, Level: run.Step, Toolchain: run.Toolchain, Pipeline: run.Pipeline}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user