diff --git a/modules/mesh-delivery-sdk/steprun/askrun_test.go b/modules/mesh-delivery-sdk/steprun/askrun_test.go index f868984f2..887a8618f 100644 --- a/modules/mesh-delivery-sdk/steprun/askrun_test.go +++ b/modules/mesh-delivery-sdk/steprun/askrun_test.go @@ -439,6 +439,10 @@ func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) { kept []string passing bool // the error is a transport fault noTests bool // the level holds no test, as the test body says it + test map[string]any + run int // which run of the ask this is (the step's Run), 0 outside a puller + notRun []sdkstep.NotRun + saidOpen string // the end's said opens with these words }{ {name: "unit: the kept package carried, the other tested and kept", ask: stepOf(sdkstep.StepUnit, sdkstep.KeptResult{Package: "x/a", Digest: "d-a", Level: "unit"}), digests: map[string]string{"x/a": "d-a", "x/b": "d-b"}, tests: map[string]bool{"x/b": true}, outcome: sdkstep.OutcomePassed, counts: 1, carried: []string{"x/a"}, kept: []string{"x/b"}}, @@ -453,6 +457,21 @@ func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) { {name: "the source cannot be fetched: could not run", ask: stepOf(sdkstep.StepUnit), fetchErr: Refuse("no commit %s", runCommit), outcome: sdkstep.OutcomeCouldNotRun}, {name: "a transport fault: unsaid, given again", ask: stepOf(sdkstep.StepUnit), fetchErr: &sdkstep.Passing{Err: errors.New("no responders")}, passing: true}, {name: "the end cannot be said: given again", ask: stepOf(sdkstep.StepLint), sayErr: errors.New("bus down"), passing: true}, + // Design 47 H2 §800: the tests that did not run are carried into the end, the level passing. + {name: "not run: passed, listed", ask: stepOf(sdkstep.StepUnit), digests: map[string]string{"x/b": "d-b"}, outcome: sdkstep.OutcomePassed, counts: 2, kept: []string{"x/b"}, + test: map[string]any{"passed": true, "tests": 2, "not-run": []sdkstep.NotRun{{Test: "x/b TestBwrap", Reason: "no bwrap"}}}, + notRun: []sdkstep.NotRun{{Test: "x/b TestBwrap", Reason: "no bwrap"}}}, + // §800, H-A-368: a starved setup runs again; its third run ends could not run, never failed. + {name: "a starved setup: given again", ask: stepOf(sdkstep.StepIntegration), digests: map[string]string{"x/b": "d-b"}, run: 2, passing: true, + test: map[string]any{"passed": false, "tests": 1, "bound": 600, "failed": []sdkstep.Failure{{Package: "x/b", Test: "TestT", Cause: "mesh: setup starved: a bus for this test did not come up within 5m0s"}}}}, + {name: "a starved setup's third run: could not run", ask: stepOf(sdkstep.StepIntegration), digests: map[string]string{"x/b": "d-b"}, run: 3, outcome: sdkstep.OutcomeCouldNotRun, + test: map[string]any{"passed": false, "tests": 1, "bound": 600, "failed": []sdkstep.Failure{{Package: "x/b", Test: "TestT", Cause: "mesh: setup starved: a bus for this test did not come up within 5m0s"}}}, + saidOpen: "the bus did not answer in setup within 5m0s under load"}, + {name: "a starved setup beside an assertion: failed", ask: stepOf(sdkstep.StepIntegration), digests: map[string]string{"x/b": "d-b"}, run: 1, outcome: sdkstep.OutcomeFailed, counts: 2, + test: map[string]any{"passed": false, "tests": 2, "why": "2 failed", "failed": []sdkstep.Failure{{Package: "x/b", Test: "TestT", Cause: "mesh: setup starved: words"}, {Package: "x/b", Test: "TestU", Cause: "got 1, want 2"}}}}, + // §799: a fetch met away on the third run ends the step failed with its why. + {name: "a fetch fault's third run: failed, fetch:", ask: stepOf(sdkstep.StepUnit), run: 3, outcome: sdkstep.OutcomeFailed, + fetchErr: &sdkstep.Passing{Err: errors.New("fetch: git fetch: status 502")}, saidOpen: "fetch: git fetch"}, } { r := &recorder{fetch: map[string]any{"workspace": "/w"}, tests: row.tests, digests: row.digests} s := r.steps() @@ -465,6 +484,13 @@ func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) { return map[string]any{"passed": true, "tests": 0, "no_tests": true}, nil } } + if row.test != nil { + s.Test = func(context.Context, map[string]any) (any, error) { return row.test, nil } + } + ctx := context.Background() + if row.run > 0 { + ctx = context.WithValue(ctx, phaseKey{}, &phases{step: sdkstep.Step{Run: row.run}}) + } var got said s.SayStep = func(e sdkstep.StepEnded) error { if row.sayErr != nil { @@ -473,7 +499,7 @@ func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) { got.ends = append(got.ends, e) return nil } - out, err := RunAsk(context.Background(), s, row.ask) + out, err := RunAsk(ctx, s, row.ask) if row.passing { if !sdkstep.IsPassing(err) || len(got.ends) != 0 { t.Errorf("%s: %v, said %+v; want a transport fault, nothing said", row.name, err, got.ends) @@ -510,6 +536,9 @@ func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) { if _, err := sdkstep.EncodeStepEnded(e); err != nil { t.Errorf("%s: the end does not encode: %v", row.name, err) } + if !slices.Equal(e.NotRun, row.notRun) || !strings.HasPrefix(e.Said, row.saidOpen) { + t.Errorf("%s: not run %+v said %q, want %+v and said opening %q", row.name, e.NotRun, e.Said, row.notRun, row.saidOpen) + } } } diff --git a/modules/mesh-delivery-sdk/steprun/conformance/conformance_test.go b/modules/mesh-delivery-sdk/steprun/conformance/conformance_test.go index 51c33107c..a3c80234b 100644 --- a/modules/mesh-delivery-sdk/steprun/conformance/conformance_test.go +++ b/modules/mesh-delivery-sdk/steprun/conformance/conformance_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "slices" "strings" "testing" "time" @@ -166,9 +167,14 @@ func TestTheFailureContract(t *testing.T) { t.Errorf("1901/AC4: failures = %+v, want the timeout's panic as the cause", got) } res := steprun.ReadGoTest(`{"Action":"run","Package":"m/a","Test":"TestA"} +{"Action":"output","Package":"m/a","Test":"TestA","Output":" a_test.go:41: NOT RUN: no bwrap installed where this check runs\n"} +{"Action":"output","Package":"m/a","Test":"TestA","Output":"--- SKIP: TestA (0.00s)\n"} {"Action":"skip","Package":"m/a","Test":"TestA","Elapsed":0} {"Action":"run","Package":"m/b","Test":"TestB"} {"Action":"pass","Package":"m/b","Test":"TestB","Elapsed":0} +{"Action":"run","Package":"m/b","Test":"TestC"} +{"Action":"output","Package":"m/b","Test":"TestC","Output":" b_test.go:7: short mode\n"} +{"Action":"skip","Package":"m/b","Test":"TestC","Elapsed":0} {"Action":"pass","Package":"m/a","Elapsed":0} {"Action":"pass","Package":"m/b","Elapsed":0} `, ".") @@ -176,8 +182,12 @@ func TestTheFailureContract(t *testing.T) { for _, p := range res.Packages { got[p.Package] = p.Skipped } - if res.Skipped != 1 || got["m/a"] != 1 || got["m/b"] != 0 { - t.Errorf("skipped = %d, per package %v, want 1 with m/a 1 and m/b 0", res.Skipped, got) + if res.Skipped != 2 || got["m/a"] != 1 || got["m/b"] != 1 { + t.Errorf("skipped = %d, per package %v, want 2 with m/a 1 and m/b 1", res.Skipped, got) + } + // Design 47 H2 §800: a skip whose words open "NOT RUN:" is a test not run, with its reason; another skip is none. + if want := []sdkstep.NotRun{{Test: "m/a TestA", Reason: "no bwrap installed where this check runs"}}; !slices.Equal(res.NotRun, want) { + t.Errorf("not run = %+v, want %+v", res.NotRun, want) } }) } diff --git a/modules/mesh-delivery-sdk/steprun/gotest.go b/modules/mesh-delivery-sdk/steprun/gotest.go index cd6d3aaa7..45f4f2a6a 100644 --- a/modules/mesh-delivery-sdk/steprun/gotest.go +++ b/modules/mesh-delivery-sdk/steprun/gotest.go @@ -41,6 +41,9 @@ type GoTest struct { Skipped int Times []TestTime Failed []sdkstep.Failure + // NotRun are the skipped tests whose words open "NOT RUN:", each with its reason (design 47 H2 §800, H-A-294): a test + // lacking tooling its step does not give, counted apart from passed and failed. A skip with other words is none. + NotRun []sdkstep.NotRun // Elapsed is the slowest package's running time in seconds. Elapsed float64 // Noise is the lines that were no event: what go said when it could not run the tests at all. @@ -163,6 +166,9 @@ func (r *goReader) event(ev goEvent) { if ev.Action == "skip" { r.res.Skipped++ p.Skipped++ + if why, ok := notRunWords(r.testOut[key]); ok { + r.res.NotRun = append(r.res.NotRun, sdkstep.NotRun{Test: ev.Package + " " + ev.Test, Reason: why}) + } } r.res.Times = append(r.res.Times, TestTime{Package: ev.Package, Test: ev.Test, Elapsed: ev.Elapsed}) if ev.Action == "fail" { @@ -179,6 +185,19 @@ func (r *goReader) event(ev goEvent) { } } +// notRunOpens opens the words of a test that says it did not run (design 47 H2 §800): `t.Skip("NOT RUN: ")`. +const notRunOpens = "NOT RUN:" + +// notRunWords is the reason a skipped test's output gives after "NOT RUN:", the first such line's, and whether it said one. +func notRunWords(lines []string) (string, bool) { + for _, l := range lines { + if why, ok := strings.CutPrefix(goErrorWords(l), notRunOpens); ok { + return sdkstep.CutLine(strings.TrimSpace(why)), true + } + } + return "", false +} + // subtestFailed says a failure of test under pkg was already reported for one of its subtests (they end first), each with // a cause of its own, and the test reported no error and no runtime cause (a subtest's panic) of its own. func (r *goReader) subtestFailed(pkg, test string) bool { diff --git a/modules/mesh-delivery-sdk/steprun/phase.go b/modules/mesh-delivery-sdk/steprun/phase.go index 72d384656..f3a5e40a1 100644 --- a/modules/mesh-delivery-sdk/steprun/phase.go +++ b/modules/mesh-delivery-sdk/steprun/phase.go @@ -72,3 +72,14 @@ func (r *phases) phase() string { defer r.mu.Unlock() return r.step.Phase } + +// runOf is the run of its ask that ctx belongs to (the step's Run, the bus's count of runs that started), 0 outside an +// ask's run. +func runOf(ctx context.Context) int { + if r, ok := ctx.Value(phaseKey{}).(*phases); ok && r != nil { + r.mu.Lock() + defer r.mu.Unlock() + return r.step.Run + } + return 0 +} diff --git a/modules/mesh-delivery-sdk/steprun/steprun.go b/modules/mesh-delivery-sdk/steprun/steprun.go index 7cd29e6e2..0a2d19581 100644 --- a/modules/mesh-delivery-sdk/steprun/steprun.go +++ b/modules/mesh-delivery-sdk/steprun/steprun.go @@ -6,8 +6,10 @@ import ( "errors" "fmt" "strings" + "time" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" + stdio "git.novox.be/novox/mesh-sdk/go" ) // **A pipeline step, run and said** (novox/hq ADR 0666 §7, §8, §17, §21; design 47 *Pipeline steps and the policy*, @@ -41,7 +43,16 @@ func (s AskSteps) runStep(ctx context.Context, ask sdkstep.StepAsk) (any, error) return nil, Refuse("%v", err) } if err := s.stepOutcome(ctx, ask, &end); err != nil { - return nil, err + if !sdkstep.IsPassing(err) || runOf(ctx) < sdkstep.AskRuns { + return nil, err + } + // Its last run met a transport fault (design 47 H2 §799, §800): the step ends here and its end is said, a starved + // setup could-not-run, never failed, a fetch or a publish met away failed with the why it opens with. + end.Outcome, end.Said = sdkstep.OutcomeFailed, err.Error() + if errors.Is(err, errSetupStarved) { + end.Outcome = sdkstep.OutcomeCouldNotRun + } + end.Tests, end.Carried, end.Kept, end.NotRun = nil, nil, nil, nil } if err := s.SayStep(end); err != nil { return nil, &sdkstep.Passing{Err: fmt.Errorf("the end of the step %s could not be said: %w", ask.ID, err)} @@ -147,25 +158,42 @@ func (s AskSteps) testStep(ctx context.Context, ask sdkstep.StepAsk, end *sdkste } tests, passed, ran := 0, true, false var why []string + var failed []sdkstep.Failure + bound := 0.0 for _, t := range la.Tests { var tr struct { Passed bool `json:"passed"` Tests int `json:"tests"` NoTests bool `json:"no_tests"` Why string `json:"why"` + // NotRun are the tests that did not run, with their reasons (design 47 H2 §800): the part's not-run, read by + // the build module's reader (ReadGoTest's NotRun). + NotRun []sdkstep.NotRun `json:"not-run"` + Failed []sdkstep.Failure `json:"failed"` + // Bound is the level's bound in seconds, the setup bound being half of it (H-A-368). + Bound float64 `json:"bound"` } if err := json.Unmarshal(t, &tr); err != nil { return fmt.Errorf("the %s level answered what is no test result: %w", level, err) } tests += tr.Tests ran = ran || (!tr.NoTests && tr.Tests > 0) || !tr.Passed + end.NotRun = append(end.NotRun, tr.NotRun...) + bound = max(bound, tr.Bound) if !tr.Passed { passed = false + failed = append(failed, tr.Failed...) if tr.Why != "" { why = append(why, tr.Why) } } } + if !passed && starvedOnly(failed) { + // Design 47 H2 §800, H-A-368: a setup whose wait on the bus passed its bound runs again by §799's delays, never + // failed; the third such run ends could-not-run (runStep). + end.NotRun = nil + return &sdkstep.Passing{Err: fmt.Errorf("%w %s under load (%s)", errSetupStarved, setupBound(bound), firstOf(failed[0].Cause))} + } for _, k := range la.Kept { end.Carried = append(end.Carried, keptOf(run, k.Package, k.Digest)) } @@ -198,6 +226,29 @@ func (s AskSteps) testStep(ctx context.Context, ask sdkstep.StepAsk, end *sdkste return nil } +// errSetupStarved opens the why of a level whose every failure is a starved setup (design 47 H2 §800): its words are the +// could-not-run end's at the third run. +var errSetupStarved = errors.New("the bus did not answer in setup within") + +// starvedOnly says a level failed only by starved setups: every failure's words open stdio.SetupStarved (H-A-368). An +// assertion that fails is never `again` (§800), so one such failure among them makes the level failed. +func starvedOnly(failed []sdkstep.Failure) bool { + for _, f := range failed { + if !strings.HasPrefix(strings.TrimSpace(f.Cause), stdio.SetupStarved) { + return false + } + } + return len(failed) > 0 +} + +// setupBound is the setup bound of a level bound in seconds: half of it, 30 s when the level names none (H-A-368). +func setupBound(levelSeconds float64) time.Duration { + if levelSeconds <= 0 { + return 30 * time.Second + } + return time.Duration(levelSeconds * float64(time.Second) / 2).Round(time.Second) +} + func keptOf(run sdkstep.StepRun, pkg, digest string) sdkstep.KeptResult { return sdkstep.KeptResult{Package: pkg, Digest: digest, Level: run.Step, Toolchain: run.Toolchain, Pipeline: run.Pipeline} }