BM2 (sdk): a test that cannot run says NOT RUN; a starved setup runs again

ReadGoTest answers the skips whose words open "NOT RUN:" as not-run with
their reasons; a pipeline test step carries them into its end, the level
passing. A level whose every failure is a starved setup (testbus.SetupStarved)
ends its run again by the again-delays; its third run ends the step could
not run "the bus did not answer in setup within <bound> under load", never
failed. A step's last run that meets a fetch or publish fault now says its
end failed with the why it opens with, so the runner hears it.

Builds: design 47 H2 §800, §799; H-A-294, H-A-367 (3), H-A-368, H-A-399 (1)
This commit is contained in:
2026-11-22 00:53:20 +01:00
parent 9acf583daf
commit 3fce8700bf
5 changed files with 124 additions and 4 deletions
@@ -439,6 +439,10 @@ func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) {
kept []string
passing bool // the error is a transport fault
noTests bool // the level holds no test, as the test body says it
test map[string]any
run int // which run of the ask this is (the step's Run), 0 outside a puller
notRun []sdkstep.NotRun
saidOpen string // the end's said opens with these words
}{
{name: "unit: the kept package carried, the other tested and kept", ask: stepOf(sdkstep.StepUnit, sdkstep.KeptResult{Package: "x/a", Digest: "d-a", Level: "unit"}),
digests: map[string]string{"x/a": "d-a", "x/b": "d-b"}, tests: map[string]bool{"x/b": true}, outcome: sdkstep.OutcomePassed, counts: 1, carried: []string{"x/a"}, kept: []string{"x/b"}},
@@ -453,6 +457,21 @@ func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) {
{name: "the source cannot be fetched: could not run", ask: stepOf(sdkstep.StepUnit), fetchErr: Refuse("no commit %s", runCommit), outcome: sdkstep.OutcomeCouldNotRun},
{name: "a transport fault: unsaid, given again", ask: stepOf(sdkstep.StepUnit), fetchErr: &sdkstep.Passing{Err: errors.New("no responders")}, passing: true},
{name: "the end cannot be said: given again", ask: stepOf(sdkstep.StepLint), sayErr: errors.New("bus down"), passing: true},
// Design 47 H2 §800: the tests that did not run are carried into the end, the level passing.
{name: "not run: passed, listed", ask: stepOf(sdkstep.StepUnit), digests: map[string]string{"x/b": "d-b"}, outcome: sdkstep.OutcomePassed, counts: 2, kept: []string{"x/b"},
test: map[string]any{"passed": true, "tests": 2, "not-run": []sdkstep.NotRun{{Test: "x/b TestBwrap", Reason: "no bwrap"}}},
notRun: []sdkstep.NotRun{{Test: "x/b TestBwrap", Reason: "no bwrap"}}},
// §800, H-A-368: a starved setup runs again; its third run ends could not run, never failed.
{name: "a starved setup: given again", ask: stepOf(sdkstep.StepIntegration), digests: map[string]string{"x/b": "d-b"}, run: 2, passing: true,
test: map[string]any{"passed": false, "tests": 1, "bound": 600, "failed": []sdkstep.Failure{{Package: "x/b", Test: "TestT", Cause: "mesh: setup starved: a bus for this test did not come up within 5m0s"}}}},
{name: "a starved setup's third run: could not run", ask: stepOf(sdkstep.StepIntegration), digests: map[string]string{"x/b": "d-b"}, run: 3, outcome: sdkstep.OutcomeCouldNotRun,
test: map[string]any{"passed": false, "tests": 1, "bound": 600, "failed": []sdkstep.Failure{{Package: "x/b", Test: "TestT", Cause: "mesh: setup starved: a bus for this test did not come up within 5m0s"}}},
saidOpen: "the bus did not answer in setup within 5m0s under load"},
{name: "a starved setup beside an assertion: failed", ask: stepOf(sdkstep.StepIntegration), digests: map[string]string{"x/b": "d-b"}, run: 1, outcome: sdkstep.OutcomeFailed, counts: 2,
test: map[string]any{"passed": false, "tests": 2, "why": "2 failed", "failed": []sdkstep.Failure{{Package: "x/b", Test: "TestT", Cause: "mesh: setup starved: words"}, {Package: "x/b", Test: "TestU", Cause: "got 1, want 2"}}}},
// §799: a fetch met away on the third run ends the step failed with its why.
{name: "a fetch fault's third run: failed, fetch:", ask: stepOf(sdkstep.StepUnit), run: 3, outcome: sdkstep.OutcomeFailed,
fetchErr: &sdkstep.Passing{Err: errors.New("fetch: git fetch: status 502")}, saidOpen: "fetch: git fetch"},
} {
r := &recorder{fetch: map[string]any{"workspace": "/w"}, tests: row.tests, digests: row.digests}
s := r.steps()
@@ -465,6 +484,13 @@ func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) {
return map[string]any{"passed": true, "tests": 0, "no_tests": true}, nil
}
}
if row.test != nil {
s.Test = func(context.Context, map[string]any) (any, error) { return row.test, nil }
}
ctx := context.Background()
if row.run > 0 {
ctx = context.WithValue(ctx, phaseKey{}, &phases{step: sdkstep.Step{Run: row.run}})
}
var got said
s.SayStep = func(e sdkstep.StepEnded) error {
if row.sayErr != nil {
@@ -473,7 +499,7 @@ func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) {
got.ends = append(got.ends, e)
return nil
}
out, err := RunAsk(context.Background(), s, row.ask)
out, err := RunAsk(ctx, s, row.ask)
if row.passing {
if !sdkstep.IsPassing(err) || len(got.ends) != 0 {
t.Errorf("%s: %v, said %+v; want a transport fault, nothing said", row.name, err, got.ends)
@@ -510,6 +536,9 @@ func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) {
if _, err := sdkstep.EncodeStepEnded(e); err != nil {
t.Errorf("%s: the end does not encode: %v", row.name, err)
}
if !slices.Equal(e.NotRun, row.notRun) || !strings.HasPrefix(e.Said, row.saidOpen) {
t.Errorf("%s: not run %+v said %q, want %+v and said opening %q", row.name, e.NotRun, e.Said, row.notRun, row.saidOpen)
}
}
}
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
"slices"
"strings"
"testing"
"time"
@@ -166,9 +167,14 @@ func TestTheFailureContract(t *testing.T) {
t.Errorf("1901/AC4: failures = %+v, want the timeout's panic as the cause", got)
}
res := steprun.ReadGoTest(`{"Action":"run","Package":"m/a","Test":"TestA"}
{"Action":"output","Package":"m/a","Test":"TestA","Output":" a_test.go:41: NOT RUN: no bwrap installed where this check runs\n"}
{"Action":"output","Package":"m/a","Test":"TestA","Output":"--- SKIP: TestA (0.00s)\n"}
{"Action":"skip","Package":"m/a","Test":"TestA","Elapsed":0}
{"Action":"run","Package":"m/b","Test":"TestB"}
{"Action":"pass","Package":"m/b","Test":"TestB","Elapsed":0}
{"Action":"run","Package":"m/b","Test":"TestC"}
{"Action":"output","Package":"m/b","Test":"TestC","Output":" b_test.go:7: short mode\n"}
{"Action":"skip","Package":"m/b","Test":"TestC","Elapsed":0}
{"Action":"pass","Package":"m/a","Elapsed":0}
{"Action":"pass","Package":"m/b","Elapsed":0}
`, ".")
@@ -176,8 +182,12 @@ func TestTheFailureContract(t *testing.T) {
for _, p := range res.Packages {
got[p.Package] = p.Skipped
}
if res.Skipped != 1 || got["m/a"] != 1 || got["m/b"] != 0 {
t.Errorf("skipped = %d, per package %v, want 1 with m/a 1 and m/b 0", res.Skipped, got)
if res.Skipped != 2 || got["m/a"] != 1 || got["m/b"] != 1 {
t.Errorf("skipped = %d, per package %v, want 2 with m/a 1 and m/b 1", res.Skipped, got)
}
// Design 47 H2 §800: a skip whose words open "NOT RUN:" is a test not run, with its reason; another skip is none.
if want := []sdkstep.NotRun{{Test: "m/a TestA", Reason: "no bwrap installed where this check runs"}}; !slices.Equal(res.NotRun, want) {
t.Errorf("not run = %+v, want %+v", res.NotRun, want)
}
})
}
@@ -41,6 +41,9 @@ type GoTest struct {
Skipped int
Times []TestTime
Failed []sdkstep.Failure
// NotRun are the skipped tests whose words open "NOT RUN:", each with its reason (design 47 H2 §800, H-A-294): a test
// lacking tooling its step does not give, counted apart from passed and failed. A skip with other words is none.
NotRun []sdkstep.NotRun
// Elapsed is the slowest package's running time in seconds.
Elapsed float64
// Noise is the lines that were no event: what go said when it could not run the tests at all.
@@ -163,6 +166,9 @@ func (r *goReader) event(ev goEvent) {
if ev.Action == "skip" {
r.res.Skipped++
p.Skipped++
if why, ok := notRunWords(r.testOut[key]); ok {
r.res.NotRun = append(r.res.NotRun, sdkstep.NotRun{Test: ev.Package + " " + ev.Test, Reason: why})
}
}
r.res.Times = append(r.res.Times, TestTime{Package: ev.Package, Test: ev.Test, Elapsed: ev.Elapsed})
if ev.Action == "fail" {
@@ -179,6 +185,19 @@ func (r *goReader) event(ev goEvent) {
}
}
// notRunOpens opens the words of a test that says it did not run (design 47 H2 §800): `t.Skip("NOT RUN: <reason>")`.
const notRunOpens = "NOT RUN:"
// notRunWords is the reason a skipped test's output gives after "NOT RUN:", the first such line's, and whether it said one.
func notRunWords(lines []string) (string, bool) {
for _, l := range lines {
if why, ok := strings.CutPrefix(goErrorWords(l), notRunOpens); ok {
return sdkstep.CutLine(strings.TrimSpace(why)), true
}
}
return "", false
}
// subtestFailed says a failure of test under pkg was already reported for one of its subtests (they end first), each with
// a cause of its own, and the test reported no error and no runtime cause (a subtest's panic) of its own.
func (r *goReader) subtestFailed(pkg, test string) bool {
@@ -72,3 +72,14 @@ func (r *phases) phase() string {
defer r.mu.Unlock()
return r.step.Phase
}
// runOf is the run of its ask that ctx belongs to (the step's Run, the bus's count of runs that started), 0 outside an
// ask's run.
func runOf(ctx context.Context) int {
if r, ok := ctx.Value(phaseKey{}).(*phases); ok && r != nil {
r.mu.Lock()
defer r.mu.Unlock()
return r.step.Run
}
return 0
}
+52 -1
View File
@@ -6,8 +6,10 @@ import (
"errors"
"fmt"
"strings"
"time"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// **A pipeline step, run and said** (novox/hq ADR 0666 §7, §8, §17, §21; design 47 *Pipeline steps and the policy*,
@@ -41,7 +43,16 @@ func (s AskSteps) runStep(ctx context.Context, ask sdkstep.StepAsk) (any, error)
return nil, Refuse("%v", err)
}
if err := s.stepOutcome(ctx, ask, &end); err != nil {
return nil, err
if !sdkstep.IsPassing(err) || runOf(ctx) < sdkstep.AskRuns {
return nil, err
}
// Its last run met a transport fault (design 47 H2 §799, §800): the step ends here and its end is said, a starved
// setup could-not-run, never failed, a fetch or a publish met away failed with the why it opens with.
end.Outcome, end.Said = sdkstep.OutcomeFailed, err.Error()
if errors.Is(err, errSetupStarved) {
end.Outcome = sdkstep.OutcomeCouldNotRun
}
end.Tests, end.Carried, end.Kept, end.NotRun = nil, nil, nil, nil
}
if err := s.SayStep(end); err != nil {
return nil, &sdkstep.Passing{Err: fmt.Errorf("the end of the step %s could not be said: %w", ask.ID, err)}
@@ -147,25 +158,42 @@ func (s AskSteps) testStep(ctx context.Context, ask sdkstep.StepAsk, end *sdkste
}
tests, passed, ran := 0, true, false
var why []string
var failed []sdkstep.Failure
bound := 0.0
for _, t := range la.Tests {
var tr struct {
Passed bool `json:"passed"`
Tests int `json:"tests"`
NoTests bool `json:"no_tests"`
Why string `json:"why"`
// NotRun are the tests that did not run, with their reasons (design 47 H2 §800): the part's not-run, read by
// the build module's reader (ReadGoTest's NotRun).
NotRun []sdkstep.NotRun `json:"not-run"`
Failed []sdkstep.Failure `json:"failed"`
// Bound is the level's bound in seconds, the setup bound being half of it (H-A-368).
Bound float64 `json:"bound"`
}
if err := json.Unmarshal(t, &tr); err != nil {
return fmt.Errorf("the %s level answered what is no test result: %w", level, err)
}
tests += tr.Tests
ran = ran || (!tr.NoTests && tr.Tests > 0) || !tr.Passed
end.NotRun = append(end.NotRun, tr.NotRun...)
bound = max(bound, tr.Bound)
if !tr.Passed {
passed = false
failed = append(failed, tr.Failed...)
if tr.Why != "" {
why = append(why, tr.Why)
}
}
}
if !passed && starvedOnly(failed) {
// Design 47 H2 §800, H-A-368: a setup whose wait on the bus passed its bound runs again by §799's delays, never
// failed; the third such run ends could-not-run (runStep).
end.NotRun = nil
return &sdkstep.Passing{Err: fmt.Errorf("%w %s under load (%s)", errSetupStarved, setupBound(bound), firstOf(failed[0].Cause))}
}
for _, k := range la.Kept {
end.Carried = append(end.Carried, keptOf(run, k.Package, k.Digest))
}
@@ -198,6 +226,29 @@ func (s AskSteps) testStep(ctx context.Context, ask sdkstep.StepAsk, end *sdkste
return nil
}
// errSetupStarved opens the why of a level whose every failure is a starved setup (design 47 H2 §800): its words are the
// could-not-run end's at the third run.
var errSetupStarved = errors.New("the bus did not answer in setup within")
// starvedOnly says a level failed only by starved setups: every failure's words open stdio.SetupStarved (H-A-368). An
// assertion that fails is never `again` (§800), so one such failure among them makes the level failed.
func starvedOnly(failed []sdkstep.Failure) bool {
for _, f := range failed {
if !strings.HasPrefix(strings.TrimSpace(f.Cause), stdio.SetupStarved) {
return false
}
}
return len(failed) > 0
}
// setupBound is the setup bound of a level bound in seconds: half of it, 30 s when the level names none (H-A-368).
func setupBound(levelSeconds float64) time.Duration {
if levelSeconds <= 0 {
return 30 * time.Second
}
return time.Duration(levelSeconds * float64(time.Second) / 2).Round(time.Second)
}
func keptOf(run sdkstep.StepRun, pkg, digest string) sdkstep.KeptResult {
return sdkstep.KeptResult{Package: pkg, Digest: digest, Level: run.Step, Toolchain: run.Toolchain, Pipeline: run.Pipeline}
}