Merge pull request 'The Go and Node build modules scale a test level's bounds by the steps in hand (H-A-488)' (#740) from fix/test-bound-load into main

This commit was merged in pull request #740.
This commit is contained in:
2026-11-22 18:56:07 +00:00
11 changed files with 127 additions and 47 deletions
@@ -34,7 +34,8 @@ func (m *Module) integrationBounds() Bounds {
return b
}
func (m *Module) integration(ctx context.Context, workspace, tree string, packages []string, pattern string) (res *TestResult, err error) {
// integration runs the level judged by b, the integration bounds at the step's start (H-A-488).
func (m *Module) integration(ctx context.Context, workspace, tree string, packages []string, pattern string, b Bounds) (res *TestResult, err error) {
major, err := m.checkPostgres(ctx)
if err != nil {
return nil, err // a *steprun.Refusal: this level only
@@ -42,14 +43,14 @@ func (m *Module) integration(ctx context.Context, workspace, tree string, packag
if !m.Inside {
// The store, the bus and the tests run together in one shut-off process (level.go), so the tests reach the
// level's own store and bus and nothing else.
return m.integrationShutOff(ctx, workspace, packages, pattern)
return m.integrationShutOff(ctx, workspace, packages, pattern, b)
}
return m.integrationHere(ctx, workspace, tree, packages, pattern, major)
return m.integrationHere(ctx, workspace, tree, packages, pattern, major, b)
}
// integrationHere is the level itself, in the shut-off process: the store and the bus started for the run, the tests
// compiled and run against them, both stopped and the run's directory removed however it ends.
func (m *Module) integrationHere(ctx context.Context, workspace, tree string, packages []string, pattern string, major int) (res *TestResult, err error) {
func (m *Module) integrationHere(ctx context.Context, workspace, tree string, packages []string, pattern string, major int, b Bounds) (res *TestResult, err error) {
run := filepath.Join(m.Work, workspace, "run-"+sdkstep.NewID())
if err := os.MkdirAll(run, 0o700); err != nil {
return nil, err
@@ -90,7 +91,7 @@ func (m *Module) integrationHere(ctx context.Context, workspace, tree string, pa
return m.runLevel(ctx, tree, levelOpts{
workspace: workspace, run: pattern,
level: LevelIntegration, tags: integrationTags, packages: packages, bounds: m.integrationBounds(),
level: LevelIntegration, tags: integrationTags, packages: packages, bounds: b,
env: env,
setup: setup, wall: true,
store: "postgres " + strconv.Itoa(major), bus: "nats-server " + m.Recipe.NatsServer,
@@ -256,6 +256,22 @@ func TestSetupAndRunningTimeAreJudgedApart(t *testing.T) {
if res, _ = runLevelTool(t, m, states, "integration"); !res.Passed || res.OverBound || res.Setup != 29 || res.Elapsed != 59 {
t.Errorf("29 s of setup and 59 s of running are within both bounds: %+v", res)
}
// Design 47 H2 H-A-488: with four steps in hand at the step's start, the bounds are four times their defaults: the
// tests get 240 s as their level's bound, the 35 s setup is within its 120 s, and the result names both.
k := newIntegFake(t, "slow")
k.advance["compile"], k.advance["start"] = 20*time.Second, 15*time.Second
m, states = integModule(t, k)
m.inHand = func() int { return 4 }
res, _ = runLevelTool(t, m, states, "integration")
if !res.Passed || res.OverBound || res.Setup != 35 || res.Bound != 240 || res.SetupBound != 120 {
t.Errorf("a setup of 35 s with four steps in hand: %+v", res)
}
for _, c := range k.calls {
if strings.Contains(c.line, " -json ") && !strings.Contains(strings.Join(c.env, " "), "MESH_TEST_LEVEL_BOUND=4m0s") {
t.Errorf("%s ran without the level's bound of 240 s: %v", c.line, c.env)
}
}
}
// 1486/AC5: a run that fails, and a run that panics: the store is stopped, the bus is stopped, the run's directory is
@@ -290,7 +306,7 @@ func TestAFailedOrPanickedRunStopsTheServersAndRemovesTheRunDirectory(t *testing
m, _ = integModule(t, g)
func() {
defer func() { recover() }()
m.integration(context.Background(), "ws1", filepath.Join(m.Work, "ws1", "tree"), nil, "")
m.integration(context.Background(), "ws1", filepath.Join(m.Work, "ws1", "tree"), nil, "", m.integrationBounds())
}()
checkCleaned(g, m)
}
@@ -65,13 +65,12 @@ func (m *Module) executable() (string, error) {
return filepath.EvalSymlinks(exe)
}
// integrationShutOff runs the integration level in one shut-off process of this program and reads its answer.
func (m *Module) integrationShutOff(ctx context.Context, workspace string, packages []string, pattern string) (*TestResult, error) {
// integrationShutOff runs the integration level, judged by b, in one shut-off process of this program and reads its answer.
func (m *Module) integrationShutOff(ctx context.Context, workspace string, packages []string, pattern string, b Bounds) (*TestResult, error) {
exe, err := m.executable()
if err != nil {
return nil, fmt.Errorf("the integration level's program: %w", err)
}
b := m.integrationBounds()
a, err := m.shutOff(ctx, exe, levelSpec{Step: stepIntegration, Workspace: workspace, Packages: packages, Run: pattern, Setup: b.Setup, Bound: b.Run})
if err != nil {
return nil, err
@@ -239,7 +238,7 @@ func (m *Module) answer(ctx context.Context, spec levelSpec) levelAnswer {
if err != nil {
return levelAnswer{Error: err.Error()}
}
res, err := m.integrationHere(ctx, spec.Workspace, tree, spec.Packages, spec.Run, major)
res, err := m.integrationHere(ctx, spec.Workspace, tree, spec.Packages, spec.Run, major, m.integrationBounds())
if err != nil {
return levelAnswer{Error: err.Error()}
}
@@ -32,6 +32,11 @@ type Bounds struct {
Setup, Run time.Duration
}
// times is the bounds of a level run while n steps are in hand (design 47 H2 H-A-488): each its default n times.
func (b Bounds) times(n int) Bounds {
return Bounds{Setup: b.Setup * time.Duration(n), Run: b.Run * time.Duration(n)}
}
// PackageResult, TestTime and Failure are the SDK's failure contract (buildstep, issue 1901), with the JSON names this
// module always sent: a Failure names its test, its file, its cause (never empty, never a passing line) and its own
// output up to 4096 bytes, cause first.
@@ -110,6 +110,8 @@ type Module struct {
// when set, answers it instead (a test).
Settings string
Capacity func() int
// inHand answers how many steps the seat has in hand in place of Held's (a test's; design 47 H2 H-A-488).
inHand func() int
// SayStep says a pipeline step's end (steprun.AskSteps.SayStep); the seat's publish on this node when nil.
SayStep func(sdkstep.StepEnded) error
// ReadEntry asks the seat mesh-catalog's verb entry for the catalog graph entry of one fingerprint, which a compose ask's
@@ -60,6 +60,21 @@ func (m *Module) unitBound() time.Duration {
return UnitBound
}
// stepsInHand is how many steps the seat has in hand at a step's start, this one counted, at least 1 (design 47 H2
// H-A-488): a level's bounds are their defaults that many times, so a level set up slowly on a build node running
// several steps at once is not a failure. In the shut-off process, which holds no puller, it is 1: its spec carries
// the bounds already multiplied.
func (m *Module) stepsInHand() int {
n := 0
switch {
case m.inHand != nil:
n = m.inHand()
case m.Held != nil:
n = m.Held.InHand()
}
return max(1, n)
}
func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
workspace, _ := args["workspace"].(string)
level, _ := args["level"].(string)
@@ -93,13 +108,15 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
}
}
var res *TestResult
// The level's bounds, read once at the step's start (H-A-488): the result's bound and setup_bound name them.
inHand := m.stepsInHand()
switch {
case runner == RunnerWorkspace && (level == LevelUnit || level == LevelIntegration):
return m.testAsCandidate(ctx, workspace, level, packages, pattern, names)
return m.testAsCandidate(ctx, workspace, level, packages, pattern, names, inHand)
case level == LevelUnit:
res, err = m.runLevel(ctx, tree, levelOpts{workspace: workspace, level: LevelUnit, packages: packages, run: pattern, bounds: Bounds{Run: m.unitBound()}, env: m.checkEnv(workspace, names)})
res, err = m.runLevel(ctx, tree, levelOpts{workspace: workspace, level: LevelUnit, packages: packages, run: pattern, bounds: Bounds{Run: m.unitBound()}.times(inHand), env: m.checkEnv(workspace, names)})
case level == LevelIntegration:
res, err = m.integration(ctx, workspace, tree, packages, pattern)
res, err = m.integration(ctx, workspace, tree, packages, pattern, m.integrationBounds().times(inHand))
default:
return nil, fmt.Errorf("the level %q is neither unit nor integration", level)
}
@@ -148,13 +165,13 @@ func compareGenerated(tree string, res *TestResult) error {
// testAsCandidate runs a level as the candidate's program (ADR 0535 §1): the integration level is refused first as the
// installed program would refuse it (no postgres), then the whole level runs in the candidate's one shut-off process.
func (m *Module) testAsCandidate(ctx context.Context, workspace, level string, packages []string, pattern string, names []string) (*TestResult, error) {
spec := levelSpec{Step: stepUnit, Workspace: workspace, Packages: packages, Run: pattern, Bound: m.unitBound(), NodeNames: names}
func (m *Module) testAsCandidate(ctx context.Context, workspace, level string, packages []string, pattern string, names []string, inHand int) (*TestResult, error) {
spec := levelSpec{Step: stepUnit, Workspace: workspace, Packages: packages, Run: pattern, Bound: m.unitBound() * time.Duration(inHand), NodeNames: names}
if level == LevelIntegration {
if _, err := m.checkPostgres(ctx); err != nil {
return nil, err // a *steprun.Refusal: this level only
}
b := m.integrationBounds()
b := m.integrationBounds().times(inHand)
spec = levelSpec{Step: stepIntegration, Workspace: workspace, Packages: packages, Run: pattern, Setup: b.Setup, Bound: b.Run, NodeNames: names}
}
a, judged, err := m.asCandidate(ctx, spec)
@@ -166,7 +166,7 @@ func TestIntegrationLevelRunsAgainstARealStoreAndBusAndLeavesNothing(t *testing.
home := t.TempDir()
m := &Module{Recipe: Current(), Home: home, Work: work, Cache: t.TempDir()}
m.Run = StepRunner(home)
res, err := m.integration(context.Background(), "ws1", filepath.Join(work, "ws1", "tree"), nil, "")
res, err := m.integration(context.Background(), "ws1", filepath.Join(work, "ws1", "tree"), nil, "", m.integrationBounds())
if err != nil {
t.Fatalf("the integration level: %v", err)
}
@@ -269,25 +269,35 @@ func TestUnitLevelOfAPackageThatDoesNotBuildFailsWithTheCompilersLine(t *testing
}
// The command is go test -json in the module's directory of the workspace, with the toolchain pinned to the installed
// one and the module's caches.
// one and the module's caches; the level's bound is its default times the steps in hand at the step's start, at least
// once (design 47 H2 H-A-488), and the result names it.
func TestUnitLevelRunsGoTestJsonInTheModulesDirectoryWithTheStepsEnvironment(t *testing.T) {
m, states, ws, calls := workspaceFor(t, "empty")
runUnit(t, m, states, ws)
var line string
var env string
for _, c := range *calls {
if strings.HasPrefix(c.line, "go ") && strings.Contains(c.line, " test ") {
line, env = c.line, strings.Join(c.env, " ")
for _, row := range []struct {
inHand int
bound time.Duration
}{{0, UnitBound}, {1, UnitBound}, {4, 4 * UnitBound}} {
m, states, ws, calls := workspaceFor(t, "empty")
m.inHand = func() int { return row.inHand }
res, _ := runUnit(t, m, states, ws)
var line string
var env string
for _, c := range *calls {
if strings.HasPrefix(c.line, "go ") && strings.Contains(c.line, " test ") {
line, env = c.line, strings.Join(c.env, " ")
}
}
}
want := "go -C " + filepath.Join(m.Work, ws, "tree", ".") + " test -json -count=1 -timeout 10m ./..."
if line != want {
t.Errorf("command = %q, want %q", line, want)
}
// Design 47 H2 §800: the level's bound reaches the tests as MESH_TEST_LEVEL_BOUND, a duration.
for _, w := range []string{"GOTOOLCHAIN=local", "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), "MESH_TEST_LEVEL_BOUND=" + m.unitBound().String()} {
if !strings.Contains(env, w) {
t.Errorf("environment %s lacks %s", env, w)
want := "go -C " + filepath.Join(m.Work, ws, "tree", ".") + " test -json -count=1 -timeout 10m ./..."
if line != want {
t.Errorf("command = %q, want %q", line, want)
}
// Design 47 H2 §800: the level's bound reaches the tests as MESH_TEST_LEVEL_BOUND, a duration.
for _, w := range []string{"GOTOOLCHAIN=local", "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), "MESH_TEST_LEVEL_BOUND=" + row.bound.String()} {
if !strings.Contains(env, w) {
t.Errorf("%d in hand: environment %s lacks %s", row.inHand, env, w)
}
}
if res.Bound != row.bound.Seconds() {
t.Errorf("%d in hand: the result's bound is %v s, want %v", row.inHand, res.Bound, row.bound.Seconds())
}
}
}
@@ -37,16 +37,30 @@ const (
// test helpers take a setup bound of half of it.
const LevelBoundEnv = "MESH_TEST_LEVEL_BOUND"
// levelBound is a level's bound.
func levelBound(level string) time.Duration {
// levelBound is a level's bound while inHand steps are in hand (design 47 H2 H-A-488): its default that many times.
func levelBound(level string, inHand int) time.Duration {
if level == "unit" {
return UnitBound
return UnitBound * time.Duration(inHand)
}
return IntegrationRunBound
return IntegrationRunBound * time.Duration(inHand)
}
// integrationSetupBound is the integration level's setup bound: half of its bound (§800).
func integrationSetupBound() time.Duration { return IntegrationRunBound / 2 }
func integrationSetupBound(inHand int) time.Duration { return levelBound("integration", inHand) / 2 }
// stepsInHand is how many steps the seat has in hand at a step's start, this one counted, at least 1 (design 47 H2
// H-A-488): a level's bounds are their defaults that many times, so a level set up slowly on a build node running
// several steps at once is not a failure.
func (m *Module) stepsInHand() int {
n := 0
switch {
case m.inHand != nil:
n = m.inHand()
case m.Held != nil:
n = m.Held.InHand()
}
return max(1, n)
}
// TestResult is what a test step passes with, with the field names of mesh-build-go's (issue 1290). The step ends
// passed whenever the level ran, whatever its verdict, and Passed is the verdict; it ends failed only when the tests
@@ -146,6 +160,8 @@ func (m *Module) testLevel(ctx context.Context, args map[string]any) (*TestResul
return nil, err
}
res := &TestResult{Workspace: workspace, Module: module, Level: level, By: sdkstep.By("mesh-build-node")}
// The level's bounds, read once at the step's start (H-A-488): the result's bound and setup_bound name them.
inHand := m.stepsInHand()
script := "test"
if level == "integration" {
script = "test:integration"
@@ -213,7 +229,7 @@ func (m *Module) testLevel(ctx context.Context, args map[string]any) (*TestResul
report := filepath.Join(root, ".mesh-node-test-"+level+".tap")
_ = os.Remove(report)
env := []string{"PATH=" + binDir(root) + ":" + pathOrDefault(), "npm_config_cache=" + filepath.Join(m.Toolchain.Paths.Cache, "npm"),
"NODE_OPTIONS=" + steprun.NodeTestOptions(report), LevelBoundEnv + "=" + levelBound(level).String()}
"NODE_OPTIONS=" + steprun.NodeTestOptions(report), LevelBoundEnv + "=" + levelBound(level, inHand).String()}
r := m.runIn(ctx, root, false, tg.dir, env, "npm", scriptArgs(script)...)
run += m.now().Sub(start)
tap, _ := os.ReadFile(report)
@@ -238,19 +254,20 @@ func (m *Module) testLevel(ctx context.Context, args map[string]any) (*TestResul
why = append(why, failureWhy(f))
}
if level == "unit" {
res.Elapsed, res.Bound = (check + run).Seconds(), UnitBound.Seconds()
res.OverBound = check+run > UnitBound
bound := levelBound(level, inHand)
res.Elapsed, res.Bound = (check + run).Seconds(), bound.Seconds()
res.OverBound = check+run > bound
if res.OverBound {
why = append(why, fmt.Sprintf("the unit level took %.2f s, over its bound of %s", res.Elapsed, UnitBound))
why = append(why, fmt.Sprintf("the unit level took %.2f s, over its bound of %s", res.Elapsed, bound))
}
} else {
res.Setup, res.Elapsed = check.Seconds(), run.Seconds()
setup := integrationSetupBound()
res.SetupBound, res.Bound = setup.Seconds(), IntegrationRunBound.Seconds()
res.OverBound = check > setup || run > IntegrationRunBound
setup, bound := integrationSetupBound(inHand), levelBound(level, inHand)
res.SetupBound, res.Bound = setup.Seconds(), bound.Seconds()
res.OverBound = check > setup || run > bound
if res.OverBound {
why = append(why, fmt.Sprintf("the integration level took %.2f s of setup and %.2f s of running, over its bounds of %s and %s",
res.Setup, res.Elapsed, setup, IntegrationRunBound))
res.Setup, res.Elapsed, setup, bound))
}
}
res.Passed = len(failed) == 0
@@ -140,6 +140,17 @@ func TestTheIntegrationLevelRunsItsScriptAndReportsSetupAndRunningApart(t *testi
if !out.(*TestResult).OverBound {
t.Errorf("61 s of running is within the bound")
}
// Design 47 H2 H-A-488: with four steps in hand at the step's start, the bounds are four times their defaults: the
// script gets 240 s as its level's bound, 40 s of setup is within its 120 s, and the result names both.
f.cost["tsc"], f.cost["npm"] = 40*time.Second, 61*time.Second
m.inHand = func() int { return 4 }
out, _ = m.test(context.Background(), testArgs("integration"))
if r := out.(*TestResult); r.OverBound || r.Bound != 240 || r.SetupBound != 120 {
t.Errorf("40 s of setup and 61 s of running with four steps in hand: %+v", r)
}
if last := f.calls[len(f.calls)-1]; !slices.Contains(last.env, "MESH_TEST_LEVEL_BOUND=4m0s") {
t.Errorf("with four steps in hand the integration script ran with %v, not MESH_TEST_LEVEL_BOUND=4m0s", last.env)
}
}
// The commands of a step get an environment built from scratch: no word of this process's, the level's bound alone (§800).
@@ -107,6 +107,8 @@ type Module struct {
// when set, answers it instead (a test).
Settings string
Capacity func() int
// inHand answers how many steps the seat has in hand in place of Held's (a test's; design 47 H2 H-A-488).
inHand func() int
spacesOnce sync.Once
workspaces *steprun.Workspaces