From af8173651b5b8750ac130a05af5432c52ad463c1 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 19:41:33 +0100 Subject: [PATCH 1/2] Go build module: a test level's bounds follow the steps in hand (H-A-488) On a loaded build node the integration level's setup took 34-37 s against its 30 s bound three times today: the node-engine twice, the assigner once. The bounds were judging the node's load, not the tests. A level's bounds, and the MESH_TEST_LEVEL_BOUND its tests get, are now their defaults times the steps the seat has in hand at the step's start (at least 1). The step's result keeps naming the bounds it was judged by (bound, setup_bound). Builds: design 47 H2 H-A-488 --- .../cmd/mesh-build-go/integration.go | 11 ++--- .../mesh-build-go/integration_level_test.go | 18 +++++++- .../mesh-build-go/cmd/mesh-build-go/level.go | 7 ++- .../mesh-build-go/cmd/mesh-build-go/levels.go | 5 +++ .../mesh-build-go/cmd/mesh-build-go/module.go | 2 + .../mesh-build-go/cmd/mesh-build-go/test.go | 29 +++++++++--- .../test_real_integration_test.go | 2 +- .../cmd/mesh-build-go/test_test.go | 44 ++++++++++++------- 8 files changed, 84 insertions(+), 34 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/integration.go b/modules/mesh-build-go/cmd/mesh-build-go/integration.go index cfa51ba98..9a12ed5a8 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/integration.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/integration.go @@ -34,7 +34,8 @@ func (m *Module) integrationBounds() Bounds { return b } -func (m *Module) integration(ctx context.Context, workspace, tree string, packages []string, pattern string) (res *TestResult, err error) { +// integration runs the level judged by b, the integration bounds at the step's start (H-A-488). +func (m *Module) integration(ctx context.Context, workspace, tree string, packages []string, pattern string, b Bounds) (res *TestResult, err error) { major, err := m.checkPostgres(ctx) if err != nil { return nil, err // a *steprun.Refusal: this level only @@ -42,14 +43,14 @@ func (m *Module) integration(ctx context.Context, workspace, tree string, packag if !m.Inside { // The store, the bus and the tests run together in one shut-off process (level.go), so the tests reach the // level's own store and bus and nothing else. - return m.integrationShutOff(ctx, workspace, packages, pattern) + return m.integrationShutOff(ctx, workspace, packages, pattern, b) } - return m.integrationHere(ctx, workspace, tree, packages, pattern, major) + return m.integrationHere(ctx, workspace, tree, packages, pattern, major, b) } // integrationHere is the level itself, in the shut-off process: the store and the bus started for the run, the tests // compiled and run against them, both stopped and the run's directory removed however it ends. -func (m *Module) integrationHere(ctx context.Context, workspace, tree string, packages []string, pattern string, major int) (res *TestResult, err error) { +func (m *Module) integrationHere(ctx context.Context, workspace, tree string, packages []string, pattern string, major int, b Bounds) (res *TestResult, err error) { run := filepath.Join(m.Work, workspace, "run-"+sdkstep.NewID()) if err := os.MkdirAll(run, 0o700); err != nil { return nil, err @@ -90,7 +91,7 @@ func (m *Module) integrationHere(ctx context.Context, workspace, tree string, pa return m.runLevel(ctx, tree, levelOpts{ workspace: workspace, run: pattern, - level: LevelIntegration, tags: integrationTags, packages: packages, bounds: m.integrationBounds(), + level: LevelIntegration, tags: integrationTags, packages: packages, bounds: b, env: env, setup: setup, wall: true, store: "postgres " + strconv.Itoa(major), bus: "nats-server " + m.Recipe.NatsServer, diff --git a/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go b/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go index 49889ee94..3f7c9facf 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go @@ -256,6 +256,22 @@ func TestSetupAndRunningTimeAreJudgedApart(t *testing.T) { if res, _ = runLevelTool(t, m, states, "integration"); !res.Passed || res.OverBound || res.Setup != 29 || res.Elapsed != 59 { t.Errorf("29 s of setup and 59 s of running are within both bounds: %+v", res) } + + // Design 47 H2 H-A-488: with four steps in hand at the step's start, the bounds are four times their defaults: the + // tests get 240 s as their level's bound, the 35 s setup is within its 120 s, and the result names both. + k := newIntegFake(t, "slow") + k.advance["compile"], k.advance["start"] = 20*time.Second, 15*time.Second + m, states = integModule(t, k) + m.inHand = func() int { return 4 } + res, _ = runLevelTool(t, m, states, "integration") + if !res.Passed || res.OverBound || res.Setup != 35 || res.Bound != 240 || res.SetupBound != 120 { + t.Errorf("a setup of 35 s with four steps in hand: %+v", res) + } + for _, c := range k.calls { + if strings.Contains(c.line, " -json ") && !strings.Contains(strings.Join(c.env, " "), "MESH_TEST_LEVEL_BOUND=4m0s") { + t.Errorf("%s ran without the level's bound of 240 s: %v", c.line, c.env) + } + } } // 1486/AC5: a run that fails, and a run that panics: the store is stopped, the bus is stopped, the run's directory is @@ -290,7 +306,7 @@ func TestAFailedOrPanickedRunStopsTheServersAndRemovesTheRunDirectory(t *testing m, _ = integModule(t, g) func() { defer func() { recover() }() - m.integration(context.Background(), "ws1", filepath.Join(m.Work, "ws1", "tree"), nil, "") + m.integration(context.Background(), "ws1", filepath.Join(m.Work, "ws1", "tree"), nil, "", m.integrationBounds()) }() checkCleaned(g, m) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/level.go b/modules/mesh-build-go/cmd/mesh-build-go/level.go index 1c50743f4..5a1e66b3e 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/level.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/level.go @@ -65,13 +65,12 @@ func (m *Module) executable() (string, error) { return filepath.EvalSymlinks(exe) } -// integrationShutOff runs the integration level in one shut-off process of this program and reads its answer. -func (m *Module) integrationShutOff(ctx context.Context, workspace string, packages []string, pattern string) (*TestResult, error) { +// integrationShutOff runs the integration level, judged by b, in one shut-off process of this program and reads its answer. +func (m *Module) integrationShutOff(ctx context.Context, workspace string, packages []string, pattern string, b Bounds) (*TestResult, error) { exe, err := m.executable() if err != nil { return nil, fmt.Errorf("the integration level's program: %w", err) } - b := m.integrationBounds() a, err := m.shutOff(ctx, exe, levelSpec{Step: stepIntegration, Workspace: workspace, Packages: packages, Run: pattern, Setup: b.Setup, Bound: b.Run}) if err != nil { return nil, err @@ -239,7 +238,7 @@ func (m *Module) answer(ctx context.Context, spec levelSpec) levelAnswer { if err != nil { return levelAnswer{Error: err.Error()} } - res, err := m.integrationHere(ctx, spec.Workspace, tree, spec.Packages, spec.Run, major) + res, err := m.integrationHere(ctx, spec.Workspace, tree, spec.Packages, spec.Run, major, m.integrationBounds()) if err != nil { return levelAnswer{Error: err.Error()} } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/levels.go b/modules/mesh-build-go/cmd/mesh-build-go/levels.go index d002de777..13662ffe9 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/levels.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/levels.go @@ -32,6 +32,11 @@ type Bounds struct { Setup, Run time.Duration } +// times is the bounds of a level run while n steps are in hand (design 47 H2 H-A-488): each its default n times. +func (b Bounds) times(n int) Bounds { + return Bounds{Setup: b.Setup * time.Duration(n), Run: b.Run * time.Duration(n)} +} + // PackageResult, TestTime and Failure are the SDK's failure contract (buildstep, issue 1901), with the JSON names this // module always sent: a Failure names its test, its file, its cause (never empty, never a passing line) and its own // output up to 4096 bytes, cause first. diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module.go b/modules/mesh-build-go/cmd/mesh-build-go/module.go index 150ab04e5..949698e3b 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/module.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/module.go @@ -110,6 +110,8 @@ type Module struct { // when set, answers it instead (a test). Settings string Capacity func() int + // inHand answers how many steps the seat has in hand in place of Held's (a test's; design 47 H2 H-A-488). + inHand func() int // SayStep says a pipeline step's end (steprun.AskSteps.SayStep); the seat's publish on this node when nil. SayStep func(sdkstep.StepEnded) error // ReadEntry asks the seat mesh-catalog's verb entry for the catalog graph entry of one fingerprint, which a compose ask's diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test.go b/modules/mesh-build-go/cmd/mesh-build-go/test.go index 543dd5222..acc11705c 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test.go @@ -60,6 +60,21 @@ func (m *Module) unitBound() time.Duration { return UnitBound } +// stepsInHand is how many steps the seat has in hand at a step's start, this one counted, at least 1 (design 47 H2 +// H-A-488): a level's bounds are their defaults that many times, so a level set up slowly on a build node running +// several steps at once is not a failure. In the shut-off process, which holds no puller, it is 1: its spec carries +// the bounds already multiplied. +func (m *Module) stepsInHand() int { + n := 0 + switch { + case m.inHand != nil: + n = m.inHand() + case m.Held != nil: + n = m.Held.InHand() + } + return max(1, n) +} + func (m *Module) test(ctx context.Context, args map[string]any) (any, error) { workspace, _ := args["workspace"].(string) level, _ := args["level"].(string) @@ -93,13 +108,15 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) { } } var res *TestResult + // The level's bounds, read once at the step's start (H-A-488): the result's bound and setup_bound name them. + inHand := m.stepsInHand() switch { case runner == RunnerWorkspace && (level == LevelUnit || level == LevelIntegration): - return m.testAsCandidate(ctx, workspace, level, packages, pattern, names) + return m.testAsCandidate(ctx, workspace, level, packages, pattern, names, inHand) case level == LevelUnit: - res, err = m.runLevel(ctx, tree, levelOpts{workspace: workspace, level: LevelUnit, packages: packages, run: pattern, bounds: Bounds{Run: m.unitBound()}, env: m.checkEnv(workspace, names)}) + res, err = m.runLevel(ctx, tree, levelOpts{workspace: workspace, level: LevelUnit, packages: packages, run: pattern, bounds: Bounds{Run: m.unitBound()}.times(inHand), env: m.checkEnv(workspace, names)}) case level == LevelIntegration: - res, err = m.integration(ctx, workspace, tree, packages, pattern) + res, err = m.integration(ctx, workspace, tree, packages, pattern, m.integrationBounds().times(inHand)) default: return nil, fmt.Errorf("the level %q is neither unit nor integration", level) } @@ -148,13 +165,13 @@ func compareGenerated(tree string, res *TestResult) error { // testAsCandidate runs a level as the candidate's program (ADR 0535 §1): the integration level is refused first as the // installed program would refuse it (no postgres), then the whole level runs in the candidate's one shut-off process. -func (m *Module) testAsCandidate(ctx context.Context, workspace, level string, packages []string, pattern string, names []string) (*TestResult, error) { - spec := levelSpec{Step: stepUnit, Workspace: workspace, Packages: packages, Run: pattern, Bound: m.unitBound(), NodeNames: names} +func (m *Module) testAsCandidate(ctx context.Context, workspace, level string, packages []string, pattern string, names []string, inHand int) (*TestResult, error) { + spec := levelSpec{Step: stepUnit, Workspace: workspace, Packages: packages, Run: pattern, Bound: m.unitBound() * time.Duration(inHand), NodeNames: names} if level == LevelIntegration { if _, err := m.checkPostgres(ctx); err != nil { return nil, err // a *steprun.Refusal: this level only } - b := m.integrationBounds() + b := m.integrationBounds().times(inHand) spec = levelSpec{Step: stepIntegration, Workspace: workspace, Packages: packages, Run: pattern, Setup: b.Setup, Bound: b.Run, NodeNames: names} } a, judged, err := m.asCandidate(ctx, spec) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test_real_integration_test.go b/modules/mesh-build-go/cmd/mesh-build-go/test_real_integration_test.go index 6cc6ae493..f5543591d 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test_real_integration_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test_real_integration_test.go @@ -166,7 +166,7 @@ func TestIntegrationLevelRunsAgainstARealStoreAndBusAndLeavesNothing(t *testing. home := t.TempDir() m := &Module{Recipe: Current(), Home: home, Work: work, Cache: t.TempDir()} m.Run = StepRunner(home) - res, err := m.integration(context.Background(), "ws1", filepath.Join(work, "ws1", "tree"), nil, "") + res, err := m.integration(context.Background(), "ws1", filepath.Join(work, "ws1", "tree"), nil, "", m.integrationBounds()) if err != nil { t.Fatalf("the integration level: %v", err) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test_test.go b/modules/mesh-build-go/cmd/mesh-build-go/test_test.go index b3fe63370..84bb0856e 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test_test.go @@ -269,25 +269,35 @@ func TestUnitLevelOfAPackageThatDoesNotBuildFailsWithTheCompilersLine(t *testing } // The command is go test -json in the module's directory of the workspace, with the toolchain pinned to the installed -// one and the module's caches. +// one and the module's caches; the level's bound is its default times the steps in hand at the step's start, at least +// once (design 47 H2 H-A-488), and the result names it. func TestUnitLevelRunsGoTestJsonInTheModulesDirectoryWithTheStepsEnvironment(t *testing.T) { - m, states, ws, calls := workspaceFor(t, "empty") - runUnit(t, m, states, ws) - var line string - var env string - for _, c := range *calls { - if strings.HasPrefix(c.line, "go ") && strings.Contains(c.line, " test ") { - line, env = c.line, strings.Join(c.env, " ") + for _, row := range []struct { + inHand int + bound time.Duration + }{{0, UnitBound}, {1, UnitBound}, {4, 4 * UnitBound}} { + m, states, ws, calls := workspaceFor(t, "empty") + m.inHand = func() int { return row.inHand } + res, _ := runUnit(t, m, states, ws) + var line string + var env string + for _, c := range *calls { + if strings.HasPrefix(c.line, "go ") && strings.Contains(c.line, " test ") { + line, env = c.line, strings.Join(c.env, " ") + } } - } - want := "go -C " + filepath.Join(m.Work, ws, "tree", ".") + " test -json -count=1 -timeout 10m ./..." - if line != want { - t.Errorf("command = %q, want %q", line, want) - } - // Design 47 H2 §800: the level's bound reaches the tests as MESH_TEST_LEVEL_BOUND, a duration. - for _, w := range []string{"GOTOOLCHAIN=local", "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), "MESH_TEST_LEVEL_BOUND=" + m.unitBound().String()} { - if !strings.Contains(env, w) { - t.Errorf("environment %s lacks %s", env, w) + want := "go -C " + filepath.Join(m.Work, ws, "tree", ".") + " test -json -count=1 -timeout 10m ./..." + if line != want { + t.Errorf("command = %q, want %q", line, want) + } + // Design 47 H2 §800: the level's bound reaches the tests as MESH_TEST_LEVEL_BOUND, a duration. + for _, w := range []string{"GOTOOLCHAIN=local", "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), "MESH_TEST_LEVEL_BOUND=" + row.bound.String()} { + if !strings.Contains(env, w) { + t.Errorf("%d in hand: environment %s lacks %s", row.inHand, env, w) + } + } + if res.Bound != row.bound.Seconds() { + t.Errorf("%d in hand: the result's bound is %v s, want %v", row.inHand, res.Bound, row.bound.Seconds()) } } } From 5326fb61426fa60d62a44e7604fbaa9b3c70d1f1 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 19:43:42 +0100 Subject: [PATCH 2/2] Node build module: a test level's bounds follow the steps in hand (H-A-488) The Node build module judged its levels by the same fixed bounds, and its integration setup took 40 s against 30 s while three nodes' checks ran. The rule is the kind's, not the Go module's: its bounds, and the MESH_TEST_LEVEL_BOUND its scripts get, are now their defaults times the steps the seat has in hand at the step's start (at least 1), and the result names them (bound, setup_bound). Builds: design 47 H2 H-A-488, H-A-488 addendum --- .../cmd/mesh-build-node/check.go | 43 +++++++++++++------ .../cmd/mesh-build-node/check_test.go | 11 +++++ .../cmd/mesh-build-node/module.go | 2 + 3 files changed, 43 insertions(+), 13 deletions(-) diff --git a/modules/mesh-build-node/cmd/mesh-build-node/check.go b/modules/mesh-build-node/cmd/mesh-build-node/check.go index 8f4a6ec57..61f739b96 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/check.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/check.go @@ -37,16 +37,30 @@ const ( // test helpers take a setup bound of half of it. const LevelBoundEnv = "MESH_TEST_LEVEL_BOUND" -// levelBound is a level's bound. -func levelBound(level string) time.Duration { +// levelBound is a level's bound while inHand steps are in hand (design 47 H2 H-A-488): its default that many times. +func levelBound(level string, inHand int) time.Duration { if level == "unit" { - return UnitBound + return UnitBound * time.Duration(inHand) } - return IntegrationRunBound + return IntegrationRunBound * time.Duration(inHand) } // integrationSetupBound is the integration level's setup bound: half of its bound (§800). -func integrationSetupBound() time.Duration { return IntegrationRunBound / 2 } +func integrationSetupBound(inHand int) time.Duration { return levelBound("integration", inHand) / 2 } + +// stepsInHand is how many steps the seat has in hand at a step's start, this one counted, at least 1 (design 47 H2 +// H-A-488): a level's bounds are their defaults that many times, so a level set up slowly on a build node running +// several steps at once is not a failure. +func (m *Module) stepsInHand() int { + n := 0 + switch { + case m.inHand != nil: + n = m.inHand() + case m.Held != nil: + n = m.Held.InHand() + } + return max(1, n) +} // TestResult is what a test step passes with, with the field names of mesh-build-go's (issue 1290). The step ends // passed whenever the level ran, whatever its verdict, and Passed is the verdict; it ends failed only when the tests @@ -146,6 +160,8 @@ func (m *Module) testLevel(ctx context.Context, args map[string]any) (*TestResul return nil, err } res := &TestResult{Workspace: workspace, Module: module, Level: level, By: sdkstep.By("mesh-build-node")} + // The level's bounds, read once at the step's start (H-A-488): the result's bound and setup_bound name them. + inHand := m.stepsInHand() script := "test" if level == "integration" { script = "test:integration" @@ -213,7 +229,7 @@ func (m *Module) testLevel(ctx context.Context, args map[string]any) (*TestResul report := filepath.Join(root, ".mesh-node-test-"+level+".tap") _ = os.Remove(report) env := []string{"PATH=" + binDir(root) + ":" + pathOrDefault(), "npm_config_cache=" + filepath.Join(m.Toolchain.Paths.Cache, "npm"), - "NODE_OPTIONS=" + steprun.NodeTestOptions(report), LevelBoundEnv + "=" + levelBound(level).String()} + "NODE_OPTIONS=" + steprun.NodeTestOptions(report), LevelBoundEnv + "=" + levelBound(level, inHand).String()} r := m.runIn(ctx, root, false, tg.dir, env, "npm", scriptArgs(script)...) run += m.now().Sub(start) tap, _ := os.ReadFile(report) @@ -238,19 +254,20 @@ func (m *Module) testLevel(ctx context.Context, args map[string]any) (*TestResul why = append(why, failureWhy(f)) } if level == "unit" { - res.Elapsed, res.Bound = (check + run).Seconds(), UnitBound.Seconds() - res.OverBound = check+run > UnitBound + bound := levelBound(level, inHand) + res.Elapsed, res.Bound = (check + run).Seconds(), bound.Seconds() + res.OverBound = check+run > bound if res.OverBound { - why = append(why, fmt.Sprintf("the unit level took %.2f s, over its bound of %s", res.Elapsed, UnitBound)) + why = append(why, fmt.Sprintf("the unit level took %.2f s, over its bound of %s", res.Elapsed, bound)) } } else { res.Setup, res.Elapsed = check.Seconds(), run.Seconds() - setup := integrationSetupBound() - res.SetupBound, res.Bound = setup.Seconds(), IntegrationRunBound.Seconds() - res.OverBound = check > setup || run > IntegrationRunBound + setup, bound := integrationSetupBound(inHand), levelBound(level, inHand) + res.SetupBound, res.Bound = setup.Seconds(), bound.Seconds() + res.OverBound = check > setup || run > bound if res.OverBound { why = append(why, fmt.Sprintf("the integration level took %.2f s of setup and %.2f s of running, over its bounds of %s and %s", - res.Setup, res.Elapsed, setup, IntegrationRunBound)) + res.Setup, res.Elapsed, setup, bound)) } } res.Passed = len(failed) == 0 diff --git a/modules/mesh-build-node/cmd/mesh-build-node/check_test.go b/modules/mesh-build-node/cmd/mesh-build-node/check_test.go index 144e38a0f..61a196003 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/check_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/check_test.go @@ -140,6 +140,17 @@ func TestTheIntegrationLevelRunsItsScriptAndReportsSetupAndRunningApart(t *testi if !out.(*TestResult).OverBound { t.Errorf("61 s of running is within the bound") } + // Design 47 H2 H-A-488: with four steps in hand at the step's start, the bounds are four times their defaults: the + // script gets 240 s as its level's bound, 40 s of setup is within its 120 s, and the result names both. + f.cost["tsc"], f.cost["npm"] = 40*time.Second, 61*time.Second + m.inHand = func() int { return 4 } + out, _ = m.test(context.Background(), testArgs("integration")) + if r := out.(*TestResult); r.OverBound || r.Bound != 240 || r.SetupBound != 120 { + t.Errorf("40 s of setup and 61 s of running with four steps in hand: %+v", r) + } + if last := f.calls[len(f.calls)-1]; !slices.Contains(last.env, "MESH_TEST_LEVEL_BOUND=4m0s") { + t.Errorf("with four steps in hand the integration script ran with %v, not MESH_TEST_LEVEL_BOUND=4m0s", last.env) + } } // The commands of a step get an environment built from scratch: no word of this process's, the level's bound alone (§800). diff --git a/modules/mesh-build-node/cmd/mesh-build-node/module.go b/modules/mesh-build-node/cmd/mesh-build-node/module.go index b121d41fd..3104d7317 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/module.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/module.go @@ -107,6 +107,8 @@ type Module struct { // when set, answers it instead (a test). Settings string Capacity func() int + // inHand answers how many steps the seat has in hand in place of Held's (a test's; design 47 H2 H-A-488). + inHand func() int spacesOnce sync.Once workspaces *steprun.Workspaces