From 47dbf87e330aad52c43059c4e9b01efd3a956ff6 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 23 Oct 2026 14:02:59 +0200 Subject: [PATCH 01/10] Fold the siblings into a package digest only when its tests reach MESH_CHECK_BESIDE, so a sibling move keeps unrelated verdicts (issue 1896) --- .../cmd/mesh-build-go/beside_digest_test.go | 69 ++++++++++++++----- .../mesh-build-go/cmd/mesh-build-go/digest.go | 31 +++++++-- .../cmd/mesh-build-go/digest_test.go | 2 +- 3 files changed, 77 insertions(+), 25 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go index 9f14bd9c..3682a61b 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go @@ -11,37 +11,72 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" ) -// The lead's decision L5 (the checker keeps a verdict of a core repository): the fetch writes the siblings it placed -// beside the workspace, and the digest folds each one's repository and commit into every package's digest and echoes -// them, so a verdict is kept only for the siblings it was judged with. -func TestTheSiblingsCommitsAreInEveryDigestAndEchoed(t *testing.T) { +// writeSiblings records the siblings the fetch placed, as the fetch does. +func writeSiblings(t *testing.T, m *Module, placed []Sibling) { + t.Helper() + raw, _ := json.Marshal(placed) + if err := os.WriteFile(filepath.Join(m.Work, "ws1", besideRecord), raw, 0o600); err != nil { + t.Fatal(err) + } +} + +// Issue 1896/AC1 and AC3 (ADR 0536 §1): a sibling's commit changes no digest of a package whose tests reach no file +// naming MESH_CHECK_BESIDE; the answer still echoes every placed sibling, in form 4. +func TestASiblingsCommitLeavesAPackageThatDoesNotReadTheSiblingsAsItWas(t *testing.T) { t.Parallel() m, _ := digestFixture(t) alone := digests(t, m, "unit") if len(alone.Beside) != 0 { t.Errorf("a workspace without siblings echoes %v", alone.Beside) } - write := func(placed []Sibling) { - raw, _ := json.Marshal(placed) - if err := os.WriteFile(filepath.Join(m.Work, "ws1", besideRecord), raw, 0o600); err != nil { - t.Fatal(err) - } - } other := strings.Repeat("b", 40) - write([]Sibling{{Repository: "novox/mesh-sdk", Commit: commitSHA, Dir: "mesh-sdk"}, {Repository: "novox/mesh-catalog", Commit: commitSHA, Dir: "mesh-catalog"}}) + writeSiblings(t, m, []Sibling{{Repository: "novox/mesh-sdk", Commit: commitSHA, Dir: "mesh-sdk"}, {Repository: "novox/mesh-catalog", Commit: commitSHA, Dir: "mesh-catalog"}}) with := digests(t, m, "unit") want := []string{"novox/mesh-catalog " + commitSHA, "novox/mesh-sdk " + commitSHA} if !slices.Equal(with.Beside, want) { t.Errorf("echoed %v, want %v", with.Beside, want) } - write([]Sibling{{Repository: "novox/mesh-sdk", Commit: other, Dir: "mesh-sdk"}, {Repository: "novox/mesh-catalog", Commit: commitSHA, Dir: "mesh-catalog"}}) + writeSiblings(t, m, []Sibling{{Repository: "novox/mesh-sdk", Commit: other, Dir: "mesh-sdk"}, {Repository: "novox/mesh-catalog", Commit: commitSHA, Dir: "mesh-catalog"}}) moved := digests(t, m, "unit") for p, d := range alone.Packages { - if with.Packages[p] == d { - t.Errorf("%s: the siblings left the digest as it was without them", p) + if with.Packages[p] != d || moved.Packages[p] != d { + t.Errorf("%s: a sibling left its digest changed though no test of it reads the siblings", p) } - if moved.Packages[p] == with.Packages[p] { - t.Errorf("%s: a sibling's new commit left the digest as it was", p) + } + if !slices.Equal(moved.Beside, []string{"novox/mesh-catalog " + commitSHA, "novox/mesh-sdk " + other}) { + t.Errorf("echoed %v after the move", moved.Beside) + } +} + +// Issue 1896/AC2: a package whose test imports an in-tree package naming MESH_CHECK_BESIDE is judged again when a +// sibling moves; a package beside it that does not import it is not. +func TestASiblingsCommitChangesTheDigestOfAPackageWhoseTestsReachAReaderOfTheSiblings(t *testing.T) { + t.Parallel() + m, tree := digestFixture(t) + for name, text := range map[string]string{ + "f/f.go": "package f\n\n// Beside is where the siblings lie.\nconst Beside = \"MESH_CHECK_BESIDE\"\n", + "e/e_test.go": "package e\n\nimport (\n\t\"testing\"\n\n\t\"fixture/digest/f\"\n)\n\nfunc TestE(t *testing.T) {\n\tif f.Beside == \"\" {\n\t\tt.Fatal(\"e\")\n\t}\n}\n", + "e/e.go": "package e\n", + "g/g_test.go": "package g\n\nimport \"testing\"\n\nfunc TestG(t *testing.T) {\n\t// MESH_CHECK_BESIDE named in the test's own file\n}\n", + "g/g.go": "package g\n", + } { + if err := os.MkdirAll(filepath.Dir(filepath.Join(tree, name)), 0o755); err != nil { + t.Fatal(err) + } + edit(t, tree, name, text) + } + writeSiblings(t, m, []Sibling{{Repository: "novox/mesh-sdk", Commit: commitSHA, Dir: "mesh-sdk"}}) + before := digests(t, m, "unit") + writeSiblings(t, m, []Sibling{{Repository: "novox/mesh-sdk", Commit: strings.Repeat("b", 40), Dir: "mesh-sdk"}}) + after := digests(t, m, "unit") + for _, p := range []string{"fixture/digest/e", "fixture/digest/g"} { + if len(before.Packages[p]) != 64 || before.Packages[p] == after.Packages[p] { + t.Errorf("%s reads the siblings and its digest stayed (%q, %q)", p, before.Packages[p], after.Packages[p]) + } + } + for _, p := range []string{"fixture/digest/a", "fixture/digest/c"} { + if before.Packages[p] != after.Packages[p] { + t.Errorf("%s does not read the siblings and its digest changed", p) } } } @@ -81,7 +116,7 @@ func TestTheIntegrationLevelDigestsUnderBothTags(t *testing.T) { t.Errorf("the integration digest listed %q", s.command) } } - if DigestVersion != "3" { + if DigestVersion != "4" { t.Errorf("the digest's form changed in kind and is still version %s", DigestVersion) } } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest.go b/modules/mesh-build-go/cmd/mesh-build-go/digest.go index 55412464..f3e2ed53 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/digest.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest.go @@ -5,6 +5,7 @@ package main // digest is unchanged is not run again. The tool answers in its call (it is short, ADR 0469 §2). import ( + "bytes" "context" "crypto/sha256" "encoding/hex" @@ -22,7 +23,12 @@ import ( // DigestVersion is the form of the digest's text: changed whenever what the digest hashes changes in kind. // 2: no Go version in the text; the toolchain enters a fingerprint as the agreed version (ADR 0477 §3). 3: the siblings' // commits (the lead's decision L5) and the integration level's tags integration,isolation (the checker's agreement). -const DigestVersion = "3" +// 4: the siblings' commits enter only the digest of a package whose tests reach a file naming MESH_CHECK_BESIDE (ADR 0536 §1). +const DigestVersion = "4" + +// besideVariable is the name a file bears when it reads where the siblings lie; a package whose tests reach such a file +// depends on the siblings' commits. +const besideVariable = "MESH_CHECK_BESIDE" // integrationTags are the build tags of the integration level, its run and its digest alike: integration, and isolation // for the bubblewrap proofs (mesh-sdk's and the build modules'), which a build node can run. @@ -132,8 +138,9 @@ func (m *Module) digest(ctx context.Context, args map[string]any) (any, error) { if err != nil { return nil, err } + besideHeader := make([]string, 0, len(beside)) for _, b := range beside { - header = append(header, "beside "+b) + besideHeader = append(besideHeader, "beside "+b) } dirs, err := m.goModDirs(ctx, tree) @@ -146,7 +153,7 @@ func (m *Module) digest(ctx context.Context, args map[string]any) (any, error) { if err != nil { return nil, err } - if err := digestModule(&res, pkgs, tree, header); err != nil { + if err := digestModule(&res, pkgs, tree, header, besideHeader); err != nil { return nil, err } } @@ -190,7 +197,7 @@ func parsePackages(out string) (map[string]*goPackage, error) { // digestModule adds to res the digest of each root package of one go.mod directory. A root is a package of the main // module that is listed, not only a dependency, and not a variant of another. -func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, header []string) error { +func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, header, beside []string) error { roots := []string{} for path, p := range pkgs { if p.ForTest == "" && !p.DepOnly && !p.Standard && !strings.HasSuffix(path, ".test") && p.Module != nil && p.Module.Main { @@ -208,7 +215,7 @@ func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, he if testMain == nil { return fmt.Errorf("go list gave no test binary for %s", path) } - d, err := packageDigest(root, testMain, pkgs, tree, header) + d, err := packageDigest(root, testMain, pkgs, tree, header, beside) if err != nil { return fmt.Errorf("%s: %w", path, err) } @@ -219,8 +226,9 @@ func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, he // packageDigest is the digest of one package: a hash over a sorted text of the header, the files of every package its // tests reach in the module's own tree (by their content, under their path in the tree), the version and checksum of -// every module outside it, and the package's own test data. A package of the standard library is the Go version's. -func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree string, header []string) (string, error) { +// every module outside it, and the package's own test data. The beside lines (the siblings' commits) join the header +// only when a file the tests reach names MESH_CHECK_BESIDE. A package of the standard library is the Go version's. +func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree string, header, beside []string) (string, error) { reached := map[string]*goPackage{root.ImportPath: root} for _, dep := range testMain.Deps { if p := pkgs[dep]; p != nil { @@ -228,6 +236,7 @@ func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree s } } lines := []string{} + readsBeside := false for _, p := range reached { if p.Standard || p.Module == nil { continue @@ -246,6 +255,11 @@ func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree s return "", err } lines = append(lines, l) + if !readsBeside { + if raw, err := os.ReadFile(filepath.Join(p.Dir, f)); err == nil && bytes.Contains(raw, []byte(besideVariable)) { + readsBeside = true + } + } } } data := filepath.Join(root.Dir, "testdata") @@ -263,6 +277,9 @@ func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree s } } sort.Strings(lines) + if readsBeside { + header = append(append([]string{}, header...), beside...) + } h := sha256.New() for _, l := range append(append([]string{}, header...), lines...) { h.Write([]byte(l + "\n")) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go index ff1e8e6e..5e2d81d0 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go @@ -173,7 +173,7 @@ func TestAModuleOutsideTheTreeCountsByVersionAndChecksum(t *testing.T) { testMain := &goPackage{ImportPath: "x/r.test", Deps: []string{ext.ImportPath}} pkgs[root.ImportPath] = root digest := func() string { - d, err := packageDigest(root, testMain, pkgs, dir, []string{"digest 1"}) + d, err := packageDigest(root, testMain, pkgs, dir, []string{"digest 1"}, nil) if err != nil { t.Fatal(err) } From 547f4ee27d441c25f3af9cc2efc7e883271d00f6 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 23 Oct 2026 14:04:43 +0200 Subject: [PATCH 02/10] Compile and run only the go.mod directories a selection lies in, and run them side by side up to a bound (issue 890, ADR 0536 s3) --- .../cmd/mesh-build-go/integration.go | 10 +- .../mesh-build-go/integration_level_test.go | 10 + .../mesh-build-go/cmd/mesh-build-go/module.go | 3 + .../cmd/mesh-build-go/selected_dirs_test.go | 172 ++++++++++++++++++ .../mesh-build-go/cmd/mesh-build-go/test.go | 94 ++++++++-- 5 files changed, 271 insertions(+), 18 deletions(-) create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/selected_dirs_test.go diff --git a/modules/mesh-build-go/cmd/mesh-build-go/integration.go b/modules/mesh-build-go/cmd/mesh-build-go/integration.go index 03de39ab..231714a0 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/integration.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/integration.go @@ -75,8 +75,16 @@ func (m *Module) integrationHere(ctx context.Context, workspace, tree string, pa env := append([]string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats}, m.besideEnv(workspace)...) // Compiling counts as setup: the tests are built once, here, with the addresses a TestMain may read, and the run // below finds them built. Whether they compile is for the run to say; its failure is not read here. + // Only the directories the selected packages lie in compile (ADR 0536 §3); no selection is every directory. + byDir, err := packagesByDir(tree, dirs, packages) + if err != nil { + return nil, err + } for _, d := range dirs { - m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), env...), "go", "-C", filepath.Join(tree, d), "test", "-tags", strings.Join(integrationTags, ","), "-run", "^$", "-count=1", "./...") + if len(byDir[d]) == 0 { + continue + } + m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), env...), "go", append([]string{"-C", filepath.Join(tree, d), "test", "-tags", strings.Join(integrationTags, ","), "-run", "^$", "-count=1"}, byDir[d]...)...) } setup := m.clock().Sub(began) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go b/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go index 9d2aafa1..fafb7bda 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go @@ -31,6 +31,10 @@ type integFake struct { busDown bool runDirs []string // run directories seen when the store was started, to be gone at the end panicOn string // a phase that panics + // gomods is what git ls-files answers ("go.mod\n" when empty); onTest is called, unlocked, as each go test -json + // command runs, with its line. + gomods string + onTest func(line string) } func newIntegFake(t *testing.T, captured string) *integFake { @@ -63,10 +67,16 @@ func (f *integFake) run(ctx context.Context, env []string, name string, args ... case name == "postgres": return forgekey.Ran{Stdout: f.postgres + "\n"} case name == "git" && len(args) > 2 && args[2] == "ls-files": + if f.gomods != "" { + return forgekey.Ran{Stdout: f.gomods} + } return forgekey.Ran{Stdout: "go.mod\n"} case name == "go" && strings.Contains(line, "-run ^$"): f.tick("compile") case name == "go": + if f.onTest != nil { + f.onTest(line) + } f.tick("test") status := 0 if strings.Contains(f.captured, `"Action":"fail"`) { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module.go b/modules/mesh-build-go/cmd/mesh-build-go/module.go index fffb3e1f..0f306017 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/module.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/module.go @@ -54,6 +54,9 @@ type Module struct { // Bubblewrap reads the installed bubblewrap's version (ProbeBubblewrap): a step is refused, and the module is // unhealthy, while it is missing or older than BubblewrapLeast. Nil probes nothing. Bubblewrap func(ctx context.Context) (string, error) + // Parallel is how many go.mod directories a level runs at once; zero is half the processors, and at least one + // (ADR 0536 §3: a node runs two checks at once). + Parallel int // Inside is the integration level's own process, started shut off in bubblewrap by a test step (level.go): its // commands run as they are, never shut off a second time. Inside bool diff --git a/modules/mesh-build-go/cmd/mesh-build-go/selected_dirs_test.go b/modules/mesh-build-go/cmd/mesh-build-go/selected_dirs_test.go new file mode 100644 index 00000000..fa86bb54 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/selected_dirs_test.go @@ -0,0 +1,172 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// threeModules gives the workspace three go.mod directories, a, b and c, whose modules are fixture/a, fixture/b and +// fixture/c. +func threeModules(t *testing.T, f *integFake) (*Module, *keptStates) { + t.Helper() + m, states := integModule(t, f) + f.gomods = "a/go.mod\nb/go.mod\nc/go.mod\n" + for _, d := range []string{"a", "b", "c"} { + dir := filepath.Join(m.Work, "ws1", "tree", d) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module fixture/"+d+"\n\ngo 1.25.0\n"), 0o644); err != nil { + t.Fatal(err) + } + } + return m, states +} + +func runIntegrationOf(t *testing.T, m *Module, states *keptStates, packages ...any) (TestResult, delivery.Step) { + t.Helper() + for _, tool := range m.Tools(m.StepTools()) { + if tool.Name != "build_go_test" { + continue + } + args := map[string]any{"workspace": "ws1", "level": "integration"} + if len(packages) > 0 { + args["packages"] = packages + } + if _, err := tool.Run(args); err != nil { + t.Fatal(err) + } + } + st := states.waitEnd(t) + var res TestResult + if st.State == delivery.StepPassed { + if err := json.Unmarshal(st.Result, &res); err != nil { + t.Fatal(err) + } + } + return res, st +} + +// Issue 890/AC1 (ADR 0536 §3): with packages selected in one of three go.mod directories, the integration level +// compiles and runs go test in that directory only. +func TestOnlyTheSelectedDirectoryCompilesAndRuns(t *testing.T) { + f := newIntegFake(t, "slow") + m, states := threeModules(t, f) + if _, st := runIntegrationOf(t, m, states, "fixture/b/x"); st.State != delivery.StepPassed { + t.Fatalf("the step ended %s: %s", st.State, st.Reason) + } + tree := filepath.Join(m.Work, "ws1", "tree") + compiled, ran := 0, 0 + for _, l := range strings.Split(f.lines(), "\n") { + if !strings.HasPrefix(l, "go ") || !strings.Contains(l, " test ") { + continue + } + if !strings.HasPrefix(l, "go -C "+filepath.Join(tree, "b")+" test ") { + t.Errorf("go test ran outside the selected directory: %s", l) + } + if strings.Contains(l, "-run ^$") { + compiled++ + } else { + ran++ + } + } + if compiled != 1 || ran != 1 { + t.Errorf("compiled %d times and ran %d times, want once each:\n%s", compiled, ran, f.lines()) + } +} + +// Issue 890/AC1: no selection keeps every directory, compiled and run. +func TestWithoutASelectionEveryDirectoryCompilesAndRuns(t *testing.T) { + f := newIntegFake(t, "slow") + m, states := threeModules(t, f) + runIntegrationOf(t, m, states) + compiled, ran := 0, 0 + for _, l := range strings.Split(f.lines(), "\n") { + if strings.HasPrefix(l, "go ") && strings.Contains(l, " test ") { + if strings.Contains(l, "-run ^$") { + compiled++ + } else { + ran++ + } + } + } + if compiled != 3 || ran != 3 { + t.Errorf("compiled %d and ran %d, want 3 and 3:\n%s", compiled, ran, f.lines()) + } +} + +// Issue 890/AC2: with two directories selected and a runner that blocks, both go test commands run at once; the +// result lists the packages in directory order. +func TestSelectedDirectoriesRunAtOnceAndAnswerInDirectoryOrder(t *testing.T) { + f := newIntegFake(t, "slow") + var mu sync.Mutex + running, most := 0, 0 + release := make(chan struct{}) + f.onTest = func(line string) { + mu.Lock() + running++ + if running > most { + most = running + } + if running == 2 { + close(release) + } + mu.Unlock() + select { + case <-release: + case <-time.After(10 * time.Second): + } + mu.Lock() + running-- + mu.Unlock() + } + m, states := threeModules(t, f) + m.Parallel = 3 + res, st := runIntegrationOf(t, m, states, "fixture/c/x", "fixture/a/x") + if st.State != delivery.StepPassed { + t.Fatalf("the step ended %s: %s", st.State, st.Reason) + } + if most != 2 { + t.Errorf("at most %d go test commands ran at once, want 2", most) + } + dirs := []string{} + for _, p := range res.Packages { + dirs = append(dirs, p.Dir) + } + if got := fmt.Sprint(dirs); !strings.HasPrefix(got, "[a ") || !strings.HasSuffix(got, " c]") { + t.Errorf("the packages lie in directories %s, want a's before c's", got) + } +} + +// The bound holds: with one directory at a time, no two go test commands run at once. +func TestTheDirectoriesAtOnceNeverPassTheBound(t *testing.T) { + f := newIntegFake(t, "slow") + var mu sync.Mutex + running, most := 0, 0 + f.onTest = func(string) { + mu.Lock() + running++ + if running > most { + most = running + } + mu.Unlock() + time.Sleep(20 * time.Millisecond) + mu.Lock() + running-- + mu.Unlock() + } + m, states := threeModules(t, f) + m.Parallel = 1 + runIntegrationOf(t, m, states) + if most != 1 { + t.Errorf("%d go test commands ran at once with a bound of 1", most) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test.go b/modules/mesh-build-go/cmd/mesh-build-go/test.go index d129c277..5add8055 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test.go @@ -10,8 +10,12 @@ import ( "os" "path/filepath" "regexp" + "runtime" "strings" + "sync" "time" + + "git.novox.be/novox/mesh-sdk/go/forgekey" ) // testTimeout is go test's own timeout for a level: a bound on a test that never ends, far above any level's bound. @@ -143,26 +147,71 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR return nil, err } began := m.clock() + // The directories that hold a selected package run side by side, up to the bound, each into a result of its own; + // the results join in directory order, so the answer does not depend on which ended first (ADR 0536 §3). + type dirRun struct { + res *TestResult + rd *reader + r runAnswer + noise []string + } + var todo []string for _, d := range dirs { - packages := byDir[d] - if len(packages) == 0 { - continue // no package asked for lies in this module + if len(byDir[d]) > 0 { // no package asked for lies in the others + todo = append(todo, d) } - args := []string{"-C", filepath.Join(tree, d), "test", "-json", "-count=1", "-timeout", testTimeout} - if len(o.tags) > 0 { - args = append(args, "-tags", strings.Join(o.tags, ",")) - } - if o.run != "" { - args = append(args, "-run", o.run) - } - args = append(args, packages...) - r := run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), o.env...), "go", args...) - rd := newReader(res, d) - noise := rd.read(r.Stdout) - rd.finish() - if (r.Status != 0 || r.Err != "") && len(rd.pkgs) == 0 { + } + runs := make([]dirRun, len(todo)) + slots := make(chan struct{}, m.parallel()) + var wg sync.WaitGroup + var crash any // a panic of a run goes on in the caller, which cleans up after it + var crashMu sync.Mutex + for i, d := range todo { + wg.Add(1) + slots <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-slots }() + defer func() { + if p := recover(); p != nil { + crashMu.Lock() + crash = p + crashMu.Unlock() + } + }() + args := []string{"-C", filepath.Join(tree, d), "test", "-json", "-count=1", "-timeout", testTimeout} + if len(o.tags) > 0 { + args = append(args, "-tags", strings.Join(o.tags, ",")) + } + if o.run != "" { + args = append(args, "-run", o.run) + } + args = append(args, byDir[d]...) + dr := &runs[i] + dr.res = &TestResult{} + dr.r = run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), o.env...), "go", args...) + dr.rd = newReader(dr.res, d) + dr.noise = dr.rd.read(dr.r.Stdout) + dr.rd.finish() + }() + } + wg.Wait() + if crash != nil { + panic(crash) + } + for i, d := range todo { + dr := runs[i] + if (dr.r.Status != 0 || dr.r.Err != "") && len(dr.rd.pkgs) == 0 { // go test said nothing a test said: it could not run (a go.mod it cannot read, a toolchain it cannot find). - return nil, fmt.Errorf("go test in %s: %s", d, firstLine(r.Err+"\n"+strings.Join(noise, "\n")+"\n"+r.Stderr)) + return nil, fmt.Errorf("go test in %s: %s", d, firstLine(dr.r.Err+"\n"+strings.Join(dr.noise, "\n")+"\n"+dr.r.Stderr)) + } + res.Tests += dr.res.Tests + res.Skipped += dr.res.Skipped + res.Failed = append(res.Failed, dr.res.Failed...) + res.Packages = append(res.Packages, dr.res.Packages...) + res.Slowest = append(res.Slowest, dr.res.Slowest...) + if dr.res.Elapsed > res.Elapsed { + res.Elapsed = dr.res.Elapsed } } if o.wall { @@ -172,6 +221,17 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR return res, nil } +// runAnswer is what a step runner answers. +type runAnswer = forgekey.Ran + +// parallel is how many directories a level runs at once. +func (m *Module) parallel() int { + if m.Parallel > 0 { + return m.Parallel + } + return max(1, runtime.NumCPU()/2) +} + // packagesByDir says which patterns run in which go.mod directory. A relative pattern (./...) runs in every directory, as // it always did; an import path, as build_go_digest keys its packages, runs only in the directory whose module holds it // (the longest module path that is the path or its prefix), as another module's path would fail the step. No pattern From 24cbc8961ef3fc1d380dfe504686d7b88171fb04 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 23 Oct 2026 14:06:42 +0200 Subject: [PATCH 03/10] Fetch a repository and its siblings through a bare mirror in the cache, updated under a lock, so a check transfers only new objects (issue 890, ADR 0536 s2) --- .../mesh-build-go/cmd/mesh-build-go/fetch.go | 2 +- .../mesh-build-go/cmd/mesh-build-go/mirror.go | 59 ++++++++ .../cmd/mesh-build-go/mirror_test.go | 134 ++++++++++++++++++ .../cmd/mesh-build-go/preview.go | 2 +- modules/mesh-build-go/go.mod | 2 +- modules/mesh-build-go/go.sum | 2 + 6 files changed, 198 insertions(+), 3 deletions(-) create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/mirror.go create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/mirror_test.go diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index 435def4d..b8392a58 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -114,7 +114,7 @@ func (m *Module) fetchInto(ctx context.Context, req buildstep.FetchRequest, id s return nil, err } remote := m.remote(repository) - if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref, m.FetchOptions...); err != nil { + if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref, m.fetchOptions(ctx, repository, commit, ref)...); err != nil { return nil, err } res := FetchResult{Workspace: id, Commit: commit} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/mirror.go b/modules/mesh-build-go/cmd/mesh-build-go/mirror.go new file mode 100644 index 00000000..555e58c6 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/mirror.go @@ -0,0 +1,59 @@ +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + "syscall" + + "git.novox.be/novox/mesh-sdk/go/buildstep" +) + +// mirrorLocks hold one mutex per mirror of this process; the lock file beside the mirror holds off another process. +var mirrorLocks sync.Map + +// mirrorDir is where the bare mirror of a repository lies: one per repository in the module's cache (ADR 0536 §2). +func (m *Module) mirrorDir(repository string) string { + return filepath.Join(m.Cache, "mirror", filepath.FromSlash(repository)+".git") +} + +// fetchOptions are the options a fetch of repository runs with: the module's own, and the mirror of the repository +// brought to the commit, so git transfers only the objects the mirror lacks. A mirror that cannot be brought to the +// commit is said and left out: the fetch is then the full one, as it was. +func (m *Module) fetchOptions(ctx context.Context, repository, commit, ref string) []buildstep.Option { + opts := m.FetchOptions + if m.Cache == "" || !repositoryOK.MatchString(repository) { + return opts + } + mirror := m.mirrorDir(repository) + if err := m.updateMirror(ctx, mirror, m.remote(repository), commit, ref); err != nil { + if m.Say != nil { + m.Say("the mirror of %s is not used: %v", repository, err) + } + return opts + } + return append(append([]buildstep.Option{}, opts...), buildstep.WithMirror(mirror)) +} + +// updateMirror brings the mirror to the commit under the mirror's lock: a mutex for the checks of this process, a lock +// file for another process of the node. +func (m *Module) updateMirror(ctx context.Context, mirror, remote, commit, ref string) error { + mu, _ := mirrorLocks.LoadOrStore(mirror, &sync.Mutex{}) + mu.(*sync.Mutex).Lock() + defer mu.(*sync.Mutex).Unlock() + if err := os.MkdirAll(mirror, 0o700); err != nil { // git init --bare takes an empty directory + return err + } + lock, err := os.OpenFile(mirror+".lock", os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return err + } + defer lock.Close() + if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil { + return fmt.Errorf("lock %s: %w", mirror, err) + } + defer syscall.Flock(int(lock.Fd()), syscall.LOCK_UN) + return buildstep.UpdateMirror(ctx, m.Run, mirror, remote, commit, ref, m.FetchOptions...) +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/mirror_test.go b/modules/mesh-build-go/cmd/mesh-build-go/mirror_test.go new file mode 100644 index 00000000..dabf67eb --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/mirror_test.go @@ -0,0 +1,134 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +func fetchOnce(t *testing.T, m *Module, states *keptStates, args map[string]any) FetchResult { + t.Helper() + if _, err := fetchTool(t, m)(args); err != nil { + t.Fatal(err) + } + st := states.waitEnd(t) + if st.State != delivery.StepPassed { + t.Fatalf("the step ended %s: %s", st.State, st.Reason) + } + var res FetchResult + if err := json.Unmarshal(st.Result, &res); err != nil { + t.Fatal(err) + } + return res +} + +// Issue 890/AC4 (ADR 0536 §2): two fetches of one repository on one module bring the same mirror to their commit, and +// each workspace's alternates file names the mirror's objects. +func TestTwoFetchesOfARepositoryUseOneMirrorAndReferenceIt(t *testing.T) { + f := &fakeStep{gomods: "go.mod\n"} + m, states := fetchModule(t, f) + mirror := filepath.Join(m.Cache, "mirror", "novox", "mesh-sdk.git") + var workspaces []string + for i := 0; i < 2; i++ { + workspaces = append(workspaces, fetchOnce(t, m, states, fetchArgs).Workspace) + } + updates := 0 + for _, l := range f.lines() { + if strings.Contains(l, "-C "+mirror+" fetch --no-tags origin "+commitSHA) { + updates++ + } + if strings.Contains(l, "mirror") && !strings.Contains(l, mirror) { + t.Errorf("a command names another mirror: %s", l) + } + } + if updates != 2 { + t.Errorf("the mirror %s was updated %d times, want 2:\n%s", mirror, updates, strings.Join(f.lines(), "\n")) + } + for _, ws := range workspaces { + raw, err := os.ReadFile(filepath.Join(m.Work, ws, "tree", ".git", "objects", "info", "alternates")) + if err != nil || strings.TrimSpace(string(raw)) != filepath.Join(mirror, "objects") { + t.Errorf("the alternates of %s are %q (%v)", ws, raw, err) + } + } +} + +// A sibling is placed through its own repository's mirror. +func TestASiblingIsPlacedThroughItsOwnMirror(t *testing.T) { + f := &fakeStep{gomods: "go.mod\n"} + m, states := fetchModule(t, f) + args := map[string]any{"repository": "novox/mesh-controller", "commit": commitSHA, "beside": []any{ + map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA}}} + ws := fetchOnce(t, m, states, args).Workspace + for _, repo := range []string{"mesh-controller", "mesh-sdk"} { + mirror := filepath.Join(m.Cache, "mirror", "novox", repo+".git") + if !strings.Contains(strings.Join(f.lines(), "\n"), "-C "+mirror+" fetch --no-tags origin "+commitSHA) { + t.Errorf("the mirror of %s was not updated", repo) + } + } + raw, _ := os.ReadFile(filepath.Join(m.Work, ws, "beside", "mesh-sdk", ".git", "objects", "info", "alternates")) + if strings.TrimSpace(string(raw)) != filepath.Join(m.Cache, "mirror", "novox", "mesh-sdk.git", "objects") { + t.Errorf("the sibling's alternates are %q", raw) + } +} + +// Two checks at once on one node update one mirror one after the other. +func TestTwoUpdatesOfOneMirrorNeverRunAtOnce(t *testing.T) { + var mu sync.Mutex + inside, most := 0, 0 + f := &fakeStep{gomods: "go.mod\n"} + f.rule = func(line string) (r runAnswer, ok bool) { + if !strings.Contains(line, "fetch --no-tags origin") || strings.Contains(line, "--depth") { + return r, false + } + mu.Lock() + inside++ + if inside > most { + most = inside + } + mu.Unlock() + sleepABit() + mu.Lock() + inside-- + mu.Unlock() + return r, true + } + m, states := fetchModule(t, f) + var wg sync.WaitGroup + for i := 0; i < 2; i++ { + wg.Add(1) + go func() { + defer wg.Done() + fetchTool(t, m)(fetchArgs) + }() + } + wg.Wait() + states.waitEnd(t) + states.waitEnd(t) + if most != 1 { + t.Errorf("%d updates of the mirror ran at once", most) + } +} + +// A mirror that cannot be updated does not fail the fetch: the fetch goes on without it. +func TestAMirrorThatCannotBeUpdatedLetsTheFetchGoOnWithoutIt(t *testing.T) { + f := &fakeStep{gomods: "go.mod\n"} + f.rule = func(line string) (r runAnswer, ok bool) { + if strings.Contains(line, "mirror") && strings.Contains(line, " fetch ") { + return runAnswer{Status: 1, Stderr: "fatal: no space left\n"}, true + } + return r, false + } + m, states := fetchModule(t, f) + ws := fetchOnce(t, m, states, fetchArgs).Workspace + if _, err := os.Stat(filepath.Join(m.Work, ws, "tree", ".git", "objects", "info", "alternates")); err == nil { + t.Errorf("the workspace borrows from a mirror that is not there") + } +} + +func sleepABit() { time.Sleep(30 * time.Millisecond) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/preview.go b/modules/mesh-build-go/cmd/mesh-build-go/preview.go index 87ae1066..192b8cbc 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/preview.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/preview.go @@ -96,7 +96,7 @@ func (m *Module) placeSiblings(ctx context.Context, root string, siblings []sibl if err := os.MkdirAll(dir, 0o700); err != nil { return nil, err } - if err := buildstep.FetchCommit(ctx, m.Run, dir, m.remote(s.repository), s.commit, s.ref, m.FetchOptions...); err != nil { + if err := buildstep.FetchCommit(ctx, m.Run, dir, m.remote(s.repository), s.commit, s.ref, m.fetchOptions(ctx, s.repository, s.commit, s.ref)...); err != nil { return nil, fmt.Errorf("beside %s: %w", s.repository, err) } if link := firstSymlink(dir); link != "" { diff --git a/modules/mesh-build-go/go.mod b/modules/mesh-build-go/go.mod index 999b31b9..9973c007 100644 --- a/modules/mesh-build-go/go.mod +++ b/modules/mesh-build-go/go.mod @@ -3,7 +3,7 @@ module meshbuildgo go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023120306-780be2fae5a1 github.com/distribution/distribution/v3 v3.1.2 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/modules/mesh-build-go/go.sum b/modules/mesh-build-go/go.sum index ccb75980..e5951bbf 100644 --- a/modules/mesh-build-go/go.sum +++ b/modules/mesh-build-go/go.sum @@ -2,6 +2,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYE git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467 h1:i85+fFT0wZiTZ9CRVfMzWg1+Zr7oOLl3UH7hEaKcOIc= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023120306-780be2fae5a1 h1:MqP6mWWVxM+niUGPxSSh4Q8lduYkkQ7oTAkDeC9nyv4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023120306-780be2fae5a1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI= github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= From 3ff6a8b5028bc7c80aa4ed8b8728591fe46edf82 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 23 Oct 2026 14:08:20 +0200 Subject: [PATCH 04/10] Download the go.mod directories side by side and verify a go.mod and go.sum pair once, so a fetch spends no minutes in serial go mod steps (issue 890, ADR 0536 s2) --- .../mesh-build-go/cmd/mesh-build-go/fetch.go | 66 ++++++++++- .../cmd/mesh-build-go/fetch_test.go | 5 + .../cmd/mesh-build-go/pins_test.go | 4 + .../cmd/mesh-build-go/verify_once_test.go | 112 ++++++++++++++++++ 4 files changed, 184 insertions(+), 3 deletions(-) create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/verify_once_test.go diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index b8392a58..37cb8b85 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -6,12 +6,15 @@ package main import ( "context" + "crypto/sha256" + "encoding/hex" "fmt" "os" "path/filepath" "regexp" "sort" "strings" + "sync" "time" "git.novox.be/novox/mesh-sdk/go/buildstep" @@ -143,11 +146,27 @@ func (m *Module) fetchInto(ctx context.Context, req buildstep.FetchRequest, id s if base != "" && res.Merge == "" { return res, nil // a conflict: the tree is at the trunk, nothing of the head is built or tested } - for _, d := range dirs { + // The directories download side by side, up to the bound (ADR 0536 §2); the first failure in directory order is the + // step's. Resolving a pin places a repository beside the workspace, which two directories may share: one at a time. + var pins sync.Mutex + errs := make([]error, len(dirs)) + slots := make(chan struct{}, m.parallel()) + var wg sync.WaitGroup + for i, d := range dirs { if _, err := os.Stat(filepath.Join(tree, d, "vendor", "modules.txt")); err == nil { continue // a module that vendors resolves nothing } - if err := m.downloadModules(ctx, root, filepath.Join(tree, d), d); err != nil { + wg.Add(1) + slots <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-slots }() + errs[i] = m.downloadModules(ctx, root, filepath.Join(tree, d), d, &pins) + }() + } + wg.Wait() + for _, err := range errs { + if err != nil { return nil, err } } @@ -157,13 +176,15 @@ func (m *Module) fetchInto(ctx context.Context, req buildstep.FetchRequest, id s // downloadModules downloads and verifies the Go modules of the directory dir (d in the tree). A pin of a forge module // that Go cannot fetch is looked for among the forge's pull request heads and handed to Go by a modfile (pins.go); a // failure that names no such pin is the first failure. -func (m *Module) downloadModules(ctx context.Context, root, dir, d string) error { +func (m *Module) downloadModules(ctx context.Context, root, dir, d string, pins *sync.Mutex) error { for { failure := m.downloadAndVerify(ctx, dir, d) if failure == nil { return nil } + pins.Lock() progress, err := m.resolvePins(ctx, root, dir, failure.Error()) + pins.Unlock() if err != nil { return err } @@ -173,8 +194,41 @@ func (m *Module) downloadModules(ctx context.Context, root, dir, d string) error } } +// verifyLocks hold one mutex per digest of a go.mod and go.sum, so two directories with the same pair verify once. +var verifyLocks sync.Map + +// verifiedMark is the file naming a pass of go mod verify for a go.mod and go.sum pair, "" when a pass is not kept: +// the module cache is only read by steps that are shut off with it read-only (ADR 0448 §3), so the pass holds only then, +// and a directory with a pin modfile verifies another file. +func (m *Module) verifiedMark(dir string) string { + if m.Cache == "" || m.Bubblewrap == nil { + return "" + } + if _, ok := m.pinModfile(dir); ok { + return "" + } + h := sha256.New() + for _, name := range []string{"go.mod", "go.sum"} { + raw, _ := os.ReadFile(filepath.Join(dir, name)) + fmt.Fprintf(h, "%s %d\n", name, len(raw)) + h.Write(raw) + } + return filepath.Join(m.Cache, "verified", hex.EncodeToString(h.Sum(nil))) +} + func (m *Module) downloadAndVerify(ctx context.Context, dir, d string) error { for _, verb := range []string{"download", "verify"} { + mark := "" + if verb == "verify" { + if mark = m.verifiedMark(dir); mark != "" { + mu, _ := verifyLocks.LoadOrStore(mark, &sync.Mutex{}) + mu.(*sync.Mutex).Lock() + defer mu.(*sync.Mutex).Unlock() + if _, err := os.Stat(mark); err == nil { + continue // this go.mod and go.sum passed before + } + } + } cmd, env := []string{"-C", dir, "mod", verb}, m.goEnvFor(dir) if modfile, ok := m.pinModfile(dir); ok && verb == "verify" { // go mod verify takes no -modfile: it is run on the modfile itself, which names the placed repositories. @@ -184,6 +238,12 @@ func (m *Module) downloadAndVerify(ctx context.Context, dir, d string) error { if r.Status != 0 || r.Err != "" { return fmt.Errorf("go mod %s in %s: %s", verb, d, causeLine(r.Err+"\n"+r.Stdout+"\n"+r.Stderr)) } + if mark != "" { + // Said after the pass; a mark that cannot be written only means the next fetch verifies again. + if os.MkdirAll(filepath.Dir(mark), 0o700) == nil { + os.WriteFile(mark, nil, 0o600) + } + } } return nil } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go index d9d092c5..04874659 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go @@ -30,6 +30,7 @@ type fakeStep struct { gomods string gate chan struct{} // when set, the first git command waits for it goRule func(line string) forgekey.Ran + files map[string]string // files (relative to the tree) written at checkout: go.mod and go.sum of the fixture vendors []string // directories (relative to the tree) that get a vendor/modules.txt at checkout rule func(line string) (forgekey.Ran, bool) // answers a command before the defaults do } @@ -55,6 +56,10 @@ func (f *fakeStep) run(ctx context.Context, env []string, name string, args ...s case name == "git" && strings.Contains(line, " checkout "): for i, a := range args { if a == "-C" { + for name, text := range f.files { + os.MkdirAll(filepath.Dir(filepath.Join(args[i+1], name)), 0o755) + os.WriteFile(filepath.Join(args[i+1], name), []byte(text), 0o644) + } for _, v := range f.vendors { os.MkdirAll(filepath.Join(args[i+1], v, "vendor"), 0o755) os.WriteFile(filepath.Join(args[i+1], v, "vendor", "modules.txt"), nil, 0o644) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go b/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go index bbdc0ff7..a29e4fac 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "strings" + "sync" "testing" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -28,6 +29,7 @@ type pinHarness struct { pulls string // what git ls-remote answers onTrunk bool // the pin is fetchable as it is: go mod download never says unknown revision cmds []cmdRecord + mu sync.Mutex } func newPinHarness(t *testing.T, gomod string, vendor bool, pulls string) (*pinHarness, *keptStates) { @@ -36,7 +38,9 @@ func newPinHarness(t *testing.T, gomod string, vendor bool, pulls string) (*pinH h := &pinHarness{t: t, m: m, f: f, pulls: pulls} m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { line := name + " " + strings.Join(args, " ") + h.mu.Lock() h.cmds = append(h.cmds, cmdRecord{env: env, line: line}) + h.mu.Unlock() switch { case name == "git" && strings.Contains(line, " checkout ") && !strings.Contains(line, "pinned"): for i, a := range args { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/verify_once_test.go b/modules/mesh-build-go/cmd/mesh-build-go/verify_once_test.go new file mode 100644 index 00000000..6d9776b4 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/verify_once_test.go @@ -0,0 +1,112 @@ +package main + +import ( + "context" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +func shutOffModule(t *testing.T, f *fakeStep) (*Module, *keptStates) { + t.Helper() + m, states := fetchModule(t, f) + // A module with bubblewrap runs its steps shut off, with the module cache read-only: a pass of go mod verify holds. + m.Bubblewrap = func(context.Context) (string, error) { return "bubblewrap 0.13.0", nil } + return m, states +} + +func countVerifies(f *fakeStep) int { + n := 0 + for _, l := range f.lines() { + if strings.HasPrefix(l, "go -C ") && strings.HasSuffix(l, " mod verify") { + n++ + } + } + return n +} + +// Issue 890/AC5 (ADR 0536 §2): two directories with the same go.mod and go.sum verify once, and a later fetch with the +// same pair verifies no time; a different go.mod verifies again. +func TestGoModVerifyRunsOncePerGoModAndGoSumDigest(t *testing.T) { + same := map[string]string{"a/go.mod": "module a\n", "a/go.sum": "x\n", "b/go.mod": "module a\n", "b/go.sum": "x\n"} + f := &fakeStep{gomods: "a/go.mod\nb/go.mod\n", files: same} + m, states := shutOffModule(t, f) + fetchOnce(t, m, states, fetchArgs) + if n := countVerifies(f); n != 1 { + t.Errorf("two directories with one go.mod and go.sum verified %d times, want 1", n) + } + fetchOnce(t, m, states, fetchArgs) + if n := countVerifies(f); n != 1 { + t.Errorf("a later fetch with the same pair verified again (%d in all)", n) + } + + g := &fakeStep{gomods: "a/go.mod\nb/go.mod\n", files: map[string]string{"a/go.mod": "module a\n", "a/go.sum": "x\n", "b/go.mod": "module b\n", "b/go.sum": "x\n"}} + m, states = shutOffModule(t, g) + fetchOnce(t, m, states, fetchArgs) + if n := countVerifies(g); n != 2 { + t.Errorf("two different go.mod verified %d times, want 2", n) + } +} + +// Without bubblewrap the module cache is not read-only to a step: a pass is never reused. +func TestAPassOfVerifyIsNotReusedWhenStepsAreNotShutOff(t *testing.T) { + f := &fakeStep{gomods: "a/go.mod\nb/go.mod\n", files: map[string]string{"a/go.mod": "m\n", "b/go.mod": "m\n"}} + m, states := fetchModule(t, f) // no Bubblewrap + fetchOnce(t, m, states, fetchArgs) + fetchOnce(t, m, states, fetchArgs) + if n := countVerifies(f); n != 4 { + t.Errorf("verified %d times, want every directory every fetch (4)", n) + } +} + +// A verify that fails leaves no mark: the next fetch verifies again. +func TestAFailedVerifyLeavesNoMark(t *testing.T) { + fail := true + f := &fakeStep{gomods: "go.mod\n", files: map[string]string{"go.mod": "m\n"}} + f.goRule = func(line string) forgekey.Ran { + if strings.HasSuffix(line, " mod verify") && fail { + return forgekey.Ran{Status: 1, Stdout: "x v1: dir has been modified\n"} + } + return forgekey.Ran{} + } + m, states := shutOffModule(t, f) + fetchTool(t, m)(fetchArgs) + states.waitEnd(t) + fail = false + fetchOnce(t, m, states, fetchArgs) + if n := countVerifies(f); n != 2 { + t.Errorf("verified %d times, want 2 (the failed one and the retry)", n) + } +} + +// Issue 890 (ADR 0536 §2): the downloads of the directories run side by side, never more at once than the bound. +func TestDownloadsRunSideBySideUpToTheBound(t *testing.T) { + var mu sync.Mutex + running, most := 0, 0 + f := &fakeStep{gomods: "a/go.mod\nb/go.mod\nc/go.mod\nd/go.mod\n", files: map[string]string{ + "a/go.mod": "a\n", "b/go.mod": "b\n", "c/go.mod": "c\n", "d/go.mod": "d\n"}} + f.goRule = func(line string) forgekey.Ran { + if strings.HasSuffix(line, " mod download") { + mu.Lock() + running++ + if running > most { + most = running + } + mu.Unlock() + time.Sleep(40 * time.Millisecond) + mu.Lock() + running-- + mu.Unlock() + } + return forgekey.Ran{} + } + m, states := fetchModule(t, f) + m.Parallel = 2 + fetchOnce(t, m, states, fetchArgs) + if most != 2 { + t.Errorf("at most %d downloads ran at once, want 2", most) + } +} From 22f7a785035e304503185e4d100bec80832ae19a Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 23 Oct 2026 14:09:34 +0200 Subject: [PATCH 05/10] Dissociate a workspace from the mirror after the fetch, so a shut-off step finds every object in the tree (issue 890 C2 follow-up) --- .../mesh-build-go/cmd/mesh-build-go/fetch.go | 3 + .../mesh-build-go/cmd/mesh-build-go/mirror.go | 14 ++++ .../cmd/mesh-build-go/mirror_test.go | 68 +++++++++++++++++-- .../cmd/mesh-build-go/preview.go | 3 + 4 files changed, 82 insertions(+), 6 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index 37cb8b85..e5a1b49b 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -126,6 +126,9 @@ func (m *Module) fetchInto(ctx context.Context, req buildstep.FetchRequest, id s return nil, err } } + if err := m.dissociate(ctx, tree); err != nil { + return nil, err + } if res.Beside, err = m.placeSiblings(ctx, root, siblings); err != nil { return nil, err } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/mirror.go b/modules/mesh-build-go/cmd/mesh-build-go/mirror.go index 555e58c6..0ad17fda 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/mirror.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/mirror.go @@ -57,3 +57,17 @@ func (m *Module) updateMirror(ctx context.Context, mirror, remote, commit, ref s defer syscall.Flock(int(lock.Fd()), syscall.LOCK_UN) return buildstep.UpdateMirror(ctx, m.Run, mirror, remote, commit, ref, m.FetchOptions...) } + +// dissociate makes the repository in dir self-contained again, as git clone --dissociate does: the objects it borrowed +// from the mirror are packed into it and its alternates file goes, so a step shut off with the workspace alone (the +// mirror is not bound into it) finds every object. The network saving stays: the borrowing was for the transfer. +func (m *Module) dissociate(ctx context.Context, dir string) error { + alternates := filepath.Join(dir, ".git", "objects", "info", "alternates") + if _, err := os.Stat(alternates); err != nil { + return nil // no mirror was used + } + if r := m.Run(ctx, []string{"GIT_TERMINAL_PROMPT=0"}, "git", "-C", dir, "repack", "-a", "-d", "-q"); r.Status != 0 || r.Err != "" { + return fmt.Errorf("git repack in %s: %s", dir, firstLine(r.Err+" "+r.Stderr)) + } + return os.Remove(alternates) +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/mirror_test.go b/modules/mesh-build-go/cmd/mesh-build-go/mirror_test.go index dabf67eb..6721155e 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/mirror_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/mirror_test.go @@ -2,7 +2,9 @@ package main import ( "encoding/json" + "git.novox.be/novox/mesh-sdk/go/buildstep" "os" + "os/exec" "path/filepath" "strings" "sync" @@ -50,10 +52,15 @@ func TestTwoFetchesOfARepositoryUseOneMirrorAndReferenceIt(t *testing.T) { if updates != 2 { t.Errorf("the mirror %s was updated %d times, want 2:\n%s", mirror, updates, strings.Join(f.lines(), "\n")) } + // The fake git writes nothing, so the alternates file the SDK wrote stands until the fetch dissociates: it is gone + // at the end and git repack -a -d ran in each tree. for _, ws := range workspaces { - raw, err := os.ReadFile(filepath.Join(m.Work, ws, "tree", ".git", "objects", "info", "alternates")) - if err != nil || strings.TrimSpace(string(raw)) != filepath.Join(mirror, "objects") { - t.Errorf("the alternates of %s are %q (%v)", ws, raw, err) + tree := filepath.Join(m.Work, ws, "tree") + if _, err := os.Stat(filepath.Join(tree, ".git", "objects", "info", "alternates")); err == nil { + t.Errorf("the workspace %s still borrows from the mirror", ws) + } + if !strings.Contains(strings.Join(f.lines(), "\n"), "-C "+tree+" repack -a -d") { + t.Errorf("git repack -a -d did not run in %s", tree) } } } @@ -71,9 +78,8 @@ func TestASiblingIsPlacedThroughItsOwnMirror(t *testing.T) { t.Errorf("the mirror of %s was not updated", repo) } } - raw, _ := os.ReadFile(filepath.Join(m.Work, ws, "beside", "mesh-sdk", ".git", "objects", "info", "alternates")) - if strings.TrimSpace(string(raw)) != filepath.Join(m.Cache, "mirror", "novox", "mesh-sdk.git", "objects") { - t.Errorf("the sibling's alternates are %q", raw) + if _, err := os.Stat(filepath.Join(m.Work, ws, "beside", "mesh-sdk", ".git", "objects", "info", "alternates")); err == nil { + t.Errorf("the sibling still borrows from the mirror") } } @@ -132,3 +138,53 @@ func TestAMirrorThatCannotBeUpdatedLetsTheFetchGoOnWithoutIt(t *testing.T) { } func sleepABit() { time.Sleep(30 * time.Millisecond) } + +// The workspace is self-contained after the fetch: no alternates file, and git finds the commit with the mirror gone, +// as a step shut off with the workspace alone needs. Real git, a local repository as the forge. +func TestTheWorkspaceIsSelfContainedAfterAFetchThroughTheMirror(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("no git") + } + forge := t.TempDir() + src := filepath.Join(forge, "novox", "mesh-sdk.git") + if err := os.MkdirAll(src, 0o755); err != nil { + t.Fatal(err) + } + git := func(dir string, args ...string) string { + cmd := exec.Command("git", append([]string{"-C", dir, "-c", "user.name=t", "-c", "user.email=t@example.org", "-c", "commit.gpgsign=false"}, args...)...) + cmd.Env = append(os.Environ(), "GIT_CONFIG_GLOBAL=/dev/null") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v %s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + git(src, "init", "-q") + if err := os.WriteFile(filepath.Join(src, "f.txt"), []byte("one\n"), 0o644); err != nil { + t.Fatal(err) + } + git(src, "add", ".") + git(src, "commit", "-q", "-m", "one") + commit := git(src, "rev-parse", "HEAD") + + states := newStates() + m := moduleWith(states, "go1.27.1") + m.Home, m.Work, m.Cache = t.TempDir(), t.TempDir(), t.TempDir() + m.Run = StepRunner(m.Home) + m.RemoteBase = "file://" + forge + "/" + m.FetchOptions = []buildstep.Option{buildstep.AllowFileRemote()} + ws := fetchOnce(t, m, states, map[string]any{"repository": "novox/mesh-sdk", "commit": commit}).Workspace + tree := filepath.Join(m.Work, ws, "tree") + if _, err := os.Stat(filepath.Join(tree, ".git", "objects", "info", "alternates")); err == nil { + t.Errorf("the workspace still has an alternates file") + } + if err := os.RemoveAll(filepath.Join(m.Cache, "mirror")); err != nil { + t.Fatal(err) + } + if out, err := exec.Command("git", "-C", tree, "cat-file", "-e", commit+"^{commit}").CombinedOutput(); err != nil { + t.Errorf("git does not find the commit with the mirror gone: %v %s", err, out) + } + if out, err := exec.Command("git", "-C", tree, "fsck", "--connectivity-only").CombinedOutput(); err != nil { + t.Errorf("the workspace is not whole: %v %s", err, out) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/preview.go b/modules/mesh-build-go/cmd/mesh-build-go/preview.go index 192b8cbc..067dcfff 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/preview.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/preview.go @@ -99,6 +99,9 @@ func (m *Module) placeSiblings(ctx context.Context, root string, siblings []sibl if err := buildstep.FetchCommit(ctx, m.Run, dir, m.remote(s.repository), s.commit, s.ref, m.fetchOptions(ctx, s.repository, s.commit, s.ref)...); err != nil { return nil, fmt.Errorf("beside %s: %w", s.repository, err) } + if err := m.dissociate(ctx, dir); err != nil { + return nil, fmt.Errorf("beside %s: %w", s.repository, err) + } if link := firstSymlink(dir); link != "" { return nil, fmt.Errorf("beside %s holds a symlink, %s", s.repository, link) } From 7ec760bca23141652624e9e583ae7e6dd24791d6 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 23 Oct 2026 14:15:22 +0200 Subject: [PATCH 06/10] Compile the workspace's own mesh-build-go and run digest and test as it under runner workspace, so a change to this module is judged by its candidate (issue 1898 K1, K2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The checker asks runner workspace when a merge changes modules/mesh-build-go; the step's shut-off process now takes a step spec (digest, unit, integration) and every answer says judged_by. ADR 0535 §1, §3, §5. --- modules/mesh-build-go/README.md | 10 + .../cmd/mesh-build-go/candidate.go | 132 +++++++++ .../candidate_integration_test.go | 127 +++++++++ .../cmd/mesh-build-go/candidate_test.go | 262 ++++++++++++++++++ .../mesh-build-go/cmd/mesh-build-go/digest.go | 57 +++- .../mesh-build-go/cmd/mesh-build-go/level.go | 114 ++++++-- .../mesh-build-go/cmd/mesh-build-go/levels.go | 3 + .../mesh-build-go/cmd/mesh-build-go/test.go | 50 +++- 8 files changed, 714 insertions(+), 41 deletions(-) create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/candidate.go create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/candidate_integration_test.go create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/candidate_test.go diff --git a/modules/mesh-build-go/README.md b/modules/mesh-build-go/README.md index 69d8cfa2..f2493e35 100644 --- a/modules/mesh-build-go/README.md +++ b/modules/mesh-build-go/README.md @@ -105,6 +105,16 @@ fingerprint once, as the version the build nodes agree on (ADR 0477 §3), and th version's. A package with no tests at the level has no digest and is named in `without_tests`. A directory that vendors is listed and tested from its vendor directory (`-mod=vendor`). +## A change to this module is judged by its candidate + +`build_go_digest` and `build_go_test` take `runner`: `installed` (the default) runs the step as this program; +`workspace` compiles the workspace's own `modules/mesh-build-go` (`go build` with `GOPROXY=off`, `GOTOOLCHAIN=local`, in a +run step shut off) into the workspace's `candidate/mesh-build-go`, once per workspace, and runs the step as that program: +` level ` in one bubblewrap, the spec naming the step (`digest`, `unit` or `integration`). This program +keeps the fetch, the forge key and the bus login (ADR 0535 §1). A candidate that does not compile fails the step with the +compiler's first error, and nothing runs (§3). Every digest and level answer says `judged_by`: `installed`, or +`candidate `. The checker asks `workspace` when a merge changes `modules/mesh-build-go/` (issue 1898). + ## Build `build_go_build {workspace, module, artifact}` builds one Go bundle artifact of a module of the workspace with the build diff --git a/modules/mesh-build-go/cmd/mesh-build-go/candidate.go b/modules/mesh-build-go/cmd/mesh-build-go/candidate.go new file mode 100644 index 00000000..38a27ad6 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/candidate.go @@ -0,0 +1,132 @@ +package main + +// A change to this module is judged by its own candidate (novox/hq ADR 0535; issue 1898): asked with the runner +// workspace, the digest and test steps compile the program of mesh-build-go from the workspace's merged tree in a run +// step, shut off, and run the step as that program in one shut-off process (` level `, level.go), where +// this program would run the step's commands. This program keeps the fetch, the forge key and the bus login; the +// candidate's program holds none of them. Each answer says which program ran it (judged_by). + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "sync" +) + +// The runners of a digest or test step: this installed program, or the candidate's compiled from the workspace. +const ( + RunnerInstalled = "installed" + RunnerWorkspace = "workspace" +) + +// runnerProperty is the runner argument as the digest and test tools announce it: a checker reads it there, and an +// installed module that does not announce it judges as it always did (ADR 0535 §2). +var runnerProperty = map[string]any{"type": "string", "enum": []string{RunnerInstalled, RunnerWorkspace}, + "description": "optional: installed (the default) runs the step as this installed program; workspace compiles the " + + "workspace's own mesh-build-go (modules/mesh-build-go of the merged tree) shut off and runs the step as it, for a " + + "change to this module (ADR 0535); the answer's judged_by names the program that ran"} + +// candidateModule is the directory of this module's go.mod in the catalogue's tree, and candidateMain its program's +// package there. +const ( + candidateModule = "modules/mesh-build-go" + candidateMain = "./cmd/mesh-build-go" +) + +// runnerArg reads the runner argument: "" (the installed program), installed or workspace. +func runnerArg(v any) (string, error) { + if v == nil { + return "", nil + } + s, ok := v.(string) + if !ok || (s != "" && s != RunnerInstalled && s != RunnerWorkspace) { + return "", fmt.Errorf("the runner %v is neither %s nor %s", v, RunnerInstalled, RunnerWorkspace) + } + return s, nil +} + +// workspaceThere refuses a workspace id that is not one, or a workspace this node never fetched. +func (m *Module) workspaceThere(workspace string) error { + if !workspaceOK.MatchString(workspace) { + return fmt.Errorf("%q is not a workspace id", workspace) + } + if _, err := os.Stat(filepath.Join(m.Work, workspace, "tree")); err != nil { + return fmt.Errorf("no workspace %s: it was never fetched here, or it is older than a day and was removed", workspace) + } + return nil +} + +// candidateLocks hold one mutex per workspace, so its candidate is compiled once. +var candidateLocks sync.Map + +// candidate is the program of mesh-build-go compiled from the workspace's tree, in the workspace beside its tree +// (candidate/mesh-build-go), compiled once per workspace, in a run step shut off with the caches read-only and nothing +// fetched (GOPROXY=off: the fetch step downloaded every module the tree's go.mod files name). A candidate that does not +// compile is the step's failure, naming the compiler's first error (ADR 0535 §3). +func (m *Module) candidate(ctx context.Context, workspace string) (string, error) { + exe := filepath.Join(m.Work, workspace, "candidate", "mesh-build-go") + mu, _ := candidateLocks.LoadOrStore(exe, &sync.Mutex{}) + mu.(*sync.Mutex).Lock() + defer mu.(*sync.Mutex).Unlock() + if st, err := os.Stat(exe); err == nil && st.Mode().IsRegular() { + return exe, nil + } + dir := filepath.Join(m.Work, workspace, "tree", candidateModule) + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err != nil { + return "", fmt.Errorf("the workspace holds no %s/go.mod, so it has no candidate mesh-build-go to run", candidateModule) + } + if err := os.MkdirAll(filepath.Dir(exe), 0o700); err != nil { + return "", err + } + run, err := m.stepRunner(workspace, false) + if err != nil { + return "", err + } + env := append(m.goEnvFor(dir), "GOPROXY=off", "GOTOOLCHAIN=local") + r := run(ctx, env, "go", "-C", dir, "build", "-o", exe, candidateMain) + if r.Status != 0 || r.Err != "" { + return "", fmt.Errorf("the candidate's mesh-build-go does not compile: %s", compileError(r.Err+"\n"+r.Stderr+"\n"+r.Stdout)) + } + return exe, nil +} + +// compileError is the first line of the compiler's output that says an error: not the `# package` heading go build +// prints above a package's errors, nor Go's progress. +func compileError(out string) string { + for _, l := range strings.Split(out, "\n") { + if l = strings.TrimSpace(l); l != "" && !strings.HasPrefix(l, "#") && !strings.HasPrefix(l, "go: downloading ") { + return l + } + } + return firstLine(out) +} + +// mergeCommit is the commit the workspace's tree holds: the merge the fetch made, or the head it fetched. +func (m *Module) mergeCommit(ctx context.Context, workspace string) (string, error) { + r := m.Run(ctx, nil, "git", "-C", filepath.Join(m.Work, workspace, "tree"), "rev-parse", "HEAD") + commit := strings.TrimSpace(r.Stdout) + if r.Status != 0 || r.Err != "" || commit == "" { + return "", fmt.Errorf("the workspace's commit: git rev-parse: %s", firstLine(r.Err+"\n"+r.Stderr)) + } + return commit, nil +} + +// asCandidate runs the spec's step as the candidate's program, shut off, and answers what it said and the judged_by of +// the answer: "candidate ". +func (m *Module) asCandidate(ctx context.Context, spec levelSpec) (levelAnswer, string, error) { + commit, err := m.mergeCommit(ctx, spec.Workspace) + if err != nil { + return levelAnswer{}, "", err + } + exe, err := m.candidate(ctx, spec.Workspace) + if err != nil { + return levelAnswer{}, "", err + } + a, err := m.shutOff(ctx, exe, spec) + if err != nil { + return levelAnswer{}, "", fmt.Errorf("the candidate's mesh-build-go: %w", err) + } + return a, "candidate " + commit, nil +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/candidate_integration_test.go b/modules/mesh-build-go/cmd/mesh-build-go/candidate_integration_test.go new file mode 100644 index 00000000..fe6dc280 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/candidate_integration_test.go @@ -0,0 +1,127 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// candidateMainSource is a candidate's program that answers a mark: asked `level `, it tries what a run step must +// not reach (the account's key and login, a name, a socket of the machine) and answers a passing level of 42 tests whose +// why is "mark" and what it reached. +const candidateMainSource = `package main + +import ( + "encoding/json" + "fmt" + "net" + "os" + "time" +) + +func main() { + if len(os.Args) != 3 || os.Args[1] != "level" { + os.Exit(2) + } + seen := map[string]bool{} + _, err := os.ReadFile(%q) + seen["key"] = err == nil + _, err = os.ReadFile(%q) + seen["login"] = err == nil + _, err = net.LookupHost("git.novox.be") + seen["resolved"] = err == nil + c, err := net.DialTimeout("tcp", %q, 2*time.Second) + if err == nil { + c.Close() + } + seen["tcp"] = err == nil + raw, _ := json.Marshal(seen) + var spec struct { + Step string ` + "`json:\"step\"`" + ` + } + json.Unmarshal([]byte(os.Args[2]), &spec) + out, _ := json.Marshal(map[string]any{"result": map[string]any{"level": spec.Step, "passed": true, "tests": 42, + "packages": []any{}, "why": "mark " + string(raw)}}) + fmt.Println(string(out)) +} +` + +// 1898/AC4 (ADR 0535 §1): a tree whose mesh-build-go program answers a mark: the unit level with the runner workspace +// compiles that program and runs it shut off, answers its mark judged by the candidate at the tree's commit, and the +// program reached no key, no login, no name and no socket of the machine. +func TestTheUnitLevelUnderTheWorkspaceRunnerRunsTheTreesProgramShutOff(t *testing.T) { + needTools(t, "the candidate's program shut off", "go", "git", "bwrap") + home := t.TempDir() + key, login := filepath.Join(home, ".ssh", "mesh_ed25519"), filepath.Join(home, "broker") + for _, f := range []string{key, login} { + if err := os.MkdirAll(filepath.Dir(f), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(f, []byte("secret"), 0o600); err != nil { + t.Fatal(err) + } + } + tcp, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer tcp.Close() + + work, err := os.MkdirTemp("", "w") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(work) }) + tree := filepath.Join(work, "ws1", "tree") + dir := filepath.Join(tree, candidateModule) + if err := os.MkdirAll(filepath.Join(dir, "cmd", "mesh-build-go"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module meshbuildgo\n\ngo 1.25.0\n"), 0o644); err != nil { + t.Fatal(err) + } + src := fmt.Sprintf(candidateMainSource, key, login, tcp.Addr().String()) + if err := os.WriteFile(filepath.Join(dir, "cmd", "mesh-build-go", "main.go"), []byte(src), 0o644); err != nil { + t.Fatal(err) + } + for _, c := range [][]string{{"init", "-q"}, {"add", "-A"}, {"-c", "user.email=a@b", "-c", "user.name=x", "commit", "-qm", "x"}} { + if out, err := exec.Command("git", append([]string{"-C", tree}, c...)...).CombinedOutput(); err != nil { + t.Fatalf("git %v: %v %s", c, err, out) + } + } + head, err := exec.Command("git", "-C", tree, "rev-parse", "HEAD").Output() + if err != nil { + t.Fatal(err) + } + + m := &Module{Recipe: Current(), Home: home, Work: work, Cache: t.TempDir()} + m.Run = StepRunner(home) + out, err := m.test(context.Background(), map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": RunnerWorkspace}) + if err != nil { + t.Fatal(err) + } + res := out.(*TestResult) + if res.Tests != 42 || !strings.HasPrefix(res.Why, "mark ") { + t.Fatalf("result = %+v, want the candidate's mark", res) + } + if want := "candidate " + strings.TrimSpace(string(head)); res.JudgedBy != want { + t.Errorf("judged by %q, want %q", res.JudgedBy, want) + } + var seen map[string]bool + if err := json.Unmarshal([]byte(strings.TrimPrefix(res.Why, "mark ")), &seen); err != nil || len(seen) != 4 { + t.Fatalf("the program said %q: %v", res.Why, err) + } + for what, reached := range seen { + if reached { + t.Errorf("the candidate's program reached %s", what) + } + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/candidate_test.go b/modules/mesh-build-go/cmd/mesh-build-go/candidate_test.go new file mode 100644 index 00000000..cf12dadd --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/candidate_test.go @@ -0,0 +1,262 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// Issue 1898 K1, K2 (ADR 0535): asked with the runner workspace, the digest and test steps compile the workspace's own +// mesh-build-go shut off and run the step as it, in one shut-off process; the answer says which program ran. + +const candidateSHA = "fedcba9876543210fedcba9876543210fedcba98" + +// candidateRig is a module over a workspace whose tree holds modules/mesh-build-go, with a runner standing for git, go +// and the candidate's program; what ran shut off is in seen, every command in ran. +type candidateRig struct { + m *Module + states *keptStates + seen []shutOff + ran []string + compile forgekey.Ran // what go build answers; Status 0 writes the program + answer func(spec levelSpec) levelAnswer // what the candidate's program answers + specs []levelSpec + compiled int +} + +func newCandidateRig(t *testing.T) *candidateRig { + t.Helper() + cr := &candidateRig{states: newStates()} + cr.m = moduleWith(cr.states, "go1.27.1") + cr.m.Home, cr.m.Work, cr.m.Cache = t.TempDir(), t.TempDir(), t.TempDir() + cr.m.Executable = "/opt/bundle/mesh-build-go" + dir := filepath.Join(cr.m.Work, "ws1", "tree", candidateModule) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module meshbuildgo\n\ngo 1.26.0\n"), 0o644); err != nil { + t.Fatal(err) + } + cr.answer = func(spec levelSpec) levelAnswer { + if spec.Step == stepDigest { + return levelAnswer{Digest: &DigestResult{Level: spec.Level, Packages: map[string]string{"meshbuildgo/cmd/mesh-build-go": "mark"}}} + } + return levelAnswer{Result: &TestResult{Level: spec.Step, Passed: true, Tests: 42, Packages: []PackageResult{}, Why: "mark"}} + } + exe := filepath.Join(cr.m.Work, "ws1", "candidate", "mesh-build-go") + cr.m.Run = throughBubblewrap(t, func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + line := name + " " + strings.Join(args, " ") + cr.ran = append(cr.ran, line) + switch { + case name == "git" && slices.Contains(args, "rev-parse"): + return forgekey.Ran{Stdout: candidateSHA + "\n"} + case name == "git" && slices.Contains(args, "ls-files"): + return forgekey.Ran{Stdout: "go.mod\n"} + case name == "postgres": + return forgekey.Ran{Stdout: "postgres (PostgreSQL) 18.6\n"} + case name == "go" && slices.Contains(args, "build"): + cr.compiled++ + for _, want := range []string{"GOPROXY=off", "GOTOOLCHAIN=local", "GOFLAGS=-mod=mod"} { + if !slices.Contains(env, want) { + t.Errorf("the candidate compiled without %s: %v", want, env) + } + } + if cr.compile.Status == 0 && cr.compile.Err == "" { + if err := os.WriteFile(args[slices.Index(args, "-o")+1], []byte("program"), 0o755); err != nil { + t.Fatal(err) + } + } + return cr.compile + case name == exe && len(args) == 2 && args[0] == levelArg: + var spec levelSpec + if err := json.Unmarshal([]byte(args[1]), &spec); err != nil { + t.Errorf("the spec %s: %v", args[1], err) + } + cr.specs = append(cr.specs, spec) + raw, _ := json.Marshal(cr.answer(spec)) + return forgekey.Ran{Stdout: string(raw) + "\n"} + } + return forgekey.Ran{Status: 1, Err: "not expected here: " + line} + }, &cr.seen) + return cr +} + +func (cr *candidateRig) test(t *testing.T, args map[string]any) (TestResult, delivery.Step) { + t.Helper() + if _, err := stepTool(t, cr.m, "build_go_test")(args); err != nil { + t.Fatal(err) + } + st := cr.states.waitEnd(t) + var res TestResult + if st.State == delivery.StepPassed { + if err := json.Unmarshal(st.Result, &res); err != nil { + t.Fatalf("result %s: %v", st.Result, err) + } + } + return res, st +} + +func TestTheDigestAndTestToolsAnnounceTheRunner(t *testing.T) { // 1898 K1, ADR 0535 §2 + m := moduleWith(newStates(), "go1.27.1") + for _, s := range m.StepTools() { + if s.Name != "build_go_digest" && s.Name != "build_go_test" { + continue + } + p, _ := s.Input["properties"].(map[string]any)["runner"].(map[string]any) + if p == nil || !slices.Equal(p["enum"].([]string), []string{RunnerInstalled, RunnerWorkspace}) { + t.Errorf("%s announces the runner %v, want installed and workspace", s.Name, p) + } + } +} + +func TestTheUnitLevelUnderTheWorkspaceRunnerRunsTheCandidateShutOff(t *testing.T) { // 1898 K1, K2 (AC4's unit part) + cr := newCandidateRig(t) + res, st := cr.test(t, map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": RunnerWorkspace, "packages": []any{"./..."}}) + if st.State != delivery.StepPassed { + t.Fatalf("the step ended %s: %s", st.State, st.Reason) + } + if res.Tests != 42 || res.Why != "mark" || res.JudgedBy != "candidate "+candidateSHA { + t.Errorf("result = %+v, want the candidate's mark judged by candidate %s", res, candidateSHA) + } + if len(cr.specs) != 1 || cr.specs[0].Step != stepUnit || cr.specs[0].Workspace != "ws1" || !slices.Equal(cr.specs[0].Packages, []string{"./..."}) || cr.specs[0].Bound != UnitBound { + t.Errorf("the candidate was asked %+v, want the unit level of ws1 with its packages and bound", cr.specs) + } + for _, l := range cr.ran { + if strings.HasPrefix(l, "go ") && strings.Contains(l, " test ") { + t.Errorf("the installed program ran %q: the level is the candidate's to run", l) + } + } + // Two commands shut off: the compile, then the candidate's process; each as a run step is (ADR 0448 §2). + if len(cr.seen) != 2 || !strings.Contains(cr.seen[0].command, " build ") || !strings.Contains(cr.seen[0].command, "-C "+filepath.Join(cr.m.Work, "ws1", "tree", candidateModule)) || + !strings.HasPrefix(cr.seen[1].command, filepath.Join(cr.m.Work, "ws1", "candidate", "mesh-build-go")+" level ") { + t.Fatalf("shut off %+v, want the compile and the candidate's process", cr.seen) + } + for _, s := range cr.seen { + checkShutOff(t, cr.m, s, false) + } +} + +func TestTheIntegrationLevelUnderTheWorkspaceRunnerRunsTheCandidate(t *testing.T) { // 1898 K2 + cr := newCandidateRig(t) + res, st := cr.test(t, map[string]any{"workspace": "ws1", "level": LevelIntegration, "runner": RunnerWorkspace}) + if st.State != delivery.StepPassed || res.JudgedBy != "candidate "+candidateSHA { + t.Fatalf("the step ended %s (%s): %+v", st.State, st.Reason, res) + } + if len(cr.specs) != 1 || cr.specs[0].Step != stepIntegration || cr.specs[0].Bound != IntegrationRun || cr.specs[0].Setup != IntegrationSetup { + t.Errorf("the candidate was asked %+v, want the integration level with its bounds", cr.specs) + } +} + +func TestTheDigestUnderTheWorkspaceRunnerIsTheCandidatesAndTheCandidateIsCompiledOnce(t *testing.T) { // 1898 K1, K2 + cr := newCandidateRig(t) + out, err := stepTool(t, cr.m, "build_go_digest")(map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": RunnerWorkspace}) + if err != nil { + t.Fatal(err) + } + d := out.(DigestResult) + if d.Packages["meshbuildgo/cmd/mesh-build-go"] != "mark" || d.JudgedBy != "candidate "+candidateSHA { + t.Errorf("digest = %+v, want the candidate's, judged by candidate %s", d, candidateSHA) + } + if len(cr.specs) != 1 || cr.specs[0].Step != stepDigest || cr.specs[0].Level != LevelUnit { + t.Errorf("the candidate was asked %+v", cr.specs) + } + cr.test(t, map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": RunnerWorkspace}) + if cr.compiled != 1 { + t.Errorf("compiled the candidate %d times for two steps of one workspace, want once", cr.compiled) + } +} + +func TestACandidateThatDoesNotCompileFailsTheStepWithTheCompilersFirstErrorAndRunsNoLevel(t *testing.T) { // 1898/AC5 + cr := newCandidateRig(t) + cr.compile = forgekey.Ran{Status: 1, Stderr: "# meshbuildgo/cmd/mesh-build-go\ncmd/mesh-build-go/digest.go:12:2: undefined: x\ncmd/mesh-build-go/digest.go:13:2: undefined: y\n"} + _, st := cr.test(t, map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": RunnerWorkspace}) + if st.State == delivery.StepPassed || !strings.Contains(st.Reason, "does not compile: cmd/mesh-build-go/digest.go:12:2: undefined: x") || strings.Contains(st.Reason, "undefined: y") { + t.Fatalf("the step ended %s: %q, want it failed naming the first compile error", st.State, st.Reason) + } + if len(cr.specs) != 0 { + t.Errorf("the candidate ran %+v after it failed to compile", cr.specs) + } + for _, l := range cr.ran { + if strings.Contains(l, " test ") { + t.Errorf("a level ran: %q", l) + } + } + if _, err := stepTool(t, cr.m, "build_go_digest")(map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": RunnerWorkspace}); err == nil || + !strings.Contains(err.Error(), "undefined: x") { + t.Errorf("the digest answered %v, want the compile error", err) + } +} + +func TestTheInstalledProgramSaysItJudged(t *testing.T) { // 1898 K1, ADR 0535 §5 + m, states, ws, _ := workspaceFor(t, "slow") + for _, runner := range []any{nil, RunnerInstalled} { + args := map[string]any{"workspace": ws, "level": LevelUnit} + if runner != nil { + args["runner"] = runner + } + res, st := runUnitWith(t, m, states, args) + if st.State != delivery.StepPassed || res.JudgedBy != RunnerInstalled { + t.Errorf("runner %v: the step ended %s, judged by %q, want installed", runner, st.State, res.JudgedBy) + } + } + dm, _ := digestFixture(t) + if d := digests(t, dm, LevelUnit); d.JudgedBy != RunnerInstalled { + t.Errorf("the installed digest is judged by %q, want installed", d.JudgedBy) + } +} + +func TestAnUnknownRunnerIsRefused(t *testing.T) { // 1898 K1 + cr := newCandidateRig(t) + if _, err := cr.m.digest(context.Background(), map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": "elsewhere"}); err == nil { + t.Error("the digest took the runner elsewhere") + } + if _, err := cr.m.test(context.Background(), map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": "elsewhere"}); err == nil { + t.Error("the test step took the runner elsewhere") + } +} + +func TestTheShutOffProcessRunsTheStepItIsAsked(t *testing.T) { // 1898 K2: the candidate's side of the spec + m, _, ws, calls := workspaceFor(t, "slow") + m.Inside = true + a := m.answer(context.Background(), levelSpec{Step: stepUnit, Workspace: ws, Packages: []string{"./..."}, Bound: UnitBound}) + if a.Error != "" || a.Result == nil || a.Result.Level != LevelUnit || a.Result.Tests == 0 { + t.Fatalf("the unit step answered %+v", a) + } + ranTest := false + for _, c := range *calls { + ranTest = ranTest || strings.HasPrefix(c.line, "go -C ") && strings.Contains(c.line, " test -json ") + } + if !ranTest { + t.Errorf("the process ran no go test: %+v", *calls) + } + dm, _ := digestFixture(t) + dm.Inside = true + if a := dm.answer(context.Background(), levelSpec{Step: stepDigest, Level: LevelUnit, Workspace: "ws1"}); a.Error != "" || a.Digest == nil || len(a.Digest.Packages) == 0 { + t.Errorf("the digest step answered %+v", a) + } + if a := m.answer(context.Background(), levelSpec{Step: "deploy", Workspace: ws}); a.Error == "" { + t.Error("the process took a step that is not one") + } +} + +func runUnitWith(t *testing.T, m *Module, states *keptStates, args map[string]any) (TestResult, delivery.Step) { + t.Helper() + if _, err := stepTool(t, m, "build_go_test")(args); err != nil { + t.Fatal(err) + } + st := states.waitEnd(t) + var res TestResult + if st.State == delivery.StepPassed { + if err := json.Unmarshal(st.Result, &res); err != nil { + t.Fatalf("result %s: %v", st.Result, err) + } + } + return res, st +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest.go b/modules/mesh-build-go/cmd/mesh-build-go/digest.go index f3e2ed53..1c92d318 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/digest.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest.go @@ -89,6 +89,9 @@ type DigestResult struct { // only when they are the siblings it gave the fetch. Beside []string `json:"beside,omitempty"` WithoutTests []string `json:"without_tests,omitempty"` + // JudgedBy is the program that ran the digest (ADR 0535 §5): "installed", or "candidate " under the + // runner workspace. + JudgedBy string `json:"judged_by,omitempty"` } func (m *Module) digestTool() StepTool { @@ -103,6 +106,7 @@ func (m *Module) digestTool() StepTool { "properties": map[string]any{ "workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"}, "level": map[string]any{"type": "string", "enum": []string{LevelUnit, LevelIntegration}, "description": "the level"}, + "runner": runnerProperty, }, "required": []string{"workspace", "level"}, }, @@ -113,12 +117,49 @@ func (m *Module) digestTool() StepTool { func (m *Module) digest(ctx context.Context, args map[string]any) (any, error) { workspace, _ := args["workspace"].(string) level, _ := args["level"].(string) + runner, err := runnerArg(args["runner"]) + if err != nil { + return nil, err + } + if runner == RunnerWorkspace { + if err := m.workspaceThere(workspace); err != nil { + return nil, err + } + switch level { + case LevelUnit: + case LevelIntegration: + if _, err := m.checkPostgres(ctx); err != nil { + return nil, err // a *buildstep.Refusal, as the installed program's digest says it + } + default: + return nil, fmt.Errorf("the level %q is neither unit nor integration", level) + } + a, judged, err := m.asCandidate(ctx, levelSpec{Step: stepDigest, Level: level, Workspace: workspace}) + if err != nil { + return nil, err + } + if a.Digest == nil { + return nil, fmt.Errorf("the candidate's mesh-build-go answered no digest") + } + a.Digest.JudgedBy = judged + return *a.Digest, nil + } + res, err := m.digestHere(ctx, workspace, level) + if err != nil { + return nil, err + } + res.JudgedBy = RunnerInstalled + return res, nil +} + +// digestHere is the digest step as this program runs it. +func (m *Module) digestHere(ctx context.Context, workspace, level string) (DigestResult, error) { if !workspaceOK.MatchString(workspace) { - return nil, fmt.Errorf("%q is not a workspace id", workspace) + return DigestResult{}, fmt.Errorf("%q is not a workspace id", workspace) } tree := filepath.Join(m.Work, workspace, "tree") if _, err := os.Stat(tree); err != nil { - return nil, fmt.Errorf("no workspace %s: it was never fetched here, or it is older than a day and was removed", workspace) + return DigestResult{}, fmt.Errorf("no workspace %s: it was never fetched here, or it is older than a day and was removed", workspace) } var tags []string header := []string{"digest " + DigestVersion, "level " + level} @@ -126,17 +167,17 @@ func (m *Module) digest(ctx context.Context, args map[string]any) (any, error) { case LevelUnit: case LevelIntegration: if _, err := m.checkPostgres(ctx); err != nil { - return nil, err + return DigestResult{}, err } tags = integrationTags header = append(header, "postgres "+strconv.Itoa(m.Recipe.PostgresMajor), "nats-server "+m.Recipe.NatsServer) default: - return nil, fmt.Errorf("the level %q is neither unit nor integration", level) + return DigestResult{}, fmt.Errorf("the level %q is neither unit nor integration", level) } header = append(header, "tags "+strings.Join(tags, ",")) beside, err := m.besideLines(workspace) if err != nil { - return nil, err + return DigestResult{}, err } besideHeader := make([]string, 0, len(beside)) for _, b := range beside { @@ -145,16 +186,16 @@ func (m *Module) digest(ctx context.Context, args map[string]any) (any, error) { dirs, err := m.goModDirs(ctx, tree) if err != nil { - return nil, err + return DigestResult{}, err } res := DigestResult{Level: level, Packages: map[string]string{}, Beside: beside} for _, d := range dirs { pkgs, err := m.listPackages(ctx, workspace, tree, d, tags) if err != nil { - return nil, err + return DigestResult{}, err } if err := digestModule(&res, pkgs, tree, header, besideHeader); err != nil { - return nil, err + return DigestResult{}, err } } sort.Strings(res.WithoutTests) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/level.go b/modules/mesh-build-go/cmd/mesh-build-go/level.go index 520962dc..6ff18aa6 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/level.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/level.go @@ -18,11 +18,22 @@ import ( "git.novox.be/novox/mesh-sdk/go/buildstep" ) -// levelArg is the first argument that makes this binary an integration level's process. +// levelArg is the first argument that makes this binary a step's shut-off process: the integration level's, and, when +// the merge changes this module, the candidate's program running a digest or a level (ADR 0535 §1, candidate.go). const levelArg = "level" -// levelSpec is what the shut-off process is asked: the workspace, the packages and the -run pattern, the level's bounds. +// The steps a shut-off process runs (levelSpec.Step); empty is the integration level, as a spec said before the runner. +const ( + stepDigest = "digest" + stepUnit = LevelUnit + stepIntegration = LevelIntegration +) + +// levelSpec is what the shut-off process is asked: the step, the workspace, the packages and the -run pattern, the +// level's bounds; for a digest, its level. type levelSpec struct { + Step string `json:"step,omitempty"` + Level string `json:"level,omitempty"` Work string `json:"work"` Cache string `json:"cache"` Workspace string `json:"workspace"` @@ -32,10 +43,11 @@ type levelSpec struct { Bound time.Duration `json:"bound"` } -// levelAnswer is the one line the process prints: the level's result, or why it could not run. +// levelAnswer is the one line the process prints: the level's result or the digest, or why it could not run. type levelAnswer struct { - Result *TestResult `json:"result,omitempty"` - Error string `json:"error,omitempty"` + Result *TestResult `json:"result,omitempty"` + Digest *DigestResult `json:"digest,omitempty"` + Error string `json:"error,omitempty"` } // executable is this program's path, bound read-only into the step: Executable when a test set it. @@ -50,35 +62,63 @@ func (m *Module) executable() (string, error) { return filepath.EvalSymlinks(exe) } -// integrationShutOff runs the integration level in one shut-off process and reads its answer. +// integrationShutOff runs the integration level in one shut-off process of this program and reads its answer. func (m *Module) integrationShutOff(ctx context.Context, workspace string, packages []string, pattern string) (*TestResult, error) { exe, err := m.executable() if err != nil { return nil, fmt.Errorf("the integration level's program: %w", err) } b := m.integrationBounds() - spec, err := json.Marshal(levelSpec{Work: m.Work, Cache: m.Cache, Workspace: workspace, Packages: packages, Run: pattern, Setup: b.Setup, Bound: b.Run}) + a, err := m.shutOff(ctx, exe, levelSpec{Step: stepIntegration, Workspace: workspace, Packages: packages, Run: pattern, Setup: b.Setup, Bound: b.Run}) if err != nil { return nil, err } + if a.Result == nil { + return nil, fmt.Errorf("the integration level's shut-off process answered no result") + } + return a.Result, nil +} + +// shutOff runs the program exe as one shut-off process of the workspace (`exe level `) and reads its answer: the +// workspace read-write, the program alone read-only, the shared caches beneath a throwaway layer, the Node build +// module's toolchains read-only for the integration level. +func (m *Module) shutOff(ctx context.Context, exe string, spec levelSpec) (levelAnswer, error) { + spec.Work, spec.Cache = m.Work, m.Cache + raw, err := json.Marshal(spec) + if err != nil { + return levelAnswer{}, err + } for _, c := range m.caches() { if err := os.MkdirAll(c, 0o700); err != nil { - return nil, fmt.Errorf("the shared cache %s: %w", c, err) + return levelAnswer{}, fmt.Errorf("the shared cache %s: %w", c, err) } } - iso := m.isolation(workspace, false) + iso := m.isolation(spec.Workspace, false) iso.ReadOnly = append(iso.ReadOnly, exe) // the program alone, not the directory it lies in - iso.ReadOnly = append(iso.ReadOnly, m.nodeToolchains()...) - r := buildstep.Isolated(m.Run, iso)(ctx, nil, exe, levelArg, string(spec)) + if spec.Step == stepIntegration || spec.Step == "" { + iso.ReadOnly = append(iso.ReadOnly, m.nodeToolchains()...) + } + r := buildstep.Isolated(m.Run, iso)(ctx, nil, exe, levelArg, string(raw)) var answer levelAnswer line := lastLine(r.Stdout) - if err := json.Unmarshal([]byte(line), &answer); err != nil || (answer.Result == nil && answer.Error == "") { - return nil, fmt.Errorf("the integration level's shut-off process answered nothing it says (status %d %s): %s", r.Status, r.Err, firstLine(r.Stderr+"\n"+r.Stdout)) + if err := json.Unmarshal([]byte(line), &answer); err != nil || (answer.Result == nil && answer.Digest == nil && answer.Error == "") { + return levelAnswer{}, fmt.Errorf("the %s step's shut-off process answered nothing it says (status %d %s): %s", stepName(spec.Step), r.Status, r.Err, firstLine(r.Stderr+"\n"+r.Stdout)) } if answer.Error != "" { - return nil, fmt.Errorf("%s", answer.Error) + return levelAnswer{}, fmt.Errorf("%s", answer.Error) } - return answer.Result, nil + return answer, nil +} + +// stepName names a spec's step in words. +func stepName(step string) string { + switch step { + case stepDigest: + return "digest" + case stepUnit: + return "unit level's" + } + return "integration level's" } // NodeToolchains is the Node build module's toolchains directory on this node: its cache directory (module.json @@ -124,18 +164,40 @@ func runLevelProcess(args []string) int { if !workspaceOK.MatchString(spec.Workspace) { return say(levelAnswer{Error: fmt.Sprintf("%q is not a workspace id", spec.Workspace)}) } - m := &Module{Inside: true, Recipe: Current(), Work: spec.Work, Cache: spec.Cache, Home: os.Getenv("HOME"), + m := &Module{Inside: true, Recipe: Current(), Work: spec.Work, Cache: spec.Cache, Home: os.Getenv("HOME"), UnitBound: spec.Bound, IntegrationBounds: Bounds{Setup: spec.Setup, Run: spec.Bound}, Say: func(string, ...any) {}} m.Run = StepRunner(m.Home) - ctx := context.Background() - major, err := m.checkPostgres(ctx) - if err != nil { - return say(levelAnswer{Error: err.Error()}) + return say(m.answer(context.Background(), spec)) +} + +// answer runs the spec's step in this process, already shut off: a digest of a level, the unit level or the integration +// level of the workspace. +func (m *Module) answer(ctx context.Context, spec levelSpec) levelAnswer { + tree := filepath.Join(m.Work, spec.Workspace, "tree") + switch spec.Step { + case stepDigest: + d, err := m.digestHere(ctx, spec.Workspace, spec.Level) + if err != nil { + return levelAnswer{Error: err.Error()} + } + return levelAnswer{Digest: &d} + case stepUnit: + res, err := m.runLevel(ctx, tree, levelOpts{workspace: spec.Workspace, level: LevelUnit, packages: spec.Packages, run: spec.Run, + bounds: Bounds{Run: m.unitBound()}, env: m.besideEnv(spec.Workspace)}) + if err != nil { + return levelAnswer{Error: err.Error()} + } + return levelAnswer{Result: res} + case stepIntegration, "": + major, err := m.checkPostgres(ctx) + if err != nil { + return levelAnswer{Error: err.Error()} + } + res, err := m.integrationHere(ctx, spec.Workspace, tree, spec.Packages, spec.Run, major) + if err != nil { + return levelAnswer{Error: err.Error()} + } + return levelAnswer{Result: res} } - tree := filepath.Join(spec.Work, spec.Workspace, "tree") - res, err := m.integrationHere(ctx, spec.Workspace, tree, spec.Packages, spec.Run, major) - if err != nil { - return say(levelAnswer{Error: err.Error()}) - } - return say(levelAnswer{Result: res}) + return levelAnswer{Error: fmt.Sprintf("the step %q is not digest, unit or integration", spec.Step)} } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/levels.go b/modules/mesh-build-go/cmd/mesh-build-go/levels.go index 9631252a..455ca95f 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/levels.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/levels.go @@ -83,6 +83,9 @@ type TestResult struct { OverWhich string `json:"over_which,omitempty"` Store string `json:"store,omitempty"` Bus string `json:"bus,omitempty"` + // JudgedBy is the program that ran the level (ADR 0535 §5): "installed", or "candidate " under the + // runner workspace. + JudgedBy string `json:"judged_by,omitempty"` } // event is one line of `go test -json` (test2json, and the build events go test adds). diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test.go b/modules/mesh-build-go/cmd/mesh-build-go/test.go index 5add8055..c2b8167b 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test.go @@ -37,7 +37,8 @@ func (m *Module) testTool() StepTool { "level": map[string]any{"type": "string", "enum": []string{LevelUnit, LevelIntegration}, "description": "the level"}, "packages": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "description": "package patterns of each module, ./... when none"}, - "run": map[string]any{"type": "string", "description": "optional go test -run pattern: only the tests it matches run (a try's tests)"}, + "run": map[string]any{"type": "string", "description": "optional go test -run pattern: only the tests it matches run (a try's tests)"}, + "runner": runnerProperty, }, "required": []string{"workspace", "level"}, }, @@ -72,13 +73,48 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) { if _, err := os.Stat(tree); err != nil { return nil, fmt.Errorf("no workspace %s: it was never fetched here, or it is older than a day and was removed", workspace) } - switch level { - case LevelUnit: - return m.runLevel(ctx, tree, levelOpts{workspace: workspace, level: LevelUnit, packages: packages, run: pattern, bounds: Bounds{Run: m.unitBound()}, env: m.besideEnv(workspace)}) - case LevelIntegration: - return m.integration(ctx, workspace, tree, packages, pattern) + runner, err := runnerArg(args["runner"]) + if err != nil { + return nil, err } - return nil, fmt.Errorf("the level %q is neither unit nor integration", level) + var res *TestResult + switch { + case runner == RunnerWorkspace && (level == LevelUnit || level == LevelIntegration): + return m.testAsCandidate(ctx, workspace, level, packages, pattern) + case level == LevelUnit: + res, err = m.runLevel(ctx, tree, levelOpts{workspace: workspace, level: LevelUnit, packages: packages, run: pattern, bounds: Bounds{Run: m.unitBound()}, env: m.besideEnv(workspace)}) + case level == LevelIntegration: + res, err = m.integration(ctx, workspace, tree, packages, pattern) + default: + return nil, fmt.Errorf("the level %q is neither unit nor integration", level) + } + if err != nil { + return nil, err + } + res.JudgedBy = RunnerInstalled + return res, nil +} + +// testAsCandidate runs a level as the candidate's program (ADR 0535 §1): the integration level is refused first as the +// installed program would refuse it (no postgres), then the whole level runs in the candidate's one shut-off process. +func (m *Module) testAsCandidate(ctx context.Context, workspace, level string, packages []string, pattern string) (*TestResult, error) { + spec := levelSpec{Step: stepUnit, Workspace: workspace, Packages: packages, Run: pattern, Bound: m.unitBound()} + if level == LevelIntegration { + if _, err := m.checkPostgres(ctx); err != nil { + return nil, err // a *buildstep.Refusal: this level only + } + b := m.integrationBounds() + spec = levelSpec{Step: stepIntegration, Workspace: workspace, Packages: packages, Run: pattern, Setup: b.Setup, Bound: b.Run} + } + a, judged, err := m.asCandidate(ctx, spec) + if err != nil { + return nil, err + } + if a.Result == nil { + return nil, fmt.Errorf("the candidate's mesh-build-go answered no result of the %s level", level) + } + a.Result.JudgedBy = judged + return a.Result, nil } // besideEnv is MESH_CHECK_BESIDE, naming the directory build_go_fetch placed the workspace's siblings in (ADR 0484 From 29f4a366c25e87ca8505570aedb486ce6bc8983d Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 23 Oct 2026 14:16:40 +0200 Subject: [PATCH 07/10] Pin mesh-build-go to mesh-sdk 2248b985, the group's final SDK head with the 0.1.16 version (issues 890, 1899) --- modules/mesh-build-go/go.mod | 2 +- modules/mesh-build-go/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/mesh-build-go/go.mod b/modules/mesh-build-go/go.mod index 9973c007..e63fc0a5 100644 --- a/modules/mesh-build-go/go.mod +++ b/modules/mesh-build-go/go.mod @@ -3,7 +3,7 @@ module meshbuildgo go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023120306-780be2fae5a1 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023121526-2248b985bfac github.com/distribution/distribution/v3 v3.1.2 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/modules/mesh-build-go/go.sum b/modules/mesh-build-go/go.sum index e5951bbf..868be86a 100644 --- a/modules/mesh-build-go/go.sum +++ b/modules/mesh-build-go/go.sum @@ -4,6 +4,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467 h1:i85+fFT0 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023120306-780be2fae5a1 h1:MqP6mWWVxM+niUGPxSSh4Q8lduYkkQ7oTAkDeC9nyv4= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023120306-780be2fae5a1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023121526-2248b985bfac h1:xx601bHwpO/aq5KyE08t4ymSXGHGnk+D9VHfmOq+5P8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023121526-2248b985bfac/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI= github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= From cf071270a0b1cf5e7000c04282d60018af8e6c47 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 23 Oct 2026 14:41:00 +0200 Subject: [PATCH 08/10] Skip the docker tests where the daemon is unreachable, not only where the binary is missing The checker's integration level runs shut off, so docker is found but its daemon is not, and TestTheDeclaredDumpRunsInTheBussImage failed the group check on g14 (issue 1823). MESH_REQUIRE_DOCKER=1 still fails. --- .../mesh-nats-snapshot/dockerguard_test.go | 53 +++++++++++++++++++ .../nats/cmd/mesh-nats-snapshot/live_test.go | 24 ++++++--- 2 files changed, 70 insertions(+), 7 deletions(-) create mode 100644 modules/nats/cmd/mesh-nats-snapshot/dockerguard_test.go diff --git a/modules/nats/cmd/mesh-nats-snapshot/dockerguard_test.go b/modules/nats/cmd/mesh-nats-snapshot/dockerguard_test.go new file mode 100644 index 00000000..7605d15c --- /dev/null +++ b/modules/nats/cmd/mesh-nats-snapshot/dockerguard_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "fmt" + "testing" +) + +// dockerVerdict decides what a test that starts containers does: run, skip or fail. found is the result of looking +// the docker program up, probeErr the result of asking the daemon for its version (nil when it answered). The checker's +// integration level runs shut off (no network, no /run), so on a build node the program is found and the daemon is not +// reachable; that is a skip, not a failure, except where docker is required (MESH_REQUIRE_DOCKER=1). +func dockerVerdict(found bool, lookErr, probeErr error, required bool) (action, reason string) { + var why string + switch { + case !found: + why = fmt.Sprintf("there is no docker in this toolchain (%v)", lookErr) + case probeErr != nil: + why = fmt.Sprintf("no docker daemon is reachable from here (%v)", probeErr) + default: + return "run", "" + } + if required { + return "fail", "MESH_REQUIRE_DOCKER=1 and " + why + } + return "skip", why +} + +func TestDockerVerdict(t *testing.T) { + boom := fmt.Errorf("cannot connect") + cases := []struct { + name string + found bool + look, probe error + required bool + want string + }{ + {"answers", true, nil, nil, false, "run"}, + {"answers, required", true, nil, nil, true, "run"}, + {"no program", false, boom, nil, false, "skip"}, + {"no program, required", false, boom, nil, true, "fail"}, + {"program but no daemon", true, nil, boom, false, "skip"}, + {"program but no daemon, required", true, nil, boom, true, "fail"}, + } + for _, c := range cases { + got, why := dockerVerdict(c.found, c.look, c.probe, c.required) + if got != c.want { + t.Errorf("%s: got %s, want %s", c.name, got, c.want) + } + if got != "run" && why == "" { + t.Errorf("%s: no reason given", c.name) + } + } +} diff --git a/modules/nats/cmd/mesh-nats-snapshot/live_test.go b/modules/nats/cmd/mesh-nats-snapshot/live_test.go index 99af6599..a9918d6c 100644 --- a/modules/nats/cmd/mesh-nats-snapshot/live_test.go +++ b/modules/nats/cmd/mesh-nats-snapshot/live_test.go @@ -60,14 +60,24 @@ var snapshotGrants = struct{ publish, subscribe []string }{ func needDocker(t *testing.T) { t.Helper() - if _, err := exec.LookPath("docker"); err != nil { - // The Go toolchain container of the merge check holds no docker. Where docker is expected - // (MESH_REQUIRE_DOCKER=1, which merge-check.sh sets for its docker step) its absence is a failure. - if os.Getenv("MESH_REQUIRE_DOCKER") == "1" { - t.Fatalf("MESH_REQUIRE_DOCKER=1 and there is no docker here: %v", err) + _, lookErr := exec.LookPath("docker") + var probeErr error + if lookErr == nil { + // The binary alone proves nothing: the checker's integration level runs shut off (no network, no /run), so on + // a build node docker is found and its daemon is not reachable (issue 1823). + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if out, err := exec.CommandContext(ctx, "docker", "version", "--format", "{{.Server.Version}}").CombinedOutput(); err != nil { + probeErr = fmt.Errorf("%v: %s", err, strings.TrimSpace(string(out))) } - t.Skipf("NOT RUN: this test starts a throwaway nats container and there is no docker in this toolchain (%v); "+ - "it is proof only where merge-check.sh's docker step runs it (MESH_CHECK_DOCKER=1)", err) + } + // Where docker is expected (MESH_REQUIRE_DOCKER=1, which merge-check.sh sets for its docker step) its absence is a failure. + switch action, why := dockerVerdict(lookErr == nil, lookErr, probeErr, os.Getenv("MESH_REQUIRE_DOCKER") == "1"); action { + case "fail": + t.Fatal(why) + case "skip": + t.Skipf("NOT RUN: this test starts a throwaway nats container and %s; "+ + "it is proof only where merge-check.sh's docker step runs it (MESH_CHECK_DOCKER=1)", why) } } From ee176b76ea3ceaa09f0598c0e926ff15d10df877 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 23 Oct 2026 14:56:04 +0200 Subject: [PATCH 09/10] Wait 20 s, not 5 s, for each step of the gate's path over the bus: a loaded build node failed the ask wait twice (issue 1404) --- .../cmd/mesh-delivery-gate/bus_test.go | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go index 76f46626..c1458d38 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go @@ -520,7 +520,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB passed := b.publish(t, "mesh.seat."+ControllerSeat+".event.checked", judgedAt("novox/app", 7, head, "pass", "t1t1t1t1t1t1")) // No review names the head, so nothing is asked (ADR 0496, issue 1803): the delivery waits for a review. - rt.until("the delivery to wait for a review", 5*time.Second, func() bool { + rt.until("the delivery to wait for a review", eventWithin, func() bool { e, err := b.kvs["pulls"].Get(ctx, "novox.app.7") var k PullKey return err == nil && json.Unmarshal(e.Value(), &k) == nil && k.Verdict == "pass" @@ -537,7 +537,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB "title": announced.Title, "head_sha": head, "base": "main", "body": "Reviewed-by: lead@shanks at " + head}) _ = passed var key string - rt.until("the ask to be written", 5*time.Second, func() bool { key = onlyAsk(b); return key != "" }) + rt.until("the ask to be written", eventWithin, func() bool { key = onlyAsk(b); return key != "" }) ask := askAt(t, b, key) t.Logf("the ask was written %s after the edit's event", time.Since(edit)) if !strings.Contains(ask.Ask.Explanation, "reviewed by lead on shanks") || ask.Ask.ID != sdk.AskID(key) { @@ -547,7 +547,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB t.Fatalf("merged before the Approve") } // The pull request's key holds its head, the trunk commit it was asked against and the verdict. - rt.until("the pull request's key to hold head, trunk and verdict", 5*time.Second, func() bool { + rt.until("the pull request's key to hold head, trunk and verdict", eventWithin, func() bool { e, err := b.kvs["pulls"].Get(ctx, "novox.app.7") var k PullKey return err == nil && json.Unmarshal(e.Value(), &k) == nil && k == PullKey{Head: head, Trunk: "t1t1t1t1t1t1", Verdict: "pass"} @@ -559,7 +559,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB t.Fatalf("the gate asked %s %d times: the bridge writes to the forge", tool, n) } } - rt.until("the owed keys to be written", 5*time.Second, func() bool { + rt.until("the owed keys to be written", eventWithin, func() bool { keys, err := b.kvs["owed"].Keys(ctx) return err == nil && len(keys) > 0 }) @@ -567,7 +567,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB // seconds, with the Approve's heads, and the gate asks the forge nothing. approved := b.answer(t, key, OptionApprove) var handed sdk.Merge - rt.until("the merge to be handed to the bridge", 5*time.Second, func() bool { handed = mergeAt(b, "fix/x"); return len(handed.Pulls) > 0 }) + rt.until("the merge to be handed to the bridge", eventWithin, func() bool { handed = mergeAt(b, "fix/x"); return len(handed.Pulls) > 0 }) if len(handed.Pulls) != 1 || handed.Pulls[0].Head.Head != head || handed.Pulls[0].Number != 7 || handed.Pulls[0].State != sdk.MergeMerging || handed.Approved[sdk.ApprovedKey("novox/app", 7)] != head || handed.Ask != key { t.Fatalf("the merge handed is %+v; wanted pull request 7 at the approved head, merging, under the ask %s", handed, key) @@ -579,7 +579,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB // The bridge says it merged; the mesh's source then holds the merge, in one write. b.forgeSays(t, sdk.Forge{Kind: sdk.ForgeMerged, Repository: "novox/app", Number: 7, Head: head, Merge: strings.Repeat("b", 40), Root: "r", Trees: map[string]string{"app": "t"}}) - rt.until("source.main to hold the merge", 5*time.Second, func() bool { + rt.until("source.main to hold the merge", eventWithin, func() bool { e, err := b.kvs["source"].Get(ctx, sdk.SourceMainKey) if err != nil { return false @@ -743,7 +743,7 @@ func TestAGroupOfTwoWithGreenChecksGetsOneAskAndOnItsApproveBothMergeOneAfterThe // One member is handed, with both heads approved, within seconds of the answer; the second only after the bridge says the // first merged. var first sdk.Merge - rt.until("the first member to be handed", 5*time.Second, func() bool { first = mergeAt(b, "feat/x"); return len(first.Pulls) > 0 }) + rt.until("the first member to be handed", eventWithin, func() bool { first = mergeAt(b, "feat/x"); return len(first.Pulls) > 0 }) if len(first.Pulls) != 1 || first.Pulls[0].State != sdk.MergeMerging || len(first.Approved) != 2 || first.Approved[sdk.ApprovedKey("novox/mesh-controller", 1)] != ctlHead || first.Approved[sdk.ApprovedKey("novox/mesh-catalog", 2)] != catHead { t.Fatalf("the first merge handed is %+v", first) @@ -757,14 +757,14 @@ func TestAGroupOfTwoWithGreenChecksGetsOneAskAndOnItsApproveBothMergeOneAfterThe b.forgeSays(t, sdk.Forge{Kind: sdk.ForgeMerged, Repository: one.Repository, Number: one.Number, Head: one.Head.Head, Merge: strings.Repeat("a", 40), Root: "r", Trees: map[string]string{"x": "t"}}) var both sdk.Merge - rt.until("the second member to be handed", 5*time.Second, func() bool { both = mergeAt(b, "feat/x"); return len(both.Pulls) == 2 }) + rt.until("the second member to be handed", eventWithin, func() bool { both = mergeAt(b, "feat/x"); return len(both.Pulls) == 2 }) if both.Pulls[0].State != sdk.MergeMerged || both.Pulls[1].State != sdk.MergeMerging || both.Pulls[1].Repository == one.Repository { t.Fatalf("the second merge handed is %+v", both.Pulls) } two := both.Pulls[1] b.forgeSays(t, sdk.Forge{Kind: sdk.ForgeMerged, Repository: two.Repository, Number: two.Number, Head: two.Head.Head, Merge: strings.Repeat("b", 40), Root: "r", Trees: map[string]string{"x": "t"}}) - rt.until("source.main to hold both merges", 5*time.Second, func() bool { + rt.until("source.main to hold both merges", eventWithin, func() bool { e, err := b.kvs["source"].Get(ctx, sdk.SourceMainKey) if err != nil { return false @@ -942,3 +942,10 @@ func mustDecode[T any](t *testing.T, raw []byte) T { } return v } + +// eventWithin is how long a test waits for one step of the gate's path over the real bus. On an idle machine each step +// takes milliseconds (the tests log it), but the build node runs the whole group's checks at once, and a step that +// fails its first try (a key rewritten under it, a request that timed out) is tried again by the gate only after +// retryFirst (10 s, effects.go), which the test cannot shorten without changing production code. 20 s covers that one +// retry plus the load; "within seconds" stays true and a stuck path still fails (issue 1404). +const eventWithin = 20 * time.Second From e9ce56e8386100a93e4fc5178f5e3ba13462cb04 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 23 Oct 2026 14:58:14 +0200 Subject: [PATCH 10/10] Say in the test why the waits are 20 s: load, not a retry (issue 1404) --- .../mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go index c1458d38..77bc463a 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go @@ -944,8 +944,7 @@ func mustDecode[T any](t *testing.T, raw []byte) T { } // eventWithin is how long a test waits for one step of the gate's path over the real bus. On an idle machine each step -// takes milliseconds (the tests log it), but the build node runs the whole group's checks at once, and a step that -// fails its first try (a key rewritten under it, a request that timed out) is tried again by the gate only after -// retryFirst (10 s, effects.go), which the test cannot shorten without changing production code. 20 s covers that one -// retry plus the load; "within seconds" stays true and a stuck path still fails (issue 1404). +// takes milliseconds (the tests log it), but a slow, loaded build node ran the ask step in 6.69 s against the old 5 s +// (issue 1404). A failed first ask is asked again only after askAgainAfter (30 s, react.go), and 20 s deliberately does not +// cover that: a failed first ask still fails the test. const eventWithin = 20 * time.Second