diff --git a/modules/mesh-build-go/README.md b/modules/mesh-build-go/README.md index 69d8cfa2..f2493e35 100644 --- a/modules/mesh-build-go/README.md +++ b/modules/mesh-build-go/README.md @@ -105,6 +105,16 @@ fingerprint once, as the version the build nodes agree on (ADR 0477 §3), and th version's. A package with no tests at the level has no digest and is named in `without_tests`. A directory that vendors is listed and tested from its vendor directory (`-mod=vendor`). +## A change to this module is judged by its candidate + +`build_go_digest` and `build_go_test` take `runner`: `installed` (the default) runs the step as this program; +`workspace` compiles the workspace's own `modules/mesh-build-go` (`go build` with `GOPROXY=off`, `GOTOOLCHAIN=local`, in a +run step shut off) into the workspace's `candidate/mesh-build-go`, once per workspace, and runs the step as that program: +` level ` in one bubblewrap, the spec naming the step (`digest`, `unit` or `integration`). This program +keeps the fetch, the forge key and the bus login (ADR 0535 §1). A candidate that does not compile fails the step with the +compiler's first error, and nothing runs (§3). Every digest and level answer says `judged_by`: `installed`, or +`candidate `. The checker asks `workspace` when a merge changes `modules/mesh-build-go/` (issue 1898). + ## Build `build_go_build {workspace, module, artifact}` builds one Go bundle artifact of a module of the workspace with the build diff --git a/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go index 9f14bd9c..3682a61b 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go @@ -11,37 +11,72 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" ) -// The lead's decision L5 (the checker keeps a verdict of a core repository): the fetch writes the siblings it placed -// beside the workspace, and the digest folds each one's repository and commit into every package's digest and echoes -// them, so a verdict is kept only for the siblings it was judged with. -func TestTheSiblingsCommitsAreInEveryDigestAndEchoed(t *testing.T) { +// writeSiblings records the siblings the fetch placed, as the fetch does. +func writeSiblings(t *testing.T, m *Module, placed []Sibling) { + t.Helper() + raw, _ := json.Marshal(placed) + if err := os.WriteFile(filepath.Join(m.Work, "ws1", besideRecord), raw, 0o600); err != nil { + t.Fatal(err) + } +} + +// Issue 1896/AC1 and AC3 (ADR 0536 §1): a sibling's commit changes no digest of a package whose tests reach no file +// naming MESH_CHECK_BESIDE; the answer still echoes every placed sibling, in form 4. +func TestASiblingsCommitLeavesAPackageThatDoesNotReadTheSiblingsAsItWas(t *testing.T) { t.Parallel() m, _ := digestFixture(t) alone := digests(t, m, "unit") if len(alone.Beside) != 0 { t.Errorf("a workspace without siblings echoes %v", alone.Beside) } - write := func(placed []Sibling) { - raw, _ := json.Marshal(placed) - if err := os.WriteFile(filepath.Join(m.Work, "ws1", besideRecord), raw, 0o600); err != nil { - t.Fatal(err) - } - } other := strings.Repeat("b", 40) - write([]Sibling{{Repository: "novox/mesh-sdk", Commit: commitSHA, Dir: "mesh-sdk"}, {Repository: "novox/mesh-catalog", Commit: commitSHA, Dir: "mesh-catalog"}}) + writeSiblings(t, m, []Sibling{{Repository: "novox/mesh-sdk", Commit: commitSHA, Dir: "mesh-sdk"}, {Repository: "novox/mesh-catalog", Commit: commitSHA, Dir: "mesh-catalog"}}) with := digests(t, m, "unit") want := []string{"novox/mesh-catalog " + commitSHA, "novox/mesh-sdk " + commitSHA} if !slices.Equal(with.Beside, want) { t.Errorf("echoed %v, want %v", with.Beside, want) } - write([]Sibling{{Repository: "novox/mesh-sdk", Commit: other, Dir: "mesh-sdk"}, {Repository: "novox/mesh-catalog", Commit: commitSHA, Dir: "mesh-catalog"}}) + writeSiblings(t, m, []Sibling{{Repository: "novox/mesh-sdk", Commit: other, Dir: "mesh-sdk"}, {Repository: "novox/mesh-catalog", Commit: commitSHA, Dir: "mesh-catalog"}}) moved := digests(t, m, "unit") for p, d := range alone.Packages { - if with.Packages[p] == d { - t.Errorf("%s: the siblings left the digest as it was without them", p) + if with.Packages[p] != d || moved.Packages[p] != d { + t.Errorf("%s: a sibling left its digest changed though no test of it reads the siblings", p) } - if moved.Packages[p] == with.Packages[p] { - t.Errorf("%s: a sibling's new commit left the digest as it was", p) + } + if !slices.Equal(moved.Beside, []string{"novox/mesh-catalog " + commitSHA, "novox/mesh-sdk " + other}) { + t.Errorf("echoed %v after the move", moved.Beside) + } +} + +// Issue 1896/AC2: a package whose test imports an in-tree package naming MESH_CHECK_BESIDE is judged again when a +// sibling moves; a package beside it that does not import it is not. +func TestASiblingsCommitChangesTheDigestOfAPackageWhoseTestsReachAReaderOfTheSiblings(t *testing.T) { + t.Parallel() + m, tree := digestFixture(t) + for name, text := range map[string]string{ + "f/f.go": "package f\n\n// Beside is where the siblings lie.\nconst Beside = \"MESH_CHECK_BESIDE\"\n", + "e/e_test.go": "package e\n\nimport (\n\t\"testing\"\n\n\t\"fixture/digest/f\"\n)\n\nfunc TestE(t *testing.T) {\n\tif f.Beside == \"\" {\n\t\tt.Fatal(\"e\")\n\t}\n}\n", + "e/e.go": "package e\n", + "g/g_test.go": "package g\n\nimport \"testing\"\n\nfunc TestG(t *testing.T) {\n\t// MESH_CHECK_BESIDE named in the test's own file\n}\n", + "g/g.go": "package g\n", + } { + if err := os.MkdirAll(filepath.Dir(filepath.Join(tree, name)), 0o755); err != nil { + t.Fatal(err) + } + edit(t, tree, name, text) + } + writeSiblings(t, m, []Sibling{{Repository: "novox/mesh-sdk", Commit: commitSHA, Dir: "mesh-sdk"}}) + before := digests(t, m, "unit") + writeSiblings(t, m, []Sibling{{Repository: "novox/mesh-sdk", Commit: strings.Repeat("b", 40), Dir: "mesh-sdk"}}) + after := digests(t, m, "unit") + for _, p := range []string{"fixture/digest/e", "fixture/digest/g"} { + if len(before.Packages[p]) != 64 || before.Packages[p] == after.Packages[p] { + t.Errorf("%s reads the siblings and its digest stayed (%q, %q)", p, before.Packages[p], after.Packages[p]) + } + } + for _, p := range []string{"fixture/digest/a", "fixture/digest/c"} { + if before.Packages[p] != after.Packages[p] { + t.Errorf("%s does not read the siblings and its digest changed", p) } } } @@ -81,7 +116,7 @@ func TestTheIntegrationLevelDigestsUnderBothTags(t *testing.T) { t.Errorf("the integration digest listed %q", s.command) } } - if DigestVersion != "3" { + if DigestVersion != "4" { t.Errorf("the digest's form changed in kind and is still version %s", DigestVersion) } } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/candidate.go b/modules/mesh-build-go/cmd/mesh-build-go/candidate.go new file mode 100644 index 00000000..38a27ad6 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/candidate.go @@ -0,0 +1,132 @@ +package main + +// A change to this module is judged by its own candidate (novox/hq ADR 0535; issue 1898): asked with the runner +// workspace, the digest and test steps compile the program of mesh-build-go from the workspace's merged tree in a run +// step, shut off, and run the step as that program in one shut-off process (` level `, level.go), where +// this program would run the step's commands. This program keeps the fetch, the forge key and the bus login; the +// candidate's program holds none of them. Each answer says which program ran it (judged_by). + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "sync" +) + +// The runners of a digest or test step: this installed program, or the candidate's compiled from the workspace. +const ( + RunnerInstalled = "installed" + RunnerWorkspace = "workspace" +) + +// runnerProperty is the runner argument as the digest and test tools announce it: a checker reads it there, and an +// installed module that does not announce it judges as it always did (ADR 0535 §2). +var runnerProperty = map[string]any{"type": "string", "enum": []string{RunnerInstalled, RunnerWorkspace}, + "description": "optional: installed (the default) runs the step as this installed program; workspace compiles the " + + "workspace's own mesh-build-go (modules/mesh-build-go of the merged tree) shut off and runs the step as it, for a " + + "change to this module (ADR 0535); the answer's judged_by names the program that ran"} + +// candidateModule is the directory of this module's go.mod in the catalogue's tree, and candidateMain its program's +// package there. +const ( + candidateModule = "modules/mesh-build-go" + candidateMain = "./cmd/mesh-build-go" +) + +// runnerArg reads the runner argument: "" (the installed program), installed or workspace. +func runnerArg(v any) (string, error) { + if v == nil { + return "", nil + } + s, ok := v.(string) + if !ok || (s != "" && s != RunnerInstalled && s != RunnerWorkspace) { + return "", fmt.Errorf("the runner %v is neither %s nor %s", v, RunnerInstalled, RunnerWorkspace) + } + return s, nil +} + +// workspaceThere refuses a workspace id that is not one, or a workspace this node never fetched. +func (m *Module) workspaceThere(workspace string) error { + if !workspaceOK.MatchString(workspace) { + return fmt.Errorf("%q is not a workspace id", workspace) + } + if _, err := os.Stat(filepath.Join(m.Work, workspace, "tree")); err != nil { + return fmt.Errorf("no workspace %s: it was never fetched here, or it is older than a day and was removed", workspace) + } + return nil +} + +// candidateLocks hold one mutex per workspace, so its candidate is compiled once. +var candidateLocks sync.Map + +// candidate is the program of mesh-build-go compiled from the workspace's tree, in the workspace beside its tree +// (candidate/mesh-build-go), compiled once per workspace, in a run step shut off with the caches read-only and nothing +// fetched (GOPROXY=off: the fetch step downloaded every module the tree's go.mod files name). A candidate that does not +// compile is the step's failure, naming the compiler's first error (ADR 0535 §3). +func (m *Module) candidate(ctx context.Context, workspace string) (string, error) { + exe := filepath.Join(m.Work, workspace, "candidate", "mesh-build-go") + mu, _ := candidateLocks.LoadOrStore(exe, &sync.Mutex{}) + mu.(*sync.Mutex).Lock() + defer mu.(*sync.Mutex).Unlock() + if st, err := os.Stat(exe); err == nil && st.Mode().IsRegular() { + return exe, nil + } + dir := filepath.Join(m.Work, workspace, "tree", candidateModule) + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err != nil { + return "", fmt.Errorf("the workspace holds no %s/go.mod, so it has no candidate mesh-build-go to run", candidateModule) + } + if err := os.MkdirAll(filepath.Dir(exe), 0o700); err != nil { + return "", err + } + run, err := m.stepRunner(workspace, false) + if err != nil { + return "", err + } + env := append(m.goEnvFor(dir), "GOPROXY=off", "GOTOOLCHAIN=local") + r := run(ctx, env, "go", "-C", dir, "build", "-o", exe, candidateMain) + if r.Status != 0 || r.Err != "" { + return "", fmt.Errorf("the candidate's mesh-build-go does not compile: %s", compileError(r.Err+"\n"+r.Stderr+"\n"+r.Stdout)) + } + return exe, nil +} + +// compileError is the first line of the compiler's output that says an error: not the `# package` heading go build +// prints above a package's errors, nor Go's progress. +func compileError(out string) string { + for _, l := range strings.Split(out, "\n") { + if l = strings.TrimSpace(l); l != "" && !strings.HasPrefix(l, "#") && !strings.HasPrefix(l, "go: downloading ") { + return l + } + } + return firstLine(out) +} + +// mergeCommit is the commit the workspace's tree holds: the merge the fetch made, or the head it fetched. +func (m *Module) mergeCommit(ctx context.Context, workspace string) (string, error) { + r := m.Run(ctx, nil, "git", "-C", filepath.Join(m.Work, workspace, "tree"), "rev-parse", "HEAD") + commit := strings.TrimSpace(r.Stdout) + if r.Status != 0 || r.Err != "" || commit == "" { + return "", fmt.Errorf("the workspace's commit: git rev-parse: %s", firstLine(r.Err+"\n"+r.Stderr)) + } + return commit, nil +} + +// asCandidate runs the spec's step as the candidate's program, shut off, and answers what it said and the judged_by of +// the answer: "candidate ". +func (m *Module) asCandidate(ctx context.Context, spec levelSpec) (levelAnswer, string, error) { + commit, err := m.mergeCommit(ctx, spec.Workspace) + if err != nil { + return levelAnswer{}, "", err + } + exe, err := m.candidate(ctx, spec.Workspace) + if err != nil { + return levelAnswer{}, "", err + } + a, err := m.shutOff(ctx, exe, spec) + if err != nil { + return levelAnswer{}, "", fmt.Errorf("the candidate's mesh-build-go: %w", err) + } + return a, "candidate " + commit, nil +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/candidate_integration_test.go b/modules/mesh-build-go/cmd/mesh-build-go/candidate_integration_test.go new file mode 100644 index 00000000..fe6dc280 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/candidate_integration_test.go @@ -0,0 +1,127 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// candidateMainSource is a candidate's program that answers a mark: asked `level `, it tries what a run step must +// not reach (the account's key and login, a name, a socket of the machine) and answers a passing level of 42 tests whose +// why is "mark" and what it reached. +const candidateMainSource = `package main + +import ( + "encoding/json" + "fmt" + "net" + "os" + "time" +) + +func main() { + if len(os.Args) != 3 || os.Args[1] != "level" { + os.Exit(2) + } + seen := map[string]bool{} + _, err := os.ReadFile(%q) + seen["key"] = err == nil + _, err = os.ReadFile(%q) + seen["login"] = err == nil + _, err = net.LookupHost("git.novox.be") + seen["resolved"] = err == nil + c, err := net.DialTimeout("tcp", %q, 2*time.Second) + if err == nil { + c.Close() + } + seen["tcp"] = err == nil + raw, _ := json.Marshal(seen) + var spec struct { + Step string ` + "`json:\"step\"`" + ` + } + json.Unmarshal([]byte(os.Args[2]), &spec) + out, _ := json.Marshal(map[string]any{"result": map[string]any{"level": spec.Step, "passed": true, "tests": 42, + "packages": []any{}, "why": "mark " + string(raw)}}) + fmt.Println(string(out)) +} +` + +// 1898/AC4 (ADR 0535 §1): a tree whose mesh-build-go program answers a mark: the unit level with the runner workspace +// compiles that program and runs it shut off, answers its mark judged by the candidate at the tree's commit, and the +// program reached no key, no login, no name and no socket of the machine. +func TestTheUnitLevelUnderTheWorkspaceRunnerRunsTheTreesProgramShutOff(t *testing.T) { + needTools(t, "the candidate's program shut off", "go", "git", "bwrap") + home := t.TempDir() + key, login := filepath.Join(home, ".ssh", "mesh_ed25519"), filepath.Join(home, "broker") + for _, f := range []string{key, login} { + if err := os.MkdirAll(filepath.Dir(f), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(f, []byte("secret"), 0o600); err != nil { + t.Fatal(err) + } + } + tcp, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer tcp.Close() + + work, err := os.MkdirTemp("", "w") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(work) }) + tree := filepath.Join(work, "ws1", "tree") + dir := filepath.Join(tree, candidateModule) + if err := os.MkdirAll(filepath.Join(dir, "cmd", "mesh-build-go"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module meshbuildgo\n\ngo 1.25.0\n"), 0o644); err != nil { + t.Fatal(err) + } + src := fmt.Sprintf(candidateMainSource, key, login, tcp.Addr().String()) + if err := os.WriteFile(filepath.Join(dir, "cmd", "mesh-build-go", "main.go"), []byte(src), 0o644); err != nil { + t.Fatal(err) + } + for _, c := range [][]string{{"init", "-q"}, {"add", "-A"}, {"-c", "user.email=a@b", "-c", "user.name=x", "commit", "-qm", "x"}} { + if out, err := exec.Command("git", append([]string{"-C", tree}, c...)...).CombinedOutput(); err != nil { + t.Fatalf("git %v: %v %s", c, err, out) + } + } + head, err := exec.Command("git", "-C", tree, "rev-parse", "HEAD").Output() + if err != nil { + t.Fatal(err) + } + + m := &Module{Recipe: Current(), Home: home, Work: work, Cache: t.TempDir()} + m.Run = StepRunner(home) + out, err := m.test(context.Background(), map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": RunnerWorkspace}) + if err != nil { + t.Fatal(err) + } + res := out.(*TestResult) + if res.Tests != 42 || !strings.HasPrefix(res.Why, "mark ") { + t.Fatalf("result = %+v, want the candidate's mark", res) + } + if want := "candidate " + strings.TrimSpace(string(head)); res.JudgedBy != want { + t.Errorf("judged by %q, want %q", res.JudgedBy, want) + } + var seen map[string]bool + if err := json.Unmarshal([]byte(strings.TrimPrefix(res.Why, "mark ")), &seen); err != nil || len(seen) != 4 { + t.Fatalf("the program said %q: %v", res.Why, err) + } + for what, reached := range seen { + if reached { + t.Errorf("the candidate's program reached %s", what) + } + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/candidate_test.go b/modules/mesh-build-go/cmd/mesh-build-go/candidate_test.go new file mode 100644 index 00000000..cf12dadd --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/candidate_test.go @@ -0,0 +1,262 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// Issue 1898 K1, K2 (ADR 0535): asked with the runner workspace, the digest and test steps compile the workspace's own +// mesh-build-go shut off and run the step as it, in one shut-off process; the answer says which program ran. + +const candidateSHA = "fedcba9876543210fedcba9876543210fedcba98" + +// candidateRig is a module over a workspace whose tree holds modules/mesh-build-go, with a runner standing for git, go +// and the candidate's program; what ran shut off is in seen, every command in ran. +type candidateRig struct { + m *Module + states *keptStates + seen []shutOff + ran []string + compile forgekey.Ran // what go build answers; Status 0 writes the program + answer func(spec levelSpec) levelAnswer // what the candidate's program answers + specs []levelSpec + compiled int +} + +func newCandidateRig(t *testing.T) *candidateRig { + t.Helper() + cr := &candidateRig{states: newStates()} + cr.m = moduleWith(cr.states, "go1.27.1") + cr.m.Home, cr.m.Work, cr.m.Cache = t.TempDir(), t.TempDir(), t.TempDir() + cr.m.Executable = "/opt/bundle/mesh-build-go" + dir := filepath.Join(cr.m.Work, "ws1", "tree", candidateModule) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module meshbuildgo\n\ngo 1.26.0\n"), 0o644); err != nil { + t.Fatal(err) + } + cr.answer = func(spec levelSpec) levelAnswer { + if spec.Step == stepDigest { + return levelAnswer{Digest: &DigestResult{Level: spec.Level, Packages: map[string]string{"meshbuildgo/cmd/mesh-build-go": "mark"}}} + } + return levelAnswer{Result: &TestResult{Level: spec.Step, Passed: true, Tests: 42, Packages: []PackageResult{}, Why: "mark"}} + } + exe := filepath.Join(cr.m.Work, "ws1", "candidate", "mesh-build-go") + cr.m.Run = throughBubblewrap(t, func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + line := name + " " + strings.Join(args, " ") + cr.ran = append(cr.ran, line) + switch { + case name == "git" && slices.Contains(args, "rev-parse"): + return forgekey.Ran{Stdout: candidateSHA + "\n"} + case name == "git" && slices.Contains(args, "ls-files"): + return forgekey.Ran{Stdout: "go.mod\n"} + case name == "postgres": + return forgekey.Ran{Stdout: "postgres (PostgreSQL) 18.6\n"} + case name == "go" && slices.Contains(args, "build"): + cr.compiled++ + for _, want := range []string{"GOPROXY=off", "GOTOOLCHAIN=local", "GOFLAGS=-mod=mod"} { + if !slices.Contains(env, want) { + t.Errorf("the candidate compiled without %s: %v", want, env) + } + } + if cr.compile.Status == 0 && cr.compile.Err == "" { + if err := os.WriteFile(args[slices.Index(args, "-o")+1], []byte("program"), 0o755); err != nil { + t.Fatal(err) + } + } + return cr.compile + case name == exe && len(args) == 2 && args[0] == levelArg: + var spec levelSpec + if err := json.Unmarshal([]byte(args[1]), &spec); err != nil { + t.Errorf("the spec %s: %v", args[1], err) + } + cr.specs = append(cr.specs, spec) + raw, _ := json.Marshal(cr.answer(spec)) + return forgekey.Ran{Stdout: string(raw) + "\n"} + } + return forgekey.Ran{Status: 1, Err: "not expected here: " + line} + }, &cr.seen) + return cr +} + +func (cr *candidateRig) test(t *testing.T, args map[string]any) (TestResult, delivery.Step) { + t.Helper() + if _, err := stepTool(t, cr.m, "build_go_test")(args); err != nil { + t.Fatal(err) + } + st := cr.states.waitEnd(t) + var res TestResult + if st.State == delivery.StepPassed { + if err := json.Unmarshal(st.Result, &res); err != nil { + t.Fatalf("result %s: %v", st.Result, err) + } + } + return res, st +} + +func TestTheDigestAndTestToolsAnnounceTheRunner(t *testing.T) { // 1898 K1, ADR 0535 §2 + m := moduleWith(newStates(), "go1.27.1") + for _, s := range m.StepTools() { + if s.Name != "build_go_digest" && s.Name != "build_go_test" { + continue + } + p, _ := s.Input["properties"].(map[string]any)["runner"].(map[string]any) + if p == nil || !slices.Equal(p["enum"].([]string), []string{RunnerInstalled, RunnerWorkspace}) { + t.Errorf("%s announces the runner %v, want installed and workspace", s.Name, p) + } + } +} + +func TestTheUnitLevelUnderTheWorkspaceRunnerRunsTheCandidateShutOff(t *testing.T) { // 1898 K1, K2 (AC4's unit part) + cr := newCandidateRig(t) + res, st := cr.test(t, map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": RunnerWorkspace, "packages": []any{"./..."}}) + if st.State != delivery.StepPassed { + t.Fatalf("the step ended %s: %s", st.State, st.Reason) + } + if res.Tests != 42 || res.Why != "mark" || res.JudgedBy != "candidate "+candidateSHA { + t.Errorf("result = %+v, want the candidate's mark judged by candidate %s", res, candidateSHA) + } + if len(cr.specs) != 1 || cr.specs[0].Step != stepUnit || cr.specs[0].Workspace != "ws1" || !slices.Equal(cr.specs[0].Packages, []string{"./..."}) || cr.specs[0].Bound != UnitBound { + t.Errorf("the candidate was asked %+v, want the unit level of ws1 with its packages and bound", cr.specs) + } + for _, l := range cr.ran { + if strings.HasPrefix(l, "go ") && strings.Contains(l, " test ") { + t.Errorf("the installed program ran %q: the level is the candidate's to run", l) + } + } + // Two commands shut off: the compile, then the candidate's process; each as a run step is (ADR 0448 §2). + if len(cr.seen) != 2 || !strings.Contains(cr.seen[0].command, " build ") || !strings.Contains(cr.seen[0].command, "-C "+filepath.Join(cr.m.Work, "ws1", "tree", candidateModule)) || + !strings.HasPrefix(cr.seen[1].command, filepath.Join(cr.m.Work, "ws1", "candidate", "mesh-build-go")+" level ") { + t.Fatalf("shut off %+v, want the compile and the candidate's process", cr.seen) + } + for _, s := range cr.seen { + checkShutOff(t, cr.m, s, false) + } +} + +func TestTheIntegrationLevelUnderTheWorkspaceRunnerRunsTheCandidate(t *testing.T) { // 1898 K2 + cr := newCandidateRig(t) + res, st := cr.test(t, map[string]any{"workspace": "ws1", "level": LevelIntegration, "runner": RunnerWorkspace}) + if st.State != delivery.StepPassed || res.JudgedBy != "candidate "+candidateSHA { + t.Fatalf("the step ended %s (%s): %+v", st.State, st.Reason, res) + } + if len(cr.specs) != 1 || cr.specs[0].Step != stepIntegration || cr.specs[0].Bound != IntegrationRun || cr.specs[0].Setup != IntegrationSetup { + t.Errorf("the candidate was asked %+v, want the integration level with its bounds", cr.specs) + } +} + +func TestTheDigestUnderTheWorkspaceRunnerIsTheCandidatesAndTheCandidateIsCompiledOnce(t *testing.T) { // 1898 K1, K2 + cr := newCandidateRig(t) + out, err := stepTool(t, cr.m, "build_go_digest")(map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": RunnerWorkspace}) + if err != nil { + t.Fatal(err) + } + d := out.(DigestResult) + if d.Packages["meshbuildgo/cmd/mesh-build-go"] != "mark" || d.JudgedBy != "candidate "+candidateSHA { + t.Errorf("digest = %+v, want the candidate's, judged by candidate %s", d, candidateSHA) + } + if len(cr.specs) != 1 || cr.specs[0].Step != stepDigest || cr.specs[0].Level != LevelUnit { + t.Errorf("the candidate was asked %+v", cr.specs) + } + cr.test(t, map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": RunnerWorkspace}) + if cr.compiled != 1 { + t.Errorf("compiled the candidate %d times for two steps of one workspace, want once", cr.compiled) + } +} + +func TestACandidateThatDoesNotCompileFailsTheStepWithTheCompilersFirstErrorAndRunsNoLevel(t *testing.T) { // 1898/AC5 + cr := newCandidateRig(t) + cr.compile = forgekey.Ran{Status: 1, Stderr: "# meshbuildgo/cmd/mesh-build-go\ncmd/mesh-build-go/digest.go:12:2: undefined: x\ncmd/mesh-build-go/digest.go:13:2: undefined: y\n"} + _, st := cr.test(t, map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": RunnerWorkspace}) + if st.State == delivery.StepPassed || !strings.Contains(st.Reason, "does not compile: cmd/mesh-build-go/digest.go:12:2: undefined: x") || strings.Contains(st.Reason, "undefined: y") { + t.Fatalf("the step ended %s: %q, want it failed naming the first compile error", st.State, st.Reason) + } + if len(cr.specs) != 0 { + t.Errorf("the candidate ran %+v after it failed to compile", cr.specs) + } + for _, l := range cr.ran { + if strings.Contains(l, " test ") { + t.Errorf("a level ran: %q", l) + } + } + if _, err := stepTool(t, cr.m, "build_go_digest")(map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": RunnerWorkspace}); err == nil || + !strings.Contains(err.Error(), "undefined: x") { + t.Errorf("the digest answered %v, want the compile error", err) + } +} + +func TestTheInstalledProgramSaysItJudged(t *testing.T) { // 1898 K1, ADR 0535 §5 + m, states, ws, _ := workspaceFor(t, "slow") + for _, runner := range []any{nil, RunnerInstalled} { + args := map[string]any{"workspace": ws, "level": LevelUnit} + if runner != nil { + args["runner"] = runner + } + res, st := runUnitWith(t, m, states, args) + if st.State != delivery.StepPassed || res.JudgedBy != RunnerInstalled { + t.Errorf("runner %v: the step ended %s, judged by %q, want installed", runner, st.State, res.JudgedBy) + } + } + dm, _ := digestFixture(t) + if d := digests(t, dm, LevelUnit); d.JudgedBy != RunnerInstalled { + t.Errorf("the installed digest is judged by %q, want installed", d.JudgedBy) + } +} + +func TestAnUnknownRunnerIsRefused(t *testing.T) { // 1898 K1 + cr := newCandidateRig(t) + if _, err := cr.m.digest(context.Background(), map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": "elsewhere"}); err == nil { + t.Error("the digest took the runner elsewhere") + } + if _, err := cr.m.test(context.Background(), map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": "elsewhere"}); err == nil { + t.Error("the test step took the runner elsewhere") + } +} + +func TestTheShutOffProcessRunsTheStepItIsAsked(t *testing.T) { // 1898 K2: the candidate's side of the spec + m, _, ws, calls := workspaceFor(t, "slow") + m.Inside = true + a := m.answer(context.Background(), levelSpec{Step: stepUnit, Workspace: ws, Packages: []string{"./..."}, Bound: UnitBound}) + if a.Error != "" || a.Result == nil || a.Result.Level != LevelUnit || a.Result.Tests == 0 { + t.Fatalf("the unit step answered %+v", a) + } + ranTest := false + for _, c := range *calls { + ranTest = ranTest || strings.HasPrefix(c.line, "go -C ") && strings.Contains(c.line, " test -json ") + } + if !ranTest { + t.Errorf("the process ran no go test: %+v", *calls) + } + dm, _ := digestFixture(t) + dm.Inside = true + if a := dm.answer(context.Background(), levelSpec{Step: stepDigest, Level: LevelUnit, Workspace: "ws1"}); a.Error != "" || a.Digest == nil || len(a.Digest.Packages) == 0 { + t.Errorf("the digest step answered %+v", a) + } + if a := m.answer(context.Background(), levelSpec{Step: "deploy", Workspace: ws}); a.Error == "" { + t.Error("the process took a step that is not one") + } +} + +func runUnitWith(t *testing.T, m *Module, states *keptStates, args map[string]any) (TestResult, delivery.Step) { + t.Helper() + if _, err := stepTool(t, m, "build_go_test")(args); err != nil { + t.Fatal(err) + } + st := states.waitEnd(t) + var res TestResult + if st.State == delivery.StepPassed { + if err := json.Unmarshal(st.Result, &res); err != nil { + t.Fatalf("result %s: %v", st.Result, err) + } + } + return res, st +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest.go b/modules/mesh-build-go/cmd/mesh-build-go/digest.go index 55412464..1c92d318 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/digest.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest.go @@ -5,6 +5,7 @@ package main // digest is unchanged is not run again. The tool answers in its call (it is short, ADR 0469 §2). import ( + "bytes" "context" "crypto/sha256" "encoding/hex" @@ -22,7 +23,12 @@ import ( // DigestVersion is the form of the digest's text: changed whenever what the digest hashes changes in kind. // 2: no Go version in the text; the toolchain enters a fingerprint as the agreed version (ADR 0477 §3). 3: the siblings' // commits (the lead's decision L5) and the integration level's tags integration,isolation (the checker's agreement). -const DigestVersion = "3" +// 4: the siblings' commits enter only the digest of a package whose tests reach a file naming MESH_CHECK_BESIDE (ADR 0536 §1). +const DigestVersion = "4" + +// besideVariable is the name a file bears when it reads where the siblings lie; a package whose tests reach such a file +// depends on the siblings' commits. +const besideVariable = "MESH_CHECK_BESIDE" // integrationTags are the build tags of the integration level, its run and its digest alike: integration, and isolation // for the bubblewrap proofs (mesh-sdk's and the build modules'), which a build node can run. @@ -83,6 +89,9 @@ type DigestResult struct { // only when they are the siblings it gave the fetch. Beside []string `json:"beside,omitempty"` WithoutTests []string `json:"without_tests,omitempty"` + // JudgedBy is the program that ran the digest (ADR 0535 §5): "installed", or "candidate " under the + // runner workspace. + JudgedBy string `json:"judged_by,omitempty"` } func (m *Module) digestTool() StepTool { @@ -97,6 +106,7 @@ func (m *Module) digestTool() StepTool { "properties": map[string]any{ "workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"}, "level": map[string]any{"type": "string", "enum": []string{LevelUnit, LevelIntegration}, "description": "the level"}, + "runner": runnerProperty, }, "required": []string{"workspace", "level"}, }, @@ -107,12 +117,49 @@ func (m *Module) digestTool() StepTool { func (m *Module) digest(ctx context.Context, args map[string]any) (any, error) { workspace, _ := args["workspace"].(string) level, _ := args["level"].(string) + runner, err := runnerArg(args["runner"]) + if err != nil { + return nil, err + } + if runner == RunnerWorkspace { + if err := m.workspaceThere(workspace); err != nil { + return nil, err + } + switch level { + case LevelUnit: + case LevelIntegration: + if _, err := m.checkPostgres(ctx); err != nil { + return nil, err // a *buildstep.Refusal, as the installed program's digest says it + } + default: + return nil, fmt.Errorf("the level %q is neither unit nor integration", level) + } + a, judged, err := m.asCandidate(ctx, levelSpec{Step: stepDigest, Level: level, Workspace: workspace}) + if err != nil { + return nil, err + } + if a.Digest == nil { + return nil, fmt.Errorf("the candidate's mesh-build-go answered no digest") + } + a.Digest.JudgedBy = judged + return *a.Digest, nil + } + res, err := m.digestHere(ctx, workspace, level) + if err != nil { + return nil, err + } + res.JudgedBy = RunnerInstalled + return res, nil +} + +// digestHere is the digest step as this program runs it. +func (m *Module) digestHere(ctx context.Context, workspace, level string) (DigestResult, error) { if !workspaceOK.MatchString(workspace) { - return nil, fmt.Errorf("%q is not a workspace id", workspace) + return DigestResult{}, fmt.Errorf("%q is not a workspace id", workspace) } tree := filepath.Join(m.Work, workspace, "tree") if _, err := os.Stat(tree); err != nil { - return nil, fmt.Errorf("no workspace %s: it was never fetched here, or it is older than a day and was removed", workspace) + return DigestResult{}, fmt.Errorf("no workspace %s: it was never fetched here, or it is older than a day and was removed", workspace) } var tags []string header := []string{"digest " + DigestVersion, "level " + level} @@ -120,34 +167,35 @@ func (m *Module) digest(ctx context.Context, args map[string]any) (any, error) { case LevelUnit: case LevelIntegration: if _, err := m.checkPostgres(ctx); err != nil { - return nil, err + return DigestResult{}, err } tags = integrationTags header = append(header, "postgres "+strconv.Itoa(m.Recipe.PostgresMajor), "nats-server "+m.Recipe.NatsServer) default: - return nil, fmt.Errorf("the level %q is neither unit nor integration", level) + return DigestResult{}, fmt.Errorf("the level %q is neither unit nor integration", level) } header = append(header, "tags "+strings.Join(tags, ",")) beside, err := m.besideLines(workspace) if err != nil { - return nil, err + return DigestResult{}, err } + besideHeader := make([]string, 0, len(beside)) for _, b := range beside { - header = append(header, "beside "+b) + besideHeader = append(besideHeader, "beside "+b) } dirs, err := m.goModDirs(ctx, tree) if err != nil { - return nil, err + return DigestResult{}, err } res := DigestResult{Level: level, Packages: map[string]string{}, Beside: beside} for _, d := range dirs { pkgs, err := m.listPackages(ctx, workspace, tree, d, tags) if err != nil { - return nil, err + return DigestResult{}, err } - if err := digestModule(&res, pkgs, tree, header); err != nil { - return nil, err + if err := digestModule(&res, pkgs, tree, header, besideHeader); err != nil { + return DigestResult{}, err } } sort.Strings(res.WithoutTests) @@ -190,7 +238,7 @@ func parsePackages(out string) (map[string]*goPackage, error) { // digestModule adds to res the digest of each root package of one go.mod directory. A root is a package of the main // module that is listed, not only a dependency, and not a variant of another. -func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, header []string) error { +func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, header, beside []string) error { roots := []string{} for path, p := range pkgs { if p.ForTest == "" && !p.DepOnly && !p.Standard && !strings.HasSuffix(path, ".test") && p.Module != nil && p.Module.Main { @@ -208,7 +256,7 @@ func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, he if testMain == nil { return fmt.Errorf("go list gave no test binary for %s", path) } - d, err := packageDigest(root, testMain, pkgs, tree, header) + d, err := packageDigest(root, testMain, pkgs, tree, header, beside) if err != nil { return fmt.Errorf("%s: %w", path, err) } @@ -219,8 +267,9 @@ func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, he // packageDigest is the digest of one package: a hash over a sorted text of the header, the files of every package its // tests reach in the module's own tree (by their content, under their path in the tree), the version and checksum of -// every module outside it, and the package's own test data. A package of the standard library is the Go version's. -func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree string, header []string) (string, error) { +// every module outside it, and the package's own test data. The beside lines (the siblings' commits) join the header +// only when a file the tests reach names MESH_CHECK_BESIDE. A package of the standard library is the Go version's. +func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree string, header, beside []string) (string, error) { reached := map[string]*goPackage{root.ImportPath: root} for _, dep := range testMain.Deps { if p := pkgs[dep]; p != nil { @@ -228,6 +277,7 @@ func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree s } } lines := []string{} + readsBeside := false for _, p := range reached { if p.Standard || p.Module == nil { continue @@ -246,6 +296,11 @@ func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree s return "", err } lines = append(lines, l) + if !readsBeside { + if raw, err := os.ReadFile(filepath.Join(p.Dir, f)); err == nil && bytes.Contains(raw, []byte(besideVariable)) { + readsBeside = true + } + } } } data := filepath.Join(root.Dir, "testdata") @@ -263,6 +318,9 @@ func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree s } } sort.Strings(lines) + if readsBeside { + header = append(append([]string{}, header...), beside...) + } h := sha256.New() for _, l := range append(append([]string{}, header...), lines...) { h.Write([]byte(l + "\n")) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go index ff1e8e6e..5e2d81d0 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go @@ -173,7 +173,7 @@ func TestAModuleOutsideTheTreeCountsByVersionAndChecksum(t *testing.T) { testMain := &goPackage{ImportPath: "x/r.test", Deps: []string{ext.ImportPath}} pkgs[root.ImportPath] = root digest := func() string { - d, err := packageDigest(root, testMain, pkgs, dir, []string{"digest 1"}) + d, err := packageDigest(root, testMain, pkgs, dir, []string{"digest 1"}, nil) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index 435def4d..e5a1b49b 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -6,12 +6,15 @@ package main import ( "context" + "crypto/sha256" + "encoding/hex" "fmt" "os" "path/filepath" "regexp" "sort" "strings" + "sync" "time" "git.novox.be/novox/mesh-sdk/go/buildstep" @@ -114,7 +117,7 @@ func (m *Module) fetchInto(ctx context.Context, req buildstep.FetchRequest, id s return nil, err } remote := m.remote(repository) - if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref, m.FetchOptions...); err != nil { + if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref, m.fetchOptions(ctx, repository, commit, ref)...); err != nil { return nil, err } res := FetchResult{Workspace: id, Commit: commit} @@ -123,6 +126,9 @@ func (m *Module) fetchInto(ctx context.Context, req buildstep.FetchRequest, id s return nil, err } } + if err := m.dissociate(ctx, tree); err != nil { + return nil, err + } if res.Beside, err = m.placeSiblings(ctx, root, siblings); err != nil { return nil, err } @@ -143,11 +149,27 @@ func (m *Module) fetchInto(ctx context.Context, req buildstep.FetchRequest, id s if base != "" && res.Merge == "" { return res, nil // a conflict: the tree is at the trunk, nothing of the head is built or tested } - for _, d := range dirs { + // The directories download side by side, up to the bound (ADR 0536 §2); the first failure in directory order is the + // step's. Resolving a pin places a repository beside the workspace, which two directories may share: one at a time. + var pins sync.Mutex + errs := make([]error, len(dirs)) + slots := make(chan struct{}, m.parallel()) + var wg sync.WaitGroup + for i, d := range dirs { if _, err := os.Stat(filepath.Join(tree, d, "vendor", "modules.txt")); err == nil { continue // a module that vendors resolves nothing } - if err := m.downloadModules(ctx, root, filepath.Join(tree, d), d); err != nil { + wg.Add(1) + slots <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-slots }() + errs[i] = m.downloadModules(ctx, root, filepath.Join(tree, d), d, &pins) + }() + } + wg.Wait() + for _, err := range errs { + if err != nil { return nil, err } } @@ -157,13 +179,15 @@ func (m *Module) fetchInto(ctx context.Context, req buildstep.FetchRequest, id s // downloadModules downloads and verifies the Go modules of the directory dir (d in the tree). A pin of a forge module // that Go cannot fetch is looked for among the forge's pull request heads and handed to Go by a modfile (pins.go); a // failure that names no such pin is the first failure. -func (m *Module) downloadModules(ctx context.Context, root, dir, d string) error { +func (m *Module) downloadModules(ctx context.Context, root, dir, d string, pins *sync.Mutex) error { for { failure := m.downloadAndVerify(ctx, dir, d) if failure == nil { return nil } + pins.Lock() progress, err := m.resolvePins(ctx, root, dir, failure.Error()) + pins.Unlock() if err != nil { return err } @@ -173,8 +197,41 @@ func (m *Module) downloadModules(ctx context.Context, root, dir, d string) error } } +// verifyLocks hold one mutex per digest of a go.mod and go.sum, so two directories with the same pair verify once. +var verifyLocks sync.Map + +// verifiedMark is the file naming a pass of go mod verify for a go.mod and go.sum pair, "" when a pass is not kept: +// the module cache is only read by steps that are shut off with it read-only (ADR 0448 §3), so the pass holds only then, +// and a directory with a pin modfile verifies another file. +func (m *Module) verifiedMark(dir string) string { + if m.Cache == "" || m.Bubblewrap == nil { + return "" + } + if _, ok := m.pinModfile(dir); ok { + return "" + } + h := sha256.New() + for _, name := range []string{"go.mod", "go.sum"} { + raw, _ := os.ReadFile(filepath.Join(dir, name)) + fmt.Fprintf(h, "%s %d\n", name, len(raw)) + h.Write(raw) + } + return filepath.Join(m.Cache, "verified", hex.EncodeToString(h.Sum(nil))) +} + func (m *Module) downloadAndVerify(ctx context.Context, dir, d string) error { for _, verb := range []string{"download", "verify"} { + mark := "" + if verb == "verify" { + if mark = m.verifiedMark(dir); mark != "" { + mu, _ := verifyLocks.LoadOrStore(mark, &sync.Mutex{}) + mu.(*sync.Mutex).Lock() + defer mu.(*sync.Mutex).Unlock() + if _, err := os.Stat(mark); err == nil { + continue // this go.mod and go.sum passed before + } + } + } cmd, env := []string{"-C", dir, "mod", verb}, m.goEnvFor(dir) if modfile, ok := m.pinModfile(dir); ok && verb == "verify" { // go mod verify takes no -modfile: it is run on the modfile itself, which names the placed repositories. @@ -184,6 +241,12 @@ func (m *Module) downloadAndVerify(ctx context.Context, dir, d string) error { if r.Status != 0 || r.Err != "" { return fmt.Errorf("go mod %s in %s: %s", verb, d, causeLine(r.Err+"\n"+r.Stdout+"\n"+r.Stderr)) } + if mark != "" { + // Said after the pass; a mark that cannot be written only means the next fetch verifies again. + if os.MkdirAll(filepath.Dir(mark), 0o700) == nil { + os.WriteFile(mark, nil, 0o600) + } + } } return nil } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go index d9d092c5..04874659 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go @@ -30,6 +30,7 @@ type fakeStep struct { gomods string gate chan struct{} // when set, the first git command waits for it goRule func(line string) forgekey.Ran + files map[string]string // files (relative to the tree) written at checkout: go.mod and go.sum of the fixture vendors []string // directories (relative to the tree) that get a vendor/modules.txt at checkout rule func(line string) (forgekey.Ran, bool) // answers a command before the defaults do } @@ -55,6 +56,10 @@ func (f *fakeStep) run(ctx context.Context, env []string, name string, args ...s case name == "git" && strings.Contains(line, " checkout "): for i, a := range args { if a == "-C" { + for name, text := range f.files { + os.MkdirAll(filepath.Dir(filepath.Join(args[i+1], name)), 0o755) + os.WriteFile(filepath.Join(args[i+1], name), []byte(text), 0o644) + } for _, v := range f.vendors { os.MkdirAll(filepath.Join(args[i+1], v, "vendor"), 0o755) os.WriteFile(filepath.Join(args[i+1], v, "vendor", "modules.txt"), nil, 0o644) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/integration.go b/modules/mesh-build-go/cmd/mesh-build-go/integration.go index 03de39ab..231714a0 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/integration.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/integration.go @@ -75,8 +75,16 @@ func (m *Module) integrationHere(ctx context.Context, workspace, tree string, pa env := append([]string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats}, m.besideEnv(workspace)...) // Compiling counts as setup: the tests are built once, here, with the addresses a TestMain may read, and the run // below finds them built. Whether they compile is for the run to say; its failure is not read here. + // Only the directories the selected packages lie in compile (ADR 0536 §3); no selection is every directory. + byDir, err := packagesByDir(tree, dirs, packages) + if err != nil { + return nil, err + } for _, d := range dirs { - m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), env...), "go", "-C", filepath.Join(tree, d), "test", "-tags", strings.Join(integrationTags, ","), "-run", "^$", "-count=1", "./...") + if len(byDir[d]) == 0 { + continue + } + m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), env...), "go", append([]string{"-C", filepath.Join(tree, d), "test", "-tags", strings.Join(integrationTags, ","), "-run", "^$", "-count=1"}, byDir[d]...)...) } setup := m.clock().Sub(began) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go b/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go index 9d2aafa1..fafb7bda 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go @@ -31,6 +31,10 @@ type integFake struct { busDown bool runDirs []string // run directories seen when the store was started, to be gone at the end panicOn string // a phase that panics + // gomods is what git ls-files answers ("go.mod\n" when empty); onTest is called, unlocked, as each go test -json + // command runs, with its line. + gomods string + onTest func(line string) } func newIntegFake(t *testing.T, captured string) *integFake { @@ -63,10 +67,16 @@ func (f *integFake) run(ctx context.Context, env []string, name string, args ... case name == "postgres": return forgekey.Ran{Stdout: f.postgres + "\n"} case name == "git" && len(args) > 2 && args[2] == "ls-files": + if f.gomods != "" { + return forgekey.Ran{Stdout: f.gomods} + } return forgekey.Ran{Stdout: "go.mod\n"} case name == "go" && strings.Contains(line, "-run ^$"): f.tick("compile") case name == "go": + if f.onTest != nil { + f.onTest(line) + } f.tick("test") status := 0 if strings.Contains(f.captured, `"Action":"fail"`) { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/level.go b/modules/mesh-build-go/cmd/mesh-build-go/level.go index 520962dc..6ff18aa6 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/level.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/level.go @@ -18,11 +18,22 @@ import ( "git.novox.be/novox/mesh-sdk/go/buildstep" ) -// levelArg is the first argument that makes this binary an integration level's process. +// levelArg is the first argument that makes this binary a step's shut-off process: the integration level's, and, when +// the merge changes this module, the candidate's program running a digest or a level (ADR 0535 §1, candidate.go). const levelArg = "level" -// levelSpec is what the shut-off process is asked: the workspace, the packages and the -run pattern, the level's bounds. +// The steps a shut-off process runs (levelSpec.Step); empty is the integration level, as a spec said before the runner. +const ( + stepDigest = "digest" + stepUnit = LevelUnit + stepIntegration = LevelIntegration +) + +// levelSpec is what the shut-off process is asked: the step, the workspace, the packages and the -run pattern, the +// level's bounds; for a digest, its level. type levelSpec struct { + Step string `json:"step,omitempty"` + Level string `json:"level,omitempty"` Work string `json:"work"` Cache string `json:"cache"` Workspace string `json:"workspace"` @@ -32,10 +43,11 @@ type levelSpec struct { Bound time.Duration `json:"bound"` } -// levelAnswer is the one line the process prints: the level's result, or why it could not run. +// levelAnswer is the one line the process prints: the level's result or the digest, or why it could not run. type levelAnswer struct { - Result *TestResult `json:"result,omitempty"` - Error string `json:"error,omitempty"` + Result *TestResult `json:"result,omitempty"` + Digest *DigestResult `json:"digest,omitempty"` + Error string `json:"error,omitempty"` } // executable is this program's path, bound read-only into the step: Executable when a test set it. @@ -50,35 +62,63 @@ func (m *Module) executable() (string, error) { return filepath.EvalSymlinks(exe) } -// integrationShutOff runs the integration level in one shut-off process and reads its answer. +// integrationShutOff runs the integration level in one shut-off process of this program and reads its answer. func (m *Module) integrationShutOff(ctx context.Context, workspace string, packages []string, pattern string) (*TestResult, error) { exe, err := m.executable() if err != nil { return nil, fmt.Errorf("the integration level's program: %w", err) } b := m.integrationBounds() - spec, err := json.Marshal(levelSpec{Work: m.Work, Cache: m.Cache, Workspace: workspace, Packages: packages, Run: pattern, Setup: b.Setup, Bound: b.Run}) + a, err := m.shutOff(ctx, exe, levelSpec{Step: stepIntegration, Workspace: workspace, Packages: packages, Run: pattern, Setup: b.Setup, Bound: b.Run}) if err != nil { return nil, err } + if a.Result == nil { + return nil, fmt.Errorf("the integration level's shut-off process answered no result") + } + return a.Result, nil +} + +// shutOff runs the program exe as one shut-off process of the workspace (`exe level `) and reads its answer: the +// workspace read-write, the program alone read-only, the shared caches beneath a throwaway layer, the Node build +// module's toolchains read-only for the integration level. +func (m *Module) shutOff(ctx context.Context, exe string, spec levelSpec) (levelAnswer, error) { + spec.Work, spec.Cache = m.Work, m.Cache + raw, err := json.Marshal(spec) + if err != nil { + return levelAnswer{}, err + } for _, c := range m.caches() { if err := os.MkdirAll(c, 0o700); err != nil { - return nil, fmt.Errorf("the shared cache %s: %w", c, err) + return levelAnswer{}, fmt.Errorf("the shared cache %s: %w", c, err) } } - iso := m.isolation(workspace, false) + iso := m.isolation(spec.Workspace, false) iso.ReadOnly = append(iso.ReadOnly, exe) // the program alone, not the directory it lies in - iso.ReadOnly = append(iso.ReadOnly, m.nodeToolchains()...) - r := buildstep.Isolated(m.Run, iso)(ctx, nil, exe, levelArg, string(spec)) + if spec.Step == stepIntegration || spec.Step == "" { + iso.ReadOnly = append(iso.ReadOnly, m.nodeToolchains()...) + } + r := buildstep.Isolated(m.Run, iso)(ctx, nil, exe, levelArg, string(raw)) var answer levelAnswer line := lastLine(r.Stdout) - if err := json.Unmarshal([]byte(line), &answer); err != nil || (answer.Result == nil && answer.Error == "") { - return nil, fmt.Errorf("the integration level's shut-off process answered nothing it says (status %d %s): %s", r.Status, r.Err, firstLine(r.Stderr+"\n"+r.Stdout)) + if err := json.Unmarshal([]byte(line), &answer); err != nil || (answer.Result == nil && answer.Digest == nil && answer.Error == "") { + return levelAnswer{}, fmt.Errorf("the %s step's shut-off process answered nothing it says (status %d %s): %s", stepName(spec.Step), r.Status, r.Err, firstLine(r.Stderr+"\n"+r.Stdout)) } if answer.Error != "" { - return nil, fmt.Errorf("%s", answer.Error) + return levelAnswer{}, fmt.Errorf("%s", answer.Error) } - return answer.Result, nil + return answer, nil +} + +// stepName names a spec's step in words. +func stepName(step string) string { + switch step { + case stepDigest: + return "digest" + case stepUnit: + return "unit level's" + } + return "integration level's" } // NodeToolchains is the Node build module's toolchains directory on this node: its cache directory (module.json @@ -124,18 +164,40 @@ func runLevelProcess(args []string) int { if !workspaceOK.MatchString(spec.Workspace) { return say(levelAnswer{Error: fmt.Sprintf("%q is not a workspace id", spec.Workspace)}) } - m := &Module{Inside: true, Recipe: Current(), Work: spec.Work, Cache: spec.Cache, Home: os.Getenv("HOME"), + m := &Module{Inside: true, Recipe: Current(), Work: spec.Work, Cache: spec.Cache, Home: os.Getenv("HOME"), UnitBound: spec.Bound, IntegrationBounds: Bounds{Setup: spec.Setup, Run: spec.Bound}, Say: func(string, ...any) {}} m.Run = StepRunner(m.Home) - ctx := context.Background() - major, err := m.checkPostgres(ctx) - if err != nil { - return say(levelAnswer{Error: err.Error()}) + return say(m.answer(context.Background(), spec)) +} + +// answer runs the spec's step in this process, already shut off: a digest of a level, the unit level or the integration +// level of the workspace. +func (m *Module) answer(ctx context.Context, spec levelSpec) levelAnswer { + tree := filepath.Join(m.Work, spec.Workspace, "tree") + switch spec.Step { + case stepDigest: + d, err := m.digestHere(ctx, spec.Workspace, spec.Level) + if err != nil { + return levelAnswer{Error: err.Error()} + } + return levelAnswer{Digest: &d} + case stepUnit: + res, err := m.runLevel(ctx, tree, levelOpts{workspace: spec.Workspace, level: LevelUnit, packages: spec.Packages, run: spec.Run, + bounds: Bounds{Run: m.unitBound()}, env: m.besideEnv(spec.Workspace)}) + if err != nil { + return levelAnswer{Error: err.Error()} + } + return levelAnswer{Result: res} + case stepIntegration, "": + major, err := m.checkPostgres(ctx) + if err != nil { + return levelAnswer{Error: err.Error()} + } + res, err := m.integrationHere(ctx, spec.Workspace, tree, spec.Packages, spec.Run, major) + if err != nil { + return levelAnswer{Error: err.Error()} + } + return levelAnswer{Result: res} } - tree := filepath.Join(spec.Work, spec.Workspace, "tree") - res, err := m.integrationHere(ctx, spec.Workspace, tree, spec.Packages, spec.Run, major) - if err != nil { - return say(levelAnswer{Error: err.Error()}) - } - return say(levelAnswer{Result: res}) + return levelAnswer{Error: fmt.Sprintf("the step %q is not digest, unit or integration", spec.Step)} } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/levels.go b/modules/mesh-build-go/cmd/mesh-build-go/levels.go index 9631252a..455ca95f 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/levels.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/levels.go @@ -83,6 +83,9 @@ type TestResult struct { OverWhich string `json:"over_which,omitempty"` Store string `json:"store,omitempty"` Bus string `json:"bus,omitempty"` + // JudgedBy is the program that ran the level (ADR 0535 §5): "installed", or "candidate " under the + // runner workspace. + JudgedBy string `json:"judged_by,omitempty"` } // event is one line of `go test -json` (test2json, and the build events go test adds). diff --git a/modules/mesh-build-go/cmd/mesh-build-go/mirror.go b/modules/mesh-build-go/cmd/mesh-build-go/mirror.go new file mode 100644 index 00000000..0ad17fda --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/mirror.go @@ -0,0 +1,73 @@ +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + "syscall" + + "git.novox.be/novox/mesh-sdk/go/buildstep" +) + +// mirrorLocks hold one mutex per mirror of this process; the lock file beside the mirror holds off another process. +var mirrorLocks sync.Map + +// mirrorDir is where the bare mirror of a repository lies: one per repository in the module's cache (ADR 0536 §2). +func (m *Module) mirrorDir(repository string) string { + return filepath.Join(m.Cache, "mirror", filepath.FromSlash(repository)+".git") +} + +// fetchOptions are the options a fetch of repository runs with: the module's own, and the mirror of the repository +// brought to the commit, so git transfers only the objects the mirror lacks. A mirror that cannot be brought to the +// commit is said and left out: the fetch is then the full one, as it was. +func (m *Module) fetchOptions(ctx context.Context, repository, commit, ref string) []buildstep.Option { + opts := m.FetchOptions + if m.Cache == "" || !repositoryOK.MatchString(repository) { + return opts + } + mirror := m.mirrorDir(repository) + if err := m.updateMirror(ctx, mirror, m.remote(repository), commit, ref); err != nil { + if m.Say != nil { + m.Say("the mirror of %s is not used: %v", repository, err) + } + return opts + } + return append(append([]buildstep.Option{}, opts...), buildstep.WithMirror(mirror)) +} + +// updateMirror brings the mirror to the commit under the mirror's lock: a mutex for the checks of this process, a lock +// file for another process of the node. +func (m *Module) updateMirror(ctx context.Context, mirror, remote, commit, ref string) error { + mu, _ := mirrorLocks.LoadOrStore(mirror, &sync.Mutex{}) + mu.(*sync.Mutex).Lock() + defer mu.(*sync.Mutex).Unlock() + if err := os.MkdirAll(mirror, 0o700); err != nil { // git init --bare takes an empty directory + return err + } + lock, err := os.OpenFile(mirror+".lock", os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return err + } + defer lock.Close() + if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil { + return fmt.Errorf("lock %s: %w", mirror, err) + } + defer syscall.Flock(int(lock.Fd()), syscall.LOCK_UN) + return buildstep.UpdateMirror(ctx, m.Run, mirror, remote, commit, ref, m.FetchOptions...) +} + +// dissociate makes the repository in dir self-contained again, as git clone --dissociate does: the objects it borrowed +// from the mirror are packed into it and its alternates file goes, so a step shut off with the workspace alone (the +// mirror is not bound into it) finds every object. The network saving stays: the borrowing was for the transfer. +func (m *Module) dissociate(ctx context.Context, dir string) error { + alternates := filepath.Join(dir, ".git", "objects", "info", "alternates") + if _, err := os.Stat(alternates); err != nil { + return nil // no mirror was used + } + if r := m.Run(ctx, []string{"GIT_TERMINAL_PROMPT=0"}, "git", "-C", dir, "repack", "-a", "-d", "-q"); r.Status != 0 || r.Err != "" { + return fmt.Errorf("git repack in %s: %s", dir, firstLine(r.Err+" "+r.Stderr)) + } + return os.Remove(alternates) +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/mirror_test.go b/modules/mesh-build-go/cmd/mesh-build-go/mirror_test.go new file mode 100644 index 00000000..6721155e --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/mirror_test.go @@ -0,0 +1,190 @@ +package main + +import ( + "encoding/json" + "git.novox.be/novox/mesh-sdk/go/buildstep" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +func fetchOnce(t *testing.T, m *Module, states *keptStates, args map[string]any) FetchResult { + t.Helper() + if _, err := fetchTool(t, m)(args); err != nil { + t.Fatal(err) + } + st := states.waitEnd(t) + if st.State != delivery.StepPassed { + t.Fatalf("the step ended %s: %s", st.State, st.Reason) + } + var res FetchResult + if err := json.Unmarshal(st.Result, &res); err != nil { + t.Fatal(err) + } + return res +} + +// Issue 890/AC4 (ADR 0536 §2): two fetches of one repository on one module bring the same mirror to their commit, and +// each workspace's alternates file names the mirror's objects. +func TestTwoFetchesOfARepositoryUseOneMirrorAndReferenceIt(t *testing.T) { + f := &fakeStep{gomods: "go.mod\n"} + m, states := fetchModule(t, f) + mirror := filepath.Join(m.Cache, "mirror", "novox", "mesh-sdk.git") + var workspaces []string + for i := 0; i < 2; i++ { + workspaces = append(workspaces, fetchOnce(t, m, states, fetchArgs).Workspace) + } + updates := 0 + for _, l := range f.lines() { + if strings.Contains(l, "-C "+mirror+" fetch --no-tags origin "+commitSHA) { + updates++ + } + if strings.Contains(l, "mirror") && !strings.Contains(l, mirror) { + t.Errorf("a command names another mirror: %s", l) + } + } + if updates != 2 { + t.Errorf("the mirror %s was updated %d times, want 2:\n%s", mirror, updates, strings.Join(f.lines(), "\n")) + } + // The fake git writes nothing, so the alternates file the SDK wrote stands until the fetch dissociates: it is gone + // at the end and git repack -a -d ran in each tree. + for _, ws := range workspaces { + tree := filepath.Join(m.Work, ws, "tree") + if _, err := os.Stat(filepath.Join(tree, ".git", "objects", "info", "alternates")); err == nil { + t.Errorf("the workspace %s still borrows from the mirror", ws) + } + if !strings.Contains(strings.Join(f.lines(), "\n"), "-C "+tree+" repack -a -d") { + t.Errorf("git repack -a -d did not run in %s", tree) + } + } +} + +// A sibling is placed through its own repository's mirror. +func TestASiblingIsPlacedThroughItsOwnMirror(t *testing.T) { + f := &fakeStep{gomods: "go.mod\n"} + m, states := fetchModule(t, f) + args := map[string]any{"repository": "novox/mesh-controller", "commit": commitSHA, "beside": []any{ + map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA}}} + ws := fetchOnce(t, m, states, args).Workspace + for _, repo := range []string{"mesh-controller", "mesh-sdk"} { + mirror := filepath.Join(m.Cache, "mirror", "novox", repo+".git") + if !strings.Contains(strings.Join(f.lines(), "\n"), "-C "+mirror+" fetch --no-tags origin "+commitSHA) { + t.Errorf("the mirror of %s was not updated", repo) + } + } + if _, err := os.Stat(filepath.Join(m.Work, ws, "beside", "mesh-sdk", ".git", "objects", "info", "alternates")); err == nil { + t.Errorf("the sibling still borrows from the mirror") + } +} + +// Two checks at once on one node update one mirror one after the other. +func TestTwoUpdatesOfOneMirrorNeverRunAtOnce(t *testing.T) { + var mu sync.Mutex + inside, most := 0, 0 + f := &fakeStep{gomods: "go.mod\n"} + f.rule = func(line string) (r runAnswer, ok bool) { + if !strings.Contains(line, "fetch --no-tags origin") || strings.Contains(line, "--depth") { + return r, false + } + mu.Lock() + inside++ + if inside > most { + most = inside + } + mu.Unlock() + sleepABit() + mu.Lock() + inside-- + mu.Unlock() + return r, true + } + m, states := fetchModule(t, f) + var wg sync.WaitGroup + for i := 0; i < 2; i++ { + wg.Add(1) + go func() { + defer wg.Done() + fetchTool(t, m)(fetchArgs) + }() + } + wg.Wait() + states.waitEnd(t) + states.waitEnd(t) + if most != 1 { + t.Errorf("%d updates of the mirror ran at once", most) + } +} + +// A mirror that cannot be updated does not fail the fetch: the fetch goes on without it. +func TestAMirrorThatCannotBeUpdatedLetsTheFetchGoOnWithoutIt(t *testing.T) { + f := &fakeStep{gomods: "go.mod\n"} + f.rule = func(line string) (r runAnswer, ok bool) { + if strings.Contains(line, "mirror") && strings.Contains(line, " fetch ") { + return runAnswer{Status: 1, Stderr: "fatal: no space left\n"}, true + } + return r, false + } + m, states := fetchModule(t, f) + ws := fetchOnce(t, m, states, fetchArgs).Workspace + if _, err := os.Stat(filepath.Join(m.Work, ws, "tree", ".git", "objects", "info", "alternates")); err == nil { + t.Errorf("the workspace borrows from a mirror that is not there") + } +} + +func sleepABit() { time.Sleep(30 * time.Millisecond) } + +// The workspace is self-contained after the fetch: no alternates file, and git finds the commit with the mirror gone, +// as a step shut off with the workspace alone needs. Real git, a local repository as the forge. +func TestTheWorkspaceIsSelfContainedAfterAFetchThroughTheMirror(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("no git") + } + forge := t.TempDir() + src := filepath.Join(forge, "novox", "mesh-sdk.git") + if err := os.MkdirAll(src, 0o755); err != nil { + t.Fatal(err) + } + git := func(dir string, args ...string) string { + cmd := exec.Command("git", append([]string{"-C", dir, "-c", "user.name=t", "-c", "user.email=t@example.org", "-c", "commit.gpgsign=false"}, args...)...) + cmd.Env = append(os.Environ(), "GIT_CONFIG_GLOBAL=/dev/null") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v %s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + git(src, "init", "-q") + if err := os.WriteFile(filepath.Join(src, "f.txt"), []byte("one\n"), 0o644); err != nil { + t.Fatal(err) + } + git(src, "add", ".") + git(src, "commit", "-q", "-m", "one") + commit := git(src, "rev-parse", "HEAD") + + states := newStates() + m := moduleWith(states, "go1.27.1") + m.Home, m.Work, m.Cache = t.TempDir(), t.TempDir(), t.TempDir() + m.Run = StepRunner(m.Home) + m.RemoteBase = "file://" + forge + "/" + m.FetchOptions = []buildstep.Option{buildstep.AllowFileRemote()} + ws := fetchOnce(t, m, states, map[string]any{"repository": "novox/mesh-sdk", "commit": commit}).Workspace + tree := filepath.Join(m.Work, ws, "tree") + if _, err := os.Stat(filepath.Join(tree, ".git", "objects", "info", "alternates")); err == nil { + t.Errorf("the workspace still has an alternates file") + } + if err := os.RemoveAll(filepath.Join(m.Cache, "mirror")); err != nil { + t.Fatal(err) + } + if out, err := exec.Command("git", "-C", tree, "cat-file", "-e", commit+"^{commit}").CombinedOutput(); err != nil { + t.Errorf("git does not find the commit with the mirror gone: %v %s", err, out) + } + if out, err := exec.Command("git", "-C", tree, "fsck", "--connectivity-only").CombinedOutput(); err != nil { + t.Errorf("the workspace is not whole: %v %s", err, out) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module.go b/modules/mesh-build-go/cmd/mesh-build-go/module.go index fffb3e1f..0f306017 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/module.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/module.go @@ -54,6 +54,9 @@ type Module struct { // Bubblewrap reads the installed bubblewrap's version (ProbeBubblewrap): a step is refused, and the module is // unhealthy, while it is missing or older than BubblewrapLeast. Nil probes nothing. Bubblewrap func(ctx context.Context) (string, error) + // Parallel is how many go.mod directories a level runs at once; zero is half the processors, and at least one + // (ADR 0536 §3: a node runs two checks at once). + Parallel int // Inside is the integration level's own process, started shut off in bubblewrap by a test step (level.go): its // commands run as they are, never shut off a second time. Inside bool diff --git a/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go b/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go index bbdc0ff7..a29e4fac 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "strings" + "sync" "testing" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -28,6 +29,7 @@ type pinHarness struct { pulls string // what git ls-remote answers onTrunk bool // the pin is fetchable as it is: go mod download never says unknown revision cmds []cmdRecord + mu sync.Mutex } func newPinHarness(t *testing.T, gomod string, vendor bool, pulls string) (*pinHarness, *keptStates) { @@ -36,7 +38,9 @@ func newPinHarness(t *testing.T, gomod string, vendor bool, pulls string) (*pinH h := &pinHarness{t: t, m: m, f: f, pulls: pulls} m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { line := name + " " + strings.Join(args, " ") + h.mu.Lock() h.cmds = append(h.cmds, cmdRecord{env: env, line: line}) + h.mu.Unlock() switch { case name == "git" && strings.Contains(line, " checkout ") && !strings.Contains(line, "pinned"): for i, a := range args { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/preview.go b/modules/mesh-build-go/cmd/mesh-build-go/preview.go index 87ae1066..067dcfff 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/preview.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/preview.go @@ -96,7 +96,10 @@ func (m *Module) placeSiblings(ctx context.Context, root string, siblings []sibl if err := os.MkdirAll(dir, 0o700); err != nil { return nil, err } - if err := buildstep.FetchCommit(ctx, m.Run, dir, m.remote(s.repository), s.commit, s.ref, m.FetchOptions...); err != nil { + if err := buildstep.FetchCommit(ctx, m.Run, dir, m.remote(s.repository), s.commit, s.ref, m.fetchOptions(ctx, s.repository, s.commit, s.ref)...); err != nil { + return nil, fmt.Errorf("beside %s: %w", s.repository, err) + } + if err := m.dissociate(ctx, dir); err != nil { return nil, fmt.Errorf("beside %s: %w", s.repository, err) } if link := firstSymlink(dir); link != "" { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/selected_dirs_test.go b/modules/mesh-build-go/cmd/mesh-build-go/selected_dirs_test.go new file mode 100644 index 00000000..fa86bb54 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/selected_dirs_test.go @@ -0,0 +1,172 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// threeModules gives the workspace three go.mod directories, a, b and c, whose modules are fixture/a, fixture/b and +// fixture/c. +func threeModules(t *testing.T, f *integFake) (*Module, *keptStates) { + t.Helper() + m, states := integModule(t, f) + f.gomods = "a/go.mod\nb/go.mod\nc/go.mod\n" + for _, d := range []string{"a", "b", "c"} { + dir := filepath.Join(m.Work, "ws1", "tree", d) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module fixture/"+d+"\n\ngo 1.25.0\n"), 0o644); err != nil { + t.Fatal(err) + } + } + return m, states +} + +func runIntegrationOf(t *testing.T, m *Module, states *keptStates, packages ...any) (TestResult, delivery.Step) { + t.Helper() + for _, tool := range m.Tools(m.StepTools()) { + if tool.Name != "build_go_test" { + continue + } + args := map[string]any{"workspace": "ws1", "level": "integration"} + if len(packages) > 0 { + args["packages"] = packages + } + if _, err := tool.Run(args); err != nil { + t.Fatal(err) + } + } + st := states.waitEnd(t) + var res TestResult + if st.State == delivery.StepPassed { + if err := json.Unmarshal(st.Result, &res); err != nil { + t.Fatal(err) + } + } + return res, st +} + +// Issue 890/AC1 (ADR 0536 §3): with packages selected in one of three go.mod directories, the integration level +// compiles and runs go test in that directory only. +func TestOnlyTheSelectedDirectoryCompilesAndRuns(t *testing.T) { + f := newIntegFake(t, "slow") + m, states := threeModules(t, f) + if _, st := runIntegrationOf(t, m, states, "fixture/b/x"); st.State != delivery.StepPassed { + t.Fatalf("the step ended %s: %s", st.State, st.Reason) + } + tree := filepath.Join(m.Work, "ws1", "tree") + compiled, ran := 0, 0 + for _, l := range strings.Split(f.lines(), "\n") { + if !strings.HasPrefix(l, "go ") || !strings.Contains(l, " test ") { + continue + } + if !strings.HasPrefix(l, "go -C "+filepath.Join(tree, "b")+" test ") { + t.Errorf("go test ran outside the selected directory: %s", l) + } + if strings.Contains(l, "-run ^$") { + compiled++ + } else { + ran++ + } + } + if compiled != 1 || ran != 1 { + t.Errorf("compiled %d times and ran %d times, want once each:\n%s", compiled, ran, f.lines()) + } +} + +// Issue 890/AC1: no selection keeps every directory, compiled and run. +func TestWithoutASelectionEveryDirectoryCompilesAndRuns(t *testing.T) { + f := newIntegFake(t, "slow") + m, states := threeModules(t, f) + runIntegrationOf(t, m, states) + compiled, ran := 0, 0 + for _, l := range strings.Split(f.lines(), "\n") { + if strings.HasPrefix(l, "go ") && strings.Contains(l, " test ") { + if strings.Contains(l, "-run ^$") { + compiled++ + } else { + ran++ + } + } + } + if compiled != 3 || ran != 3 { + t.Errorf("compiled %d and ran %d, want 3 and 3:\n%s", compiled, ran, f.lines()) + } +} + +// Issue 890/AC2: with two directories selected and a runner that blocks, both go test commands run at once; the +// result lists the packages in directory order. +func TestSelectedDirectoriesRunAtOnceAndAnswerInDirectoryOrder(t *testing.T) { + f := newIntegFake(t, "slow") + var mu sync.Mutex + running, most := 0, 0 + release := make(chan struct{}) + f.onTest = func(line string) { + mu.Lock() + running++ + if running > most { + most = running + } + if running == 2 { + close(release) + } + mu.Unlock() + select { + case <-release: + case <-time.After(10 * time.Second): + } + mu.Lock() + running-- + mu.Unlock() + } + m, states := threeModules(t, f) + m.Parallel = 3 + res, st := runIntegrationOf(t, m, states, "fixture/c/x", "fixture/a/x") + if st.State != delivery.StepPassed { + t.Fatalf("the step ended %s: %s", st.State, st.Reason) + } + if most != 2 { + t.Errorf("at most %d go test commands ran at once, want 2", most) + } + dirs := []string{} + for _, p := range res.Packages { + dirs = append(dirs, p.Dir) + } + if got := fmt.Sprint(dirs); !strings.HasPrefix(got, "[a ") || !strings.HasSuffix(got, " c]") { + t.Errorf("the packages lie in directories %s, want a's before c's", got) + } +} + +// The bound holds: with one directory at a time, no two go test commands run at once. +func TestTheDirectoriesAtOnceNeverPassTheBound(t *testing.T) { + f := newIntegFake(t, "slow") + var mu sync.Mutex + running, most := 0, 0 + f.onTest = func(string) { + mu.Lock() + running++ + if running > most { + most = running + } + mu.Unlock() + time.Sleep(20 * time.Millisecond) + mu.Lock() + running-- + mu.Unlock() + } + m, states := threeModules(t, f) + m.Parallel = 1 + runIntegrationOf(t, m, states) + if most != 1 { + t.Errorf("%d go test commands ran at once with a bound of 1", most) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test.go b/modules/mesh-build-go/cmd/mesh-build-go/test.go index d129c277..c2b8167b 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test.go @@ -10,8 +10,12 @@ import ( "os" "path/filepath" "regexp" + "runtime" "strings" + "sync" "time" + + "git.novox.be/novox/mesh-sdk/go/forgekey" ) // testTimeout is go test's own timeout for a level: a bound on a test that never ends, far above any level's bound. @@ -33,7 +37,8 @@ func (m *Module) testTool() StepTool { "level": map[string]any{"type": "string", "enum": []string{LevelUnit, LevelIntegration}, "description": "the level"}, "packages": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "description": "package patterns of each module, ./... when none"}, - "run": map[string]any{"type": "string", "description": "optional go test -run pattern: only the tests it matches run (a try's tests)"}, + "run": map[string]any{"type": "string", "description": "optional go test -run pattern: only the tests it matches run (a try's tests)"}, + "runner": runnerProperty, }, "required": []string{"workspace", "level"}, }, @@ -68,13 +73,48 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) { if _, err := os.Stat(tree); err != nil { return nil, fmt.Errorf("no workspace %s: it was never fetched here, or it is older than a day and was removed", workspace) } - switch level { - case LevelUnit: - return m.runLevel(ctx, tree, levelOpts{workspace: workspace, level: LevelUnit, packages: packages, run: pattern, bounds: Bounds{Run: m.unitBound()}, env: m.besideEnv(workspace)}) - case LevelIntegration: - return m.integration(ctx, workspace, tree, packages, pattern) + runner, err := runnerArg(args["runner"]) + if err != nil { + return nil, err } - return nil, fmt.Errorf("the level %q is neither unit nor integration", level) + var res *TestResult + switch { + case runner == RunnerWorkspace && (level == LevelUnit || level == LevelIntegration): + return m.testAsCandidate(ctx, workspace, level, packages, pattern) + case level == LevelUnit: + res, err = m.runLevel(ctx, tree, levelOpts{workspace: workspace, level: LevelUnit, packages: packages, run: pattern, bounds: Bounds{Run: m.unitBound()}, env: m.besideEnv(workspace)}) + case level == LevelIntegration: + res, err = m.integration(ctx, workspace, tree, packages, pattern) + default: + return nil, fmt.Errorf("the level %q is neither unit nor integration", level) + } + if err != nil { + return nil, err + } + res.JudgedBy = RunnerInstalled + return res, nil +} + +// testAsCandidate runs a level as the candidate's program (ADR 0535 §1): the integration level is refused first as the +// installed program would refuse it (no postgres), then the whole level runs in the candidate's one shut-off process. +func (m *Module) testAsCandidate(ctx context.Context, workspace, level string, packages []string, pattern string) (*TestResult, error) { + spec := levelSpec{Step: stepUnit, Workspace: workspace, Packages: packages, Run: pattern, Bound: m.unitBound()} + if level == LevelIntegration { + if _, err := m.checkPostgres(ctx); err != nil { + return nil, err // a *buildstep.Refusal: this level only + } + b := m.integrationBounds() + spec = levelSpec{Step: stepIntegration, Workspace: workspace, Packages: packages, Run: pattern, Setup: b.Setup, Bound: b.Run} + } + a, judged, err := m.asCandidate(ctx, spec) + if err != nil { + return nil, err + } + if a.Result == nil { + return nil, fmt.Errorf("the candidate's mesh-build-go answered no result of the %s level", level) + } + a.Result.JudgedBy = judged + return a.Result, nil } // besideEnv is MESH_CHECK_BESIDE, naming the directory build_go_fetch placed the workspace's siblings in (ADR 0484 @@ -143,26 +183,71 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR return nil, err } began := m.clock() + // The directories that hold a selected package run side by side, up to the bound, each into a result of its own; + // the results join in directory order, so the answer does not depend on which ended first (ADR 0536 §3). + type dirRun struct { + res *TestResult + rd *reader + r runAnswer + noise []string + } + var todo []string for _, d := range dirs { - packages := byDir[d] - if len(packages) == 0 { - continue // no package asked for lies in this module + if len(byDir[d]) > 0 { // no package asked for lies in the others + todo = append(todo, d) } - args := []string{"-C", filepath.Join(tree, d), "test", "-json", "-count=1", "-timeout", testTimeout} - if len(o.tags) > 0 { - args = append(args, "-tags", strings.Join(o.tags, ",")) - } - if o.run != "" { - args = append(args, "-run", o.run) - } - args = append(args, packages...) - r := run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), o.env...), "go", args...) - rd := newReader(res, d) - noise := rd.read(r.Stdout) - rd.finish() - if (r.Status != 0 || r.Err != "") && len(rd.pkgs) == 0 { + } + runs := make([]dirRun, len(todo)) + slots := make(chan struct{}, m.parallel()) + var wg sync.WaitGroup + var crash any // a panic of a run goes on in the caller, which cleans up after it + var crashMu sync.Mutex + for i, d := range todo { + wg.Add(1) + slots <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-slots }() + defer func() { + if p := recover(); p != nil { + crashMu.Lock() + crash = p + crashMu.Unlock() + } + }() + args := []string{"-C", filepath.Join(tree, d), "test", "-json", "-count=1", "-timeout", testTimeout} + if len(o.tags) > 0 { + args = append(args, "-tags", strings.Join(o.tags, ",")) + } + if o.run != "" { + args = append(args, "-run", o.run) + } + args = append(args, byDir[d]...) + dr := &runs[i] + dr.res = &TestResult{} + dr.r = run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), o.env...), "go", args...) + dr.rd = newReader(dr.res, d) + dr.noise = dr.rd.read(dr.r.Stdout) + dr.rd.finish() + }() + } + wg.Wait() + if crash != nil { + panic(crash) + } + for i, d := range todo { + dr := runs[i] + if (dr.r.Status != 0 || dr.r.Err != "") && len(dr.rd.pkgs) == 0 { // go test said nothing a test said: it could not run (a go.mod it cannot read, a toolchain it cannot find). - return nil, fmt.Errorf("go test in %s: %s", d, firstLine(r.Err+"\n"+strings.Join(noise, "\n")+"\n"+r.Stderr)) + return nil, fmt.Errorf("go test in %s: %s", d, firstLine(dr.r.Err+"\n"+strings.Join(dr.noise, "\n")+"\n"+dr.r.Stderr)) + } + res.Tests += dr.res.Tests + res.Skipped += dr.res.Skipped + res.Failed = append(res.Failed, dr.res.Failed...) + res.Packages = append(res.Packages, dr.res.Packages...) + res.Slowest = append(res.Slowest, dr.res.Slowest...) + if dr.res.Elapsed > res.Elapsed { + res.Elapsed = dr.res.Elapsed } } if o.wall { @@ -172,6 +257,17 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR return res, nil } +// runAnswer is what a step runner answers. +type runAnswer = forgekey.Ran + +// parallel is how many directories a level runs at once. +func (m *Module) parallel() int { + if m.Parallel > 0 { + return m.Parallel + } + return max(1, runtime.NumCPU()/2) +} + // packagesByDir says which patterns run in which go.mod directory. A relative pattern (./...) runs in every directory, as // it always did; an import path, as build_go_digest keys its packages, runs only in the directory whose module holds it // (the longest module path that is the path or its prefix), as another module's path would fail the step. No pattern diff --git a/modules/mesh-build-go/cmd/mesh-build-go/verify_once_test.go b/modules/mesh-build-go/cmd/mesh-build-go/verify_once_test.go new file mode 100644 index 00000000..6d9776b4 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/verify_once_test.go @@ -0,0 +1,112 @@ +package main + +import ( + "context" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +func shutOffModule(t *testing.T, f *fakeStep) (*Module, *keptStates) { + t.Helper() + m, states := fetchModule(t, f) + // A module with bubblewrap runs its steps shut off, with the module cache read-only: a pass of go mod verify holds. + m.Bubblewrap = func(context.Context) (string, error) { return "bubblewrap 0.13.0", nil } + return m, states +} + +func countVerifies(f *fakeStep) int { + n := 0 + for _, l := range f.lines() { + if strings.HasPrefix(l, "go -C ") && strings.HasSuffix(l, " mod verify") { + n++ + } + } + return n +} + +// Issue 890/AC5 (ADR 0536 §2): two directories with the same go.mod and go.sum verify once, and a later fetch with the +// same pair verifies no time; a different go.mod verifies again. +func TestGoModVerifyRunsOncePerGoModAndGoSumDigest(t *testing.T) { + same := map[string]string{"a/go.mod": "module a\n", "a/go.sum": "x\n", "b/go.mod": "module a\n", "b/go.sum": "x\n"} + f := &fakeStep{gomods: "a/go.mod\nb/go.mod\n", files: same} + m, states := shutOffModule(t, f) + fetchOnce(t, m, states, fetchArgs) + if n := countVerifies(f); n != 1 { + t.Errorf("two directories with one go.mod and go.sum verified %d times, want 1", n) + } + fetchOnce(t, m, states, fetchArgs) + if n := countVerifies(f); n != 1 { + t.Errorf("a later fetch with the same pair verified again (%d in all)", n) + } + + g := &fakeStep{gomods: "a/go.mod\nb/go.mod\n", files: map[string]string{"a/go.mod": "module a\n", "a/go.sum": "x\n", "b/go.mod": "module b\n", "b/go.sum": "x\n"}} + m, states = shutOffModule(t, g) + fetchOnce(t, m, states, fetchArgs) + if n := countVerifies(g); n != 2 { + t.Errorf("two different go.mod verified %d times, want 2", n) + } +} + +// Without bubblewrap the module cache is not read-only to a step: a pass is never reused. +func TestAPassOfVerifyIsNotReusedWhenStepsAreNotShutOff(t *testing.T) { + f := &fakeStep{gomods: "a/go.mod\nb/go.mod\n", files: map[string]string{"a/go.mod": "m\n", "b/go.mod": "m\n"}} + m, states := fetchModule(t, f) // no Bubblewrap + fetchOnce(t, m, states, fetchArgs) + fetchOnce(t, m, states, fetchArgs) + if n := countVerifies(f); n != 4 { + t.Errorf("verified %d times, want every directory every fetch (4)", n) + } +} + +// A verify that fails leaves no mark: the next fetch verifies again. +func TestAFailedVerifyLeavesNoMark(t *testing.T) { + fail := true + f := &fakeStep{gomods: "go.mod\n", files: map[string]string{"go.mod": "m\n"}} + f.goRule = func(line string) forgekey.Ran { + if strings.HasSuffix(line, " mod verify") && fail { + return forgekey.Ran{Status: 1, Stdout: "x v1: dir has been modified\n"} + } + return forgekey.Ran{} + } + m, states := shutOffModule(t, f) + fetchTool(t, m)(fetchArgs) + states.waitEnd(t) + fail = false + fetchOnce(t, m, states, fetchArgs) + if n := countVerifies(f); n != 2 { + t.Errorf("verified %d times, want 2 (the failed one and the retry)", n) + } +} + +// Issue 890 (ADR 0536 §2): the downloads of the directories run side by side, never more at once than the bound. +func TestDownloadsRunSideBySideUpToTheBound(t *testing.T) { + var mu sync.Mutex + running, most := 0, 0 + f := &fakeStep{gomods: "a/go.mod\nb/go.mod\nc/go.mod\nd/go.mod\n", files: map[string]string{ + "a/go.mod": "a\n", "b/go.mod": "b\n", "c/go.mod": "c\n", "d/go.mod": "d\n"}} + f.goRule = func(line string) forgekey.Ran { + if strings.HasSuffix(line, " mod download") { + mu.Lock() + running++ + if running > most { + most = running + } + mu.Unlock() + time.Sleep(40 * time.Millisecond) + mu.Lock() + running-- + mu.Unlock() + } + return forgekey.Ran{} + } + m, states := fetchModule(t, f) + m.Parallel = 2 + fetchOnce(t, m, states, fetchArgs) + if most != 2 { + t.Errorf("at most %d downloads ran at once, want 2", most) + } +} diff --git a/modules/mesh-build-go/go.mod b/modules/mesh-build-go/go.mod index 999b31b9..e63fc0a5 100644 --- a/modules/mesh-build-go/go.mod +++ b/modules/mesh-build-go/go.mod @@ -3,7 +3,7 @@ module meshbuildgo go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023121526-2248b985bfac github.com/distribution/distribution/v3 v3.1.2 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/modules/mesh-build-go/go.sum b/modules/mesh-build-go/go.sum index ccb75980..868be86a 100644 --- a/modules/mesh-build-go/go.sum +++ b/modules/mesh-build-go/go.sum @@ -2,6 +2,10 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYE git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467 h1:i85+fFT0wZiTZ9CRVfMzWg1+Zr7oOLl3UH7hEaKcOIc= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023120306-780be2fae5a1 h1:MqP6mWWVxM+niUGPxSSh4Q8lduYkkQ7oTAkDeC9nyv4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023120306-780be2fae5a1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023121526-2248b985bfac h1:xx601bHwpO/aq5KyE08t4ymSXGHGnk+D9VHfmOq+5P8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023121526-2248b985bfac/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI= github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go index 76f46626..77bc463a 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go @@ -520,7 +520,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB passed := b.publish(t, "mesh.seat."+ControllerSeat+".event.checked", judgedAt("novox/app", 7, head, "pass", "t1t1t1t1t1t1")) // No review names the head, so nothing is asked (ADR 0496, issue 1803): the delivery waits for a review. - rt.until("the delivery to wait for a review", 5*time.Second, func() bool { + rt.until("the delivery to wait for a review", eventWithin, func() bool { e, err := b.kvs["pulls"].Get(ctx, "novox.app.7") var k PullKey return err == nil && json.Unmarshal(e.Value(), &k) == nil && k.Verdict == "pass" @@ -537,7 +537,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB "title": announced.Title, "head_sha": head, "base": "main", "body": "Reviewed-by: lead@shanks at " + head}) _ = passed var key string - rt.until("the ask to be written", 5*time.Second, func() bool { key = onlyAsk(b); return key != "" }) + rt.until("the ask to be written", eventWithin, func() bool { key = onlyAsk(b); return key != "" }) ask := askAt(t, b, key) t.Logf("the ask was written %s after the edit's event", time.Since(edit)) if !strings.Contains(ask.Ask.Explanation, "reviewed by lead on shanks") || ask.Ask.ID != sdk.AskID(key) { @@ -547,7 +547,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB t.Fatalf("merged before the Approve") } // The pull request's key holds its head, the trunk commit it was asked against and the verdict. - rt.until("the pull request's key to hold head, trunk and verdict", 5*time.Second, func() bool { + rt.until("the pull request's key to hold head, trunk and verdict", eventWithin, func() bool { e, err := b.kvs["pulls"].Get(ctx, "novox.app.7") var k PullKey return err == nil && json.Unmarshal(e.Value(), &k) == nil && k == PullKey{Head: head, Trunk: "t1t1t1t1t1t1", Verdict: "pass"} @@ -559,7 +559,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB t.Fatalf("the gate asked %s %d times: the bridge writes to the forge", tool, n) } } - rt.until("the owed keys to be written", 5*time.Second, func() bool { + rt.until("the owed keys to be written", eventWithin, func() bool { keys, err := b.kvs["owed"].Keys(ctx) return err == nil && len(keys) > 0 }) @@ -567,7 +567,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB // seconds, with the Approve's heads, and the gate asks the forge nothing. approved := b.answer(t, key, OptionApprove) var handed sdk.Merge - rt.until("the merge to be handed to the bridge", 5*time.Second, func() bool { handed = mergeAt(b, "fix/x"); return len(handed.Pulls) > 0 }) + rt.until("the merge to be handed to the bridge", eventWithin, func() bool { handed = mergeAt(b, "fix/x"); return len(handed.Pulls) > 0 }) if len(handed.Pulls) != 1 || handed.Pulls[0].Head.Head != head || handed.Pulls[0].Number != 7 || handed.Pulls[0].State != sdk.MergeMerging || handed.Approved[sdk.ApprovedKey("novox/app", 7)] != head || handed.Ask != key { t.Fatalf("the merge handed is %+v; wanted pull request 7 at the approved head, merging, under the ask %s", handed, key) @@ -579,7 +579,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB // The bridge says it merged; the mesh's source then holds the merge, in one write. b.forgeSays(t, sdk.Forge{Kind: sdk.ForgeMerged, Repository: "novox/app", Number: 7, Head: head, Merge: strings.Repeat("b", 40), Root: "r", Trees: map[string]string{"app": "t"}}) - rt.until("source.main to hold the merge", 5*time.Second, func() bool { + rt.until("source.main to hold the merge", eventWithin, func() bool { e, err := b.kvs["source"].Get(ctx, sdk.SourceMainKey) if err != nil { return false @@ -743,7 +743,7 @@ func TestAGroupOfTwoWithGreenChecksGetsOneAskAndOnItsApproveBothMergeOneAfterThe // One member is handed, with both heads approved, within seconds of the answer; the second only after the bridge says the // first merged. var first sdk.Merge - rt.until("the first member to be handed", 5*time.Second, func() bool { first = mergeAt(b, "feat/x"); return len(first.Pulls) > 0 }) + rt.until("the first member to be handed", eventWithin, func() bool { first = mergeAt(b, "feat/x"); return len(first.Pulls) > 0 }) if len(first.Pulls) != 1 || first.Pulls[0].State != sdk.MergeMerging || len(first.Approved) != 2 || first.Approved[sdk.ApprovedKey("novox/mesh-controller", 1)] != ctlHead || first.Approved[sdk.ApprovedKey("novox/mesh-catalog", 2)] != catHead { t.Fatalf("the first merge handed is %+v", first) @@ -757,14 +757,14 @@ func TestAGroupOfTwoWithGreenChecksGetsOneAskAndOnItsApproveBothMergeOneAfterThe b.forgeSays(t, sdk.Forge{Kind: sdk.ForgeMerged, Repository: one.Repository, Number: one.Number, Head: one.Head.Head, Merge: strings.Repeat("a", 40), Root: "r", Trees: map[string]string{"x": "t"}}) var both sdk.Merge - rt.until("the second member to be handed", 5*time.Second, func() bool { both = mergeAt(b, "feat/x"); return len(both.Pulls) == 2 }) + rt.until("the second member to be handed", eventWithin, func() bool { both = mergeAt(b, "feat/x"); return len(both.Pulls) == 2 }) if both.Pulls[0].State != sdk.MergeMerged || both.Pulls[1].State != sdk.MergeMerging || both.Pulls[1].Repository == one.Repository { t.Fatalf("the second merge handed is %+v", both.Pulls) } two := both.Pulls[1] b.forgeSays(t, sdk.Forge{Kind: sdk.ForgeMerged, Repository: two.Repository, Number: two.Number, Head: two.Head.Head, Merge: strings.Repeat("b", 40), Root: "r", Trees: map[string]string{"x": "t"}}) - rt.until("source.main to hold both merges", 5*time.Second, func() bool { + rt.until("source.main to hold both merges", eventWithin, func() bool { e, err := b.kvs["source"].Get(ctx, sdk.SourceMainKey) if err != nil { return false @@ -942,3 +942,9 @@ func mustDecode[T any](t *testing.T, raw []byte) T { } return v } + +// eventWithin is how long a test waits for one step of the gate's path over the real bus. On an idle machine each step +// takes milliseconds (the tests log it), but a slow, loaded build node ran the ask step in 6.69 s against the old 5 s +// (issue 1404). A failed first ask is asked again only after askAgainAfter (30 s, react.go), and 20 s deliberately does not +// cover that: a failed first ask still fails the test. +const eventWithin = 20 * time.Second diff --git a/modules/nats/cmd/mesh-nats-snapshot/dockerguard_test.go b/modules/nats/cmd/mesh-nats-snapshot/dockerguard_test.go new file mode 100644 index 00000000..7605d15c --- /dev/null +++ b/modules/nats/cmd/mesh-nats-snapshot/dockerguard_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "fmt" + "testing" +) + +// dockerVerdict decides what a test that starts containers does: run, skip or fail. found is the result of looking +// the docker program up, probeErr the result of asking the daemon for its version (nil when it answered). The checker's +// integration level runs shut off (no network, no /run), so on a build node the program is found and the daemon is not +// reachable; that is a skip, not a failure, except where docker is required (MESH_REQUIRE_DOCKER=1). +func dockerVerdict(found bool, lookErr, probeErr error, required bool) (action, reason string) { + var why string + switch { + case !found: + why = fmt.Sprintf("there is no docker in this toolchain (%v)", lookErr) + case probeErr != nil: + why = fmt.Sprintf("no docker daemon is reachable from here (%v)", probeErr) + default: + return "run", "" + } + if required { + return "fail", "MESH_REQUIRE_DOCKER=1 and " + why + } + return "skip", why +} + +func TestDockerVerdict(t *testing.T) { + boom := fmt.Errorf("cannot connect") + cases := []struct { + name string + found bool + look, probe error + required bool + want string + }{ + {"answers", true, nil, nil, false, "run"}, + {"answers, required", true, nil, nil, true, "run"}, + {"no program", false, boom, nil, false, "skip"}, + {"no program, required", false, boom, nil, true, "fail"}, + {"program but no daemon", true, nil, boom, false, "skip"}, + {"program but no daemon, required", true, nil, boom, true, "fail"}, + } + for _, c := range cases { + got, why := dockerVerdict(c.found, c.look, c.probe, c.required) + if got != c.want { + t.Errorf("%s: got %s, want %s", c.name, got, c.want) + } + if got != "run" && why == "" { + t.Errorf("%s: no reason given", c.name) + } + } +} diff --git a/modules/nats/cmd/mesh-nats-snapshot/live_test.go b/modules/nats/cmd/mesh-nats-snapshot/live_test.go index 99af6599..a9918d6c 100644 --- a/modules/nats/cmd/mesh-nats-snapshot/live_test.go +++ b/modules/nats/cmd/mesh-nats-snapshot/live_test.go @@ -60,14 +60,24 @@ var snapshotGrants = struct{ publish, subscribe []string }{ func needDocker(t *testing.T) { t.Helper() - if _, err := exec.LookPath("docker"); err != nil { - // The Go toolchain container of the merge check holds no docker. Where docker is expected - // (MESH_REQUIRE_DOCKER=1, which merge-check.sh sets for its docker step) its absence is a failure. - if os.Getenv("MESH_REQUIRE_DOCKER") == "1" { - t.Fatalf("MESH_REQUIRE_DOCKER=1 and there is no docker here: %v", err) + _, lookErr := exec.LookPath("docker") + var probeErr error + if lookErr == nil { + // The binary alone proves nothing: the checker's integration level runs shut off (no network, no /run), so on + // a build node docker is found and its daemon is not reachable (issue 1823). + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if out, err := exec.CommandContext(ctx, "docker", "version", "--format", "{{.Server.Version}}").CombinedOutput(); err != nil { + probeErr = fmt.Errorf("%v: %s", err, strings.TrimSpace(string(out))) } - t.Skipf("NOT RUN: this test starts a throwaway nats container and there is no docker in this toolchain (%v); "+ - "it is proof only where merge-check.sh's docker step runs it (MESH_CHECK_DOCKER=1)", err) + } + // Where docker is expected (MESH_REQUIRE_DOCKER=1, which merge-check.sh sets for its docker step) its absence is a failure. + switch action, why := dockerVerdict(lookErr == nil, lookErr, probeErr, os.Getenv("MESH_REQUIRE_DOCKER") == "1"); action { + case "fail": + t.Fatal(why) + case "skip": + t.Skipf("NOT RUN: this test starts a throwaway nats container and %s; "+ + "it is proof only where merge-check.sh's docker step runs it (MESH_CHECK_DOCKER=1)", why) } }