From 36c933541fd4e1a0c2e0fbd8d6b5b2aefb57f6b8 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 20:51:21 +0000 Subject: [PATCH] Add the Go build module's fetch, build and integration level steps, and the build tests (issues 1485, 1486, 1487), part 8 of 20 --- .../mesh-build-go/cmd/mesh-build-go/build.go | 164 +++++++++++++ .../cmd/mesh-build-go/build_test.go | 217 ++++++++++++++++++ .../mesh-build-go/cmd/mesh-build-go/fetch.go | 174 ++++++++++++++ .../cmd/mesh-build-go/integration.go | 77 +++++++ 4 files changed, 632 insertions(+) create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/build.go create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/build_test.go create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/fetch.go create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/integration.go diff --git a/modules/mesh-build-go/cmd/mesh-build-go/build.go b/modules/mesh-build-go/cmd/mesh-build-go/build.go new file mode 100644 index 00000000..b18915bf --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/build.go @@ -0,0 +1,164 @@ +package main + +// The build step (novox/hq ADR 0405 §3 §7, ADR 0453 §7; issue 1487): a Go bundle artifact of a module of a fetched +// workspace, built with today's recipe — the build agent's, mesh-controller's internal/builder/toolchain.go (the Go +// entry) and builder.go (compile, binaryName, pack) — by the installed Go and no network, packed with mesh-sdk's Pack +// so that its digest is the build agent's for the same tree. The archive is left in the workspace; publishing it is +// the builder's, in a later release. + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// The Go recipe of the build agent (toolchain.go, the entry of language go): static, trimmed of its own paths and of +// the commit stamp, stripped of symbols and debug info, and told the system it was built for. +const ( + recipeFlags = "-trimpath -buildvcs=false" + recipeLinkerFlags = "-s -w" + recipeSystemStamp = "main.builtFor" + // outDir is where a build leaves an artifact in the module's directory (builder.Out). + outDir = ".mesh-build" +) + +// BuildResult is what a passed build step says. +type BuildResult struct { + Artifact string `json:"artifact"` + Binary string `json:"binary"` + Digest string `json:"digest"` + Size int `json:"size"` + // Path is the archive, in the workspace's directory of the work directory. + Path string `json:"path"` +} + +func (m *Module) buildTool() StepTool { + return StepTool{ + Name: "build_go_build", + Description: "Build one Go bundle artifact of a module of a fetched workspace with the build agent's recipe, by the " + + "installed Go and no network, and pack it. Answers a step id at once; the step ends on its key with the " + + "artifact's digest, size and the archive's path.", + Input: map[string]any{ + "type": "object", + "properties": map[string]any{ + "workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"}, + "module": map[string]any{"type": "string", "description": "the module's directory in the workspace, . for the repository's root"}, + "artifact": map[string]any{"type": "string", "description": "the artifact's name in the module's manifest"}, + }, + "required": []string{"workspace", "module", "artifact"}, + }, + Long: true, + Run: m.build, + } +} + +// binaryName is what a compiled bundle's executable is called (builder.go binaryName): what the artifact says, or the +// name of the package it is built from, or the artifact's. +func binaryName(a catalogue.Artifact) string { + if name := catalogue.BinaryOf(a); name != "" { + return name + } + if name := strings.TrimSpace(a.Binary); name != "" { + return name + } + if from := strings.Trim(a.From, "./"); from != "" { + return filepath.Base(from) + } + return a.Name +} + +func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { + workspace, _ := args["workspace"].(string) + module, _ := args["module"].(string) + name, _ := args["artifact"].(string) + if !workspaceOK.MatchString(workspace) { + return nil, fmt.Errorf("%q is not a workspace id", workspace) + } + tree := filepath.Join(m.Work, workspace, "tree") + dir := filepath.Join(tree, filepath.FromSlash(module)) + if rel, err := filepath.Rel(tree, dir); err != nil || strings.HasPrefix(rel, "..") { + return nil, fmt.Errorf("the module directory %q lies outside the workspace", module) + } + raw, err := os.ReadFile(filepath.Join(dir, "module.json")) + if err != nil { + return nil, fmt.Errorf("no module in %q of workspace %s: %w", module, workspace, err) + } + manifest, err := catalogue.ParseManifest(raw) + if err != nil { + return nil, fmt.Errorf("the manifest in %q: %w", module, err) + } + var art *catalogue.Artifact + names := []string{} + if manifest.Build != nil { + for i := range manifest.Build.Artifacts { + names = append(names, manifest.Build.Artifacts[i].Name) + if manifest.Build.Artifacts[i].Name == name { + art = &manifest.Build.Artifacts[i] + } + } + } + if art == nil { + return nil, fmt.Errorf("the module %s has no artifact %q (it has %s)", manifest.Module, name, strings.Join(names, ", ")) + } + if art.Kind != "bundle" || art.Language != "go" { + return nil, fmt.Errorf("the artifact %s of %s is a %s of language %q: this module builds Go bundles", name, manifest.Module, art.Kind, art.Language) + } + if strings.TrimSpace(art.From) == "" { + return nil, fmt.Errorf("the artifact %s of %s says nothing it is built from", name, manifest.Module) + } + + // Its own output directory, made empty: a build of the same tree twice is one build. + out := filepath.Join(dir, outDir, art.Name) + if err := os.RemoveAll(out); err != nil { + return nil, err + } + if err := os.MkdirAll(out, 0o755); err != nil { + return nil, fmt.Errorf("making the output directory for %s: %w", art.Name, err) + } + binary := binaryName(*art) + // One -ldflags, composed: a repeated flag is not a merged one (the build agent's issue 161). + linker := recipeLinkerFlags + if system := strings.TrimSpace(art.System); system != "" { + linker += " -X " + recipeSystemStamp + "=" + system + } + cmd := []string{"-C", dir, "build", "-ldflags", linker, "-o", filepath.Join(out, binary), "./" + strings.Trim(art.From, "./")} + if r := m.Run(ctx, m.buildEnv(dir), "go", cmd...); r.Status != 0 || r.Err != "" { + return nil, fmt.Errorf("go build of %s: %s", art.Name, firstLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout)) + } + archive, digest, err := buildstep.Pack(out) + if err != nil { + return nil, fmt.Errorf("packing %s: %w", art.Name, err) + } + artifacts := filepath.Join(m.Work, workspace, "artifacts") + if err := os.MkdirAll(artifacts, 0o700); err != nil { + return nil, err + } + path := filepath.Join(artifacts, strings.ReplaceAll(module, "/", "_")+"."+art.Name+".tar.gz") + if err := os.WriteFile(path, archive, 0o600); err != nil { + return nil, err + } + return BuildResult{Artifact: art.Name, Binary: binary, Digest: digest, Size: len(archive), Path: path}, nil +} + +// buildEnv is the environment of the compile: the build agent's (CGO_ENABLED=0, GOFLAGS) with the installed toolchain +// alone, the modules the fetch put in the module's cache and no way to fetch another (GOPROXY=off), so a build never +// reaches the network. A directory that vendors is built from its vendor directory. +func (m *Module) buildEnv(dir string) []string { + mod := "-mod=mod" + if _, err := os.Stat(filepath.Join(dir, "vendor", "modules.txt")); err == nil { + mod = "-mod=vendor" + } + return []string{ + "CGO_ENABLED=0", + "GOFLAGS=" + recipeFlags + " " + mod, + "GOTOOLCHAIN=local", + "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), + "GOCACHE=" + filepath.Join(m.Cache, "gobuild"), + "GOPROXY=off", + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/build_test.go b/modules/mesh-build-go/cmd/mesh-build-go/build_test.go new file mode 100644 index 00000000..181f7d87 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/build_test.go @@ -0,0 +1,217 @@ +package main + +import ( + "archive/tar" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// Issue 1487: the build step, with the compiler faked at the process boundary: it writes the file `go build -o` names. + +func manifestWith(artifact string) string { + return `{"module":"demo","version":"1","build":{"artifacts":[` + artifact + `]}}` +} + +const goBundle = `{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/demo-tools","binary":"demo-bin","loads":["demo-bin"]}` + +// buildModule is a module with a workspace holding a module "modules/demo" with this manifest; the compiler writes +// content to the file go build is told. +func buildModule(t *testing.T, manifest, content string) (*Module, *keptStates, *[]cmdRecord) { + t.Helper() + calls := &[]cmdRecord{} + states := newStates() + m := moduleWith(states, "go1.27.1") + m.Home, m.Work, m.Cache = t.TempDir(), t.TempDir(), t.TempDir() + dir := filepath.Join(m.Work, "ws1", "tree", "modules", "demo") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(manifest), 0o644); err != nil { + t.Fatal(err) + } + m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + *calls = append(*calls, cmdRecord{env: env, line: name + " " + strings.Join(args, " ")}) + for i, a := range args { + if a == "-o" { + if err := os.WriteFile(args[i+1], []byte(content), 0o755); err != nil { + return forgekey.Ran{Status: 1, Stderr: err.Error()} + } + } + } + return forgekey.Ran{} + } + return m, states, calls +} + +func runBuild(t *testing.T, m *Module, states *keptStates, artifact string) (BuildResult, delivery.Step) { + t.Helper() + for _, tool := range m.Tools(m.StepTools()) { + if tool.Name == "build_go_build" { + if _, err := tool.Run(map[string]any{"workspace": "ws1", "module": "modules/demo", "artifact": artifact}); err != nil { + t.Fatal(err) + } + } + } + st := states.waitEnd(t) + var res BuildResult + if st.State == delivery.StepPassed { + if err := json.Unmarshal(st.Result, &res); err != nil { + t.Fatal(err) + } + } + return res, st +} + +// goEntry reads what the build agent's recipe says out of lines 155-191 of mesh-control's internal/builder/toolchain.go +// (testdata/captured/toolchain-go-entry.txt, copied from origin/main 073ccc29): the environment words and flags of the +// compile, the linker flags and the system stamp. +func goEntry(t *testing.T) (compile []string, linker []string, stamp string) { + t.Helper() + raw, err := os.ReadFile(filepath.Join("testdata", "captured", "toolchain-go-entry.txt")) + if err != nil { + t.Fatal(err) + } + text := string(raw) + find := func(pattern string) string { + m := regexp.MustCompile(pattern).FindStringSubmatch(text) + if m == nil { + t.Fatalf("the copied recipe has nothing like %s", pattern) + } + return m[1] + } + for _, q := range regexp.MustCompile(`"([^"]+)"`).FindAllStringSubmatch(find(`Compile: \[\]string\{([^}]*)\}`), -1) { + compile = append(compile, q[1]) + } + for _, q := range regexp.MustCompile(`"([^"]+)"`).FindAllStringSubmatch(find(`LinkerFlags: \[\]string\{([^}]*)\}`), -1) { + linker = append(linker, q[1]) + } + return compile, linker, find(`SystemStamp: "([^"]+)"`) +} + +// 1487/AC1: the build command carries today's flags and environment: the copied recipe says env CGO_ENABLED=0 +// GOFLAGS=-trimpath -buildvcs=false, go build, -ldflags -s -w -X main.builtFor=, -o /, the package. +func TestTheBuildCommandCarriesTodaysFlagsAndEnvironment(t *testing.T) { + m, states, calls := buildModule(t, manifestWith(goBundle), "binary") + res, st := runBuild(t, m, states, "tools") + if st.State != delivery.StepPassed { + t.Fatalf("the step ended %s: %s", st.State, st.Reason) + } + compile, linker, stamp := goEntry(t) + // compile is: env CGO_ENABLED=0 "GOFLAGS=-trimpath -buildvcs=false" go build + if len(compile) != 5 || compile[0] != "env" || compile[3] != "go" || compile[4] != "build" { + t.Fatalf("the copied recipe's compile is %v", compile) + } + var c cmdRecord + for _, r := range *calls { + if strings.HasPrefix(r.line, "go ") { + c = r + } + } + env := strings.Join(c.env, "\n") + for _, want := range []string{compile[1], "GOFLAGS=" + strings.TrimPrefix(compile[2], "GOFLAGS=") + " -mod=mod", "GOTOOLCHAIN=local", "GOPROXY=off", + "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), "GOCACHE=" + filepath.Join(m.Cache, "gobuild")} { + if !strings.Contains(env, want) { + t.Errorf("the environment lacks %q:\n%s", want, env) + } + } + dir := filepath.Join(m.Work, "ws1", "tree", "modules", "demo") + out := filepath.Join(dir, ".mesh-build", "tools") + want := "go -C " + dir + " build -ldflags " + strings.Join(linker, " ") + " -X " + stamp + "=arch -o " + filepath.Join(out, "demo-bin") + " ./cmd/demo-tools" + if c.line != want { + t.Errorf("command = %q\nwant %q", c.line, want) + } + if res.Artifact != "tools" || res.Binary != "demo-bin" || !strings.HasPrefix(res.Digest, "sha256:") || res.Size == 0 { + t.Errorf("result = %+v", res) + } +} + +// The archive is the file the digest names, holding one executable of the name binaryName says; two builds of one tree +// are one digest. +func TestTheArchiveHoldsOneExecutableAndTwoBuildsAreOneDigest(t *testing.T) { + m, states, _ := buildModule(t, manifestWith(goBundle), "the program") + first, _ := runBuild(t, m, states, "tools") + second, _ := runBuild(t, m, states, "tools") + if first.Digest != second.Digest || first.Path != second.Path { + t.Errorf("two builds: %+v and %+v", first, second) + } + raw, err := os.ReadFile(first.Path) + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(raw) + if first.Digest != "sha256:"+hex.EncodeToString(sum[:]) { + t.Errorf("the digest %s is not the archive's", first.Digest) + } + zr, err := gzip.NewReader(strings.NewReader(string(raw))) + if err != nil { + t.Fatal(err) + } + tr := tar.NewReader(zr) + h, err := tr.Next() + if err != nil || h.Name != "demo-bin" || h.Mode != 0o755 { + t.Fatalf("first entry = %+v, %v", h, err) + } + if body, _ := io.ReadAll(tr); string(body) != "the program" { + t.Errorf("the executable holds %q", body) + } + if _, err := tr.Next(); err != io.EOF { + t.Errorf("the archive holds more than one file") + } +} + +// 1487/AC3: an artifact of another language, or another kind, fails the step naming it. +func TestAnArtifactOfAnotherLanguageFailsTheStepNamingIt(t *testing.T) { + for artifact, want := range map[string]string{ + `{"name":"tools","kind":"bundle","language":"typescript","entrypoints":["index.js"],"loads":["index.js"]}`: `language "typescript"`, + `{"name":"tools","kind":"archive","from":"files"}`: "archive", + } { + m, states, calls := buildModule(t, manifestWith(artifact), "x") + _, st := runBuild(t, m, states, "tools") + if st.State != delivery.StepFailed || !strings.Contains(st.Reason, "tools") || !strings.Contains(st.Reason, want) || len(*calls) != 0 { + t.Errorf("%s: the step ended %s: %q after %d commands", artifact, st.State, st.Reason, len(*calls)) + } + } + m, states, _ := buildModule(t, manifestWith(goBundle), "x") + if _, st := runBuild(t, m, states, "nope"); st.State != delivery.StepFailed || !strings.Contains(st.Reason, `"nope"`) || !strings.Contains(st.Reason, "tools") { + t.Errorf("an unknown artifact: %s %q", st.State, st.Reason) + } +} + +// The executable is named as the build agent names it: the artifact's binary, else the package's directory, else the +// artifact. +func TestTheBinaryIsNamedAsTheBuildAgentNamesIt(t *testing.T) { + for artifact, want := range map[string]string{ + goBundle: "demo-bin", + `{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/demo-tools","loads":["demo-tools"]}`: "demo-tools", + `{"name":"tools","kind":"bundle","language":"go","system":"arch","from":".","loads":["tools"]}`: "tools", + } { + m, states, _ := buildModule(t, manifestWith(artifact), "x") + if res, st := runBuild(t, m, states, "tools"); res.Binary != want { + t.Errorf("%s: binary %q, want %q (%s %s)", artifact, res.Binary, want, st.State, st.Reason) + } + } +} + +func TestAModuleDirectoryOutsideTheWorkspaceIsRefused(t *testing.T) { + m, states, calls := buildModule(t, manifestWith(goBundle), "x") + for _, tool := range m.Tools(m.StepTools()) { + if tool.Name == "build_go_build" { + tool.Run(map[string]any{"workspace": "ws1", "module": "../../..", "artifact": "tools"}) + } + } + if st := states.waitEnd(t); st.State != delivery.StepFailed || len(*calls) != 0 { + t.Errorf("the step ended %s after %d commands", st.State, len(*calls)) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go new file mode 100644 index 00000000..bc09c9b3 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -0,0 +1,174 @@ +package main + +// The fetch step (novox/hq ADR 0405 §5 §6, ADR 0466 §4 §5, ADR 0469 §2; issue 1485): a commit of a repository of +// the mesh's forge, fetched over SSH as the module's own account into a workspace of its own, and the Go modules every +// go.mod in it names, downloaded into the module's cache and verified against go.sum. + +import ( + "context" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/buildstep" +) + +// ForgeAlias is the ssh name the forge-account provision sets up in the account's ssh files (ADR 0409 §1); the +// remote of a fetch is ssh:////.git. +const ForgeAlias = "mesh-forge" + +// WorkspaceAge is how long a workspace is kept: a module removes the workspaces older than a day, each time it +// fetches and every round it checks its toolchain. A workspace lies in the module's work directory, never in its +// home (ADR 0466 §1). +const WorkspaceAge = 24 * time.Hour + +var ( + repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`) + workspaceOK = regexp.MustCompile(`^[a-z0-9]+$`) +) + +// FetchResult is what a passed fetch step says: the workspace it made, the commit in it and the directories whose +// Go modules it downloaded. +type FetchResult struct { + Workspace string `json:"workspace"` + Commit string `json:"commit"` + Modules []string `json:"modules"` +} + +// fetchTool is the fetch step tool. +func (m *Module) fetchTool() StepTool { + return StepTool{ + Name: "build_go_fetch", + Description: "Fetch a commit of a repository of the mesh's forge over SSH, as this module's own account, into a " + + "workspace of its own, and download and verify the Go modules every go.mod in it names. Answers a step id at " + + "once; the step ends on its key in the steps state with the workspace's id.", + Input: map[string]any{ + "type": "object", + "properties": map[string]any{ + "repository": map[string]any{"type": "string", "description": "/ on the forge"}, + "commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"}, + "ref": map[string]any{"type": "string", "description": "refs/pull//head when only a pull request's head carries the commit"}, + }, + "required": []string{"repository", "commit"}, + }, + Long: true, + Run: m.fetch, + } +} + +func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) { + repository, _ := args["repository"].(string) + commit, _ := args["commit"].(string) + ref, _ := args["ref"].(string) + if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") { + return nil, fmt.Errorf("%q is not a repository, /", repository) + } + m.CleanWorkspaces(m.now()) + id := buildstep.NewID() + root := filepath.Join(m.Work, id) + tree := filepath.Join(root, "tree") + if err := os.MkdirAll(tree, 0o700); err != nil { + return nil, err + } + remote := "ssh://" + ForgeAlias + "/" + repository + ".git" + if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref); err != nil { + return nil, err + } + dirs, err := m.goModDirs(ctx, tree) + if err != nil { + return nil, err + } + for _, d := range dirs { + if _, err := os.Stat(filepath.Join(tree, d, "vendor", "modules.txt")); err == nil { + continue // a module that vendors resolves nothing + } + for _, verb := range []string{"download", "verify"} { + r := m.Run(ctx, m.goEnv(), "go", "-C", filepath.Join(tree, d), "mod", verb) + if r.Status != 0 || r.Err != "" { + return nil, fmt.Errorf("go mod %s in %s: %s", verb, d, firstLine(r.Err+"\n"+r.Stdout+"\n"+r.Stderr)) + } + } + } + return FetchResult{Workspace: id, Commit: commit, Modules: dirs}, nil +} + +// goModDirs are the directories of the tree that hold a go.mod, relative to it, in order. +func (m *Module) goModDirs(ctx context.Context, tree string) ([]string, error) { + r := m.Run(ctx, nil, "git", "-C", tree, "ls-files", "--", "go.mod", "*/go.mod") + if r.Status != 0 || r.Err != "" { + return nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr)) + } + dirs := []string{} + for _, l := range strings.Split(r.Stdout, "\n") { + if l = strings.TrimSpace(l); l != "" { + dirs = append(dirs, filepath.ToSlash(filepath.Dir(l))) + } + } + sort.Strings(dirs) + return dirs, nil +} + +// goEnv is what a go command of a step is given beyond the base environment: the installed toolchain and no other, +// modules resolved by their go.mod and go.sum, the module's own caches, the mesh's own modules fetched directly (by +// git, through the account's rewrite of the forge's address to ssh), and nothing that asks a question. For a directory +// that vendors, see goEnvFor. +func (m *Module) goEnv() []string { + return []string{ + "GOTOOLCHAIN=local", + "GOFLAGS=-mod=mod", + "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), + "GOCACHE=" + filepath.Join(m.Cache, "gobuild"), + "GOPRIVATE=git.novox.be", + "GIT_TERMINAL_PROMPT=0", + "GIT_SSH_COMMAND=ssh -o BatchMode=yes", + } +} + +// goEnvFor is goEnv for a command run in dir: a directory that vendors (vendor/modules.txt) is read from its vendor +// directory, as fetch downloaded nothing for it, and every other by its go.mod and go.sum. +func (m *Module) goEnvFor(dir string) []string { + env := m.goEnv() + if _, err := os.Stat(filepath.Join(dir, "vendor", "modules.txt")); err == nil { + env = append(env, "GOFLAGS=-mod=vendor") // the last value of a name is the one used + } + return env +} + +// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory. +func (m *Module) CleanWorkspaces(now time.Time) { + entries, err := os.ReadDir(m.Work) + if err != nil { + return + } + for _, e := range entries { + info, err := e.Info() + if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge { + continue + } + if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil { + m.Say("the workspace %s could not be removed: %v", e.Name(), err) + } + } +} + +// clock is the time now: Now when a test set it, else the wall clock. +func (m *Module) clock() time.Time { + if m.Now != nil { + return m.Now() + } + return time.Now() +} +func (m *Module) now() time.Time { return m.clock() } + +func firstLine(s string) string { + for _, l := range strings.Split(s, "\n") { + if l = strings.TrimSpace(l); l != "" { + return l + } + } + return "" +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/integration.go b/modules/mesh-build-go/cmd/mesh-build-go/integration.go new file mode 100644 index 00000000..d96c4bb5 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/integration.go @@ -0,0 +1,77 @@ +package main + +// The integration level (novox/hq ADR 0445 §2 §3, ADR 0470; issue 1486): `go test -tags integration` over the +// workspace, against a throwaway postgres and a throwaway bus started for this run alone, handed to the tests as +// MESH_TEST_POSTGRES and MESH_TEST_NATS. Its setup (compiling, the store, the bus) and its running time are measured and +// judged apart, 30 s and 60 s; both servers are stopped and the run's directory removed whatever the tests did. + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strconv" + "time" + + "git.novox.be/novox/mesh-sdk/go/buildstep" +) + +// The integration level's bounds (ADR 0445 §3). +const ( + IntegrationSetup = 30 * time.Second + IntegrationRun = 60 * time.Second +) + +func (m *Module) integrationBounds() Bounds { + b := m.IntegrationBounds + if b.Setup <= 0 { + b.Setup = IntegrationSetup + } + if b.Run <= 0 { + b.Run = IntegrationRun + } + return b +} + +func (m *Module) integration(ctx context.Context, workspace, tree string, packages []string) (res *TestResult, err error) { + major, err := m.checkPostgres(ctx) + if err != nil { + return nil, err // a *buildstep.Refusal: this level only + } + run := filepath.Join(m.Work, workspace, "run-"+buildstep.NewID()) + if err := os.MkdirAll(run, 0o700); err != nil { + return nil, err + } + // The run's directory goes, and the servers stop, however the level ends: passed, failed, a panic. + defer os.RemoveAll(run) + + began := m.clock() + dirs, err := m.goModDirs(ctx, tree) + if err != nil { + return nil, err + } + postgres, stopStore, err := m.startStore(ctx, run) + if err != nil { + return nil, fmt.Errorf("the integration level's store: %w", err) + } + defer stopStore() + nats, stopBus, err := m.startBus(ctx, filepath.Join(run, "nats")) + if err != nil { + return nil, fmt.Errorf("the integration level's bus: %w", err) + } + defer stopBus() + env := []string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats} + // Compiling counts as setup: the tests are built once, here, with the addresses a TestMain may read, and the run + // below finds them built. Whether they compile is for the run to say; its failure is not read here. + for _, d := range dirs { + m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), env...), "go", "-C", filepath.Join(tree, d), "test", "-tags", "integration", "-run", "^$", "-count=1", "./...") + } + setup := m.clock().Sub(began) + + return m.runLevel(ctx, tree, levelOpts{ + level: LevelIntegration, tags: []string{"integration"}, packages: packages, bounds: m.integrationBounds(), + env: env, + setup: setup, wall: true, + store: "postgres " + strconv.Itoa(major), bus: "nats-server " + m.Recipe.NatsServer, + }) +}