Add the Go build module's fetch, build and integration level steps, and the build tests (issues 1485, 1486, 1487), part 8 of 20

modules/mesh-build-go/cmd/mesh-build-go/build.go

@@ -0,0 +1,164 @@
+package main
+
+// The build step (novox/hq ADR 0405 §3 §7, ADR 0453 §7; issue 1487): a Go bundle artifact of a module of a fetched
+// workspace, built with today's recipe — the build agent's, mesh-controller's internal/builder/toolchain.go (the Go
+// entry) and builder.go (compile, binaryName, pack) — by the installed Go and no network, packed with mesh-sdk's Pack
+// so that its digest is the build agent's for the same tree. The archive is left in the workspace; publishing it is
+// the builder's, in a later release.
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"git.novox.be/novox/mesh-sdk/go/buildstep"
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+)
+
+// The Go recipe of the build agent (toolchain.go, the entry of language go): static, trimmed of its own paths and of
+// the commit stamp, stripped of symbols and debug info, and told the system it was built for.
+const (
+	recipeFlags       = "-trimpath -buildvcs=false"
+	recipeLinkerFlags = "-s -w"
+	recipeSystemStamp = "main.builtFor"
+	// outDir is where a build leaves an artifact in the module's directory (builder.Out).
+	outDir = ".mesh-build"
+)
+
+// BuildResult is what a passed build step says.
+type BuildResult struct {
+	Artifact string `json:"artifact"`
+	Binary   string `json:"binary"`
+	Digest   string `json:"digest"`
+	Size     int    `json:"size"`
+	// Path is the archive, in the workspace's directory of the work directory.
+	Path string `json:"path"`
+}
+
+func (m *Module) buildTool() StepTool {
+	return StepTool{
+		Name: "build_go_build",
+		Description: "Build one Go bundle artifact of a module of a fetched workspace with the build agent's recipe, by the " +
+			"installed Go and no network, and pack it. Answers a step id at once; the step ends on its key with the " +
+			"artifact's digest, size and the archive's path.",
+		Input: map[string]any{
+			"type": "object",
+			"properties": map[string]any{
+				"workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"},
+				"module":    map[string]any{"type": "string", "description": "the module's directory in the workspace, . for the repository's root"},
+				"artifact":  map[string]any{"type": "string", "description": "the artifact's name in the module's manifest"},
+			},
+			"required": []string{"workspace", "module", "artifact"},
+		},
+		Long: true,
+		Run:  m.build,
+	}
+}
+
+// binaryName is what a compiled bundle's executable is called (builder.go binaryName): what the artifact says, or the
+// name of the package it is built from, or the artifact's.
+func binaryName(a catalogue.Artifact) string {
+	if name := catalogue.BinaryOf(a); name != "" {
+		return name
+	}
+	if name := strings.TrimSpace(a.Binary); name != "" {
+		return name
+	}
+	if from := strings.Trim(a.From, "./"); from != "" {
+		return filepath.Base(from)
+	}
+	return a.Name
+}
+
+func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
+	workspace, _ := args["workspace"].(string)
+	module, _ := args["module"].(string)
+	name, _ := args["artifact"].(string)
+	if !workspaceOK.MatchString(workspace) {
+		return nil, fmt.Errorf("%q is not a workspace id", workspace)
+	}
+	tree := filepath.Join(m.Work, workspace, "tree")
+	dir := filepath.Join(tree, filepath.FromSlash(module))
+	if rel, err := filepath.Rel(tree, dir); err != nil || strings.HasPrefix(rel, "..") {
+		return nil, fmt.Errorf("the module directory %q lies outside the workspace", module)
+	}
+	raw, err := os.ReadFile(filepath.Join(dir, "module.json"))
+	if err != nil {
+		return nil, fmt.Errorf("no module in %q of workspace %s: %w", module, workspace, err)
+	}
+	manifest, err := catalogue.ParseManifest(raw)
+	if err != nil {
+		return nil, fmt.Errorf("the manifest in %q: %w", module, err)
+	}
+	var art *catalogue.Artifact
+	names := []string{}
+	if manifest.Build != nil {
+		for i := range manifest.Build.Artifacts {
+			names = append(names, manifest.Build.Artifacts[i].Name)
+			if manifest.Build.Artifacts[i].Name == name {
+				art = &manifest.Build.Artifacts[i]
+			}
+		}
+	}
+	if art == nil {
+		return nil, fmt.Errorf("the module %s has no artifact %q (it has %s)", manifest.Module, name, strings.Join(names, ", "))
+	}
+	if art.Kind != "bundle" || art.Language != "go" {
+		return nil, fmt.Errorf("the artifact %s of %s is a %s of language %q: this module builds Go bundles", name, manifest.Module, art.Kind, art.Language)
+	}
+	if strings.TrimSpace(art.From) == "" {
+		return nil, fmt.Errorf("the artifact %s of %s says nothing it is built from", name, manifest.Module)
+	}
+
+	// Its own output directory, made empty: a build of the same tree twice is one build.
+	out := filepath.Join(dir, outDir, art.Name)
+	if err := os.RemoveAll(out); err != nil {
+		return nil, err
+	}
+	if err := os.MkdirAll(out, 0o755); err != nil {
+		return nil, fmt.Errorf("making the output directory for %s: %w", art.Name, err)
+	}
+	binary := binaryName(*art)
+	// One -ldflags, composed: a repeated flag is not a merged one (the build agent's issue 161).
+	linker := recipeLinkerFlags
+	if system := strings.TrimSpace(art.System); system != "" {
+		linker += " -X " + recipeSystemStamp + "=" + system
+	}
+	cmd := []string{"-C", dir, "build", "-ldflags", linker, "-o", filepath.Join(out, binary), "./" + strings.Trim(art.From, "./")}
+	if r := m.Run(ctx, m.buildEnv(dir), "go", cmd...); r.Status != 0 || r.Err != "" {
+		return nil, fmt.Errorf("go build of %s: %s", art.Name, firstLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout))
+	}
+	archive, digest, err := buildstep.Pack(out)
+	if err != nil {
+		return nil, fmt.Errorf("packing %s: %w", art.Name, err)
+	}
+	artifacts := filepath.Join(m.Work, workspace, "artifacts")
+	if err := os.MkdirAll(artifacts, 0o700); err != nil {
+		return nil, err
+	}
+	path := filepath.Join(artifacts, strings.ReplaceAll(module, "/", "_")+"."+art.Name+".tar.gz")
+	if err := os.WriteFile(path, archive, 0o600); err != nil {
+		return nil, err
+	}
+	return BuildResult{Artifact: art.Name, Binary: binary, Digest: digest, Size: len(archive), Path: path}, nil
+}
+
+// buildEnv is the environment of the compile: the build agent's (CGO_ENABLED=0, GOFLAGS) with the installed toolchain
+// alone, the modules the fetch put in the module's cache and no way to fetch another (GOPROXY=off), so a build never
+// reaches the network. A directory that vendors is built from its vendor directory.
+func (m *Module) buildEnv(dir string) []string {
+	mod := "-mod=mod"
+	if _, err := os.Stat(filepath.Join(dir, "vendor", "modules.txt")); err == nil {
+		mod = "-mod=vendor"
+	}
+	return []string{
+		"CGO_ENABLED=0",
+		"GOFLAGS=" + recipeFlags + " " + mod,
+		"GOTOOLCHAIN=local",
+		"GOMODCACHE=" + filepath.Join(m.Cache, "gomod"),
+		"GOCACHE=" + filepath.Join(m.Cache, "gobuild"),
+		"GOPROXY=off",
+	}
+}

modules/mesh-build-go/cmd/mesh-build-go/build_test.go

@@ -0,0 +1,217 @@
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+	"git.novox.be/novox/mesh-sdk/go/forgekey"
+)
+
+// Issue 1487: the build step, with the compiler faked at the process boundary: it writes the file `go build -o` names.
+
+func manifestWith(artifact string) string {
+	return `{"module":"demo","version":"1","build":{"artifacts":[` + artifact + `]}}`
+}
+
+const goBundle = `{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/demo-tools","binary":"demo-bin","loads":["demo-bin"]}`
+
+// buildModule is a module with a workspace holding a module "modules/demo" with this manifest; the compiler writes
+// content to the file go build is told.
+func buildModule(t *testing.T, manifest, content string) (*Module, *keptStates, *[]cmdRecord) {
+	t.Helper()
+	calls := &[]cmdRecord{}
+	states := newStates()
+	m := moduleWith(states, "go1.27.1")
+	m.Home, m.Work, m.Cache = t.TempDir(), t.TempDir(), t.TempDir()
+	dir := filepath.Join(m.Work, "ws1", "tree", "modules", "demo")
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(manifest), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
+		*calls = append(*calls, cmdRecord{env: env, line: name + " " + strings.Join(args, " ")})
+		for i, a := range args {
+			if a == "-o" {
+				if err := os.WriteFile(args[i+1], []byte(content), 0o755); err != nil {
+					return forgekey.Ran{Status: 1, Stderr: err.Error()}
+				}
+			}
+		}
+		return forgekey.Ran{}
+	}
+	return m, states, calls
+}
+
+func runBuild(t *testing.T, m *Module, states *keptStates, artifact string) (BuildResult, delivery.Step) {
+	t.Helper()
+	for _, tool := range m.Tools(m.StepTools()) {
+		if tool.Name == "build_go_build" {
+			if _, err := tool.Run(map[string]any{"workspace": "ws1", "module": "modules/demo", "artifact": artifact}); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+	st := states.waitEnd(t)
+	var res BuildResult
+	if st.State == delivery.StepPassed {
+		if err := json.Unmarshal(st.Result, &res); err != nil {
+			t.Fatal(err)
+		}
+	}
+	return res, st
+}
+
+// goEntry reads what the build agent's recipe says out of lines 155-191 of mesh-control's internal/builder/toolchain.go
+// (testdata/captured/toolchain-go-entry.txt, copied from origin/main 073ccc29): the environment words and flags of the
+// compile, the linker flags and the system stamp.
+func goEntry(t *testing.T) (compile []string, linker []string, stamp string) {
+	t.Helper()
+	raw, err := os.ReadFile(filepath.Join("testdata", "captured", "toolchain-go-entry.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	text := string(raw)
+	find := func(pattern string) string {
+		m := regexp.MustCompile(pattern).FindStringSubmatch(text)
+		if m == nil {
+			t.Fatalf("the copied recipe has nothing like %s", pattern)
+		}
+		return m[1]
+	}
+	for _, q := range regexp.MustCompile(`"([^"]+)"`).FindAllStringSubmatch(find(`Compile: \[\]string\{([^}]*)\}`), -1) {
+		compile = append(compile, q[1])
+	}
+	for _, q := range regexp.MustCompile(`"([^"]+)"`).FindAllStringSubmatch(find(`LinkerFlags: \[\]string\{([^}]*)\}`), -1) {
+		linker = append(linker, q[1])
+	}
+	return compile, linker, find(`SystemStamp: "([^"]+)"`)
+}
+
+// 1487/AC1: the build command carries today's flags and environment: the copied recipe says env CGO_ENABLED=0
+// GOFLAGS=-trimpath -buildvcs=false, go build, -ldflags -s -w -X main.builtFor=<system>, -o <out>/<binary>, the package.
+func TestTheBuildCommandCarriesTodaysFlagsAndEnvironment(t *testing.T) {
+	m, states, calls := buildModule(t, manifestWith(goBundle), "binary")
+	res, st := runBuild(t, m, states, "tools")
+	if st.State != delivery.StepPassed {
+		t.Fatalf("the step ended %s: %s", st.State, st.Reason)
+	}
+	compile, linker, stamp := goEntry(t)
+	// compile is: env CGO_ENABLED=0 "GOFLAGS=-trimpath -buildvcs=false" go build
+	if len(compile) != 5 || compile[0] != "env" || compile[3] != "go" || compile[4] != "build" {
+		t.Fatalf("the copied recipe's compile is %v", compile)
+	}
+	var c cmdRecord
+	for _, r := range *calls {
+		if strings.HasPrefix(r.line, "go ") {
+			c = r
+		}
+	}
+	env := strings.Join(c.env, "\n")
+	for _, want := range []string{compile[1], "GOFLAGS=" + strings.TrimPrefix(compile[2], "GOFLAGS=") + " -mod=mod", "GOTOOLCHAIN=local", "GOPROXY=off",
+		"GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), "GOCACHE=" + filepath.Join(m.Cache, "gobuild")} {
+		if !strings.Contains(env, want) {
+			t.Errorf("the environment lacks %q:\n%s", want, env)
+		}
+	}
+	dir := filepath.Join(m.Work, "ws1", "tree", "modules", "demo")
+	out := filepath.Join(dir, ".mesh-build", "tools")
+	want := "go -C " + dir + " build -ldflags " + strings.Join(linker, " ") + " -X " + stamp + "=arch -o " + filepath.Join(out, "demo-bin") + " ./cmd/demo-tools"
+	if c.line != want {
+		t.Errorf("command = %q\nwant      %q", c.line, want)
+	}
+	if res.Artifact != "tools" || res.Binary != "demo-bin" || !strings.HasPrefix(res.Digest, "sha256:") || res.Size == 0 {
+		t.Errorf("result = %+v", res)
+	}
+}
+
+// The archive is the file the digest names, holding one executable of the name binaryName says; two builds of one tree
+// are one digest.
+func TestTheArchiveHoldsOneExecutableAndTwoBuildsAreOneDigest(t *testing.T) {
+	m, states, _ := buildModule(t, manifestWith(goBundle), "the program")
+	first, _ := runBuild(t, m, states, "tools")
+	second, _ := runBuild(t, m, states, "tools")
+	if first.Digest != second.Digest || first.Path != second.Path {
+		t.Errorf("two builds: %+v and %+v", first, second)
+	}
+	raw, err := os.ReadFile(first.Path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sum := sha256.Sum256(raw)
+	if first.Digest != "sha256:"+hex.EncodeToString(sum[:]) {
+		t.Errorf("the digest %s is not the archive's", first.Digest)
+	}
+	zr, err := gzip.NewReader(strings.NewReader(string(raw)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	tr := tar.NewReader(zr)
+	h, err := tr.Next()
+	if err != nil || h.Name != "demo-bin" || h.Mode != 0o755 {
+		t.Fatalf("first entry = %+v, %v", h, err)
+	}
+	if body, _ := io.ReadAll(tr); string(body) != "the program" {
+		t.Errorf("the executable holds %q", body)
+	}
+	if _, err := tr.Next(); err != io.EOF {
+		t.Errorf("the archive holds more than one file")
+	}
+}
+
+// 1487/AC3: an artifact of another language, or another kind, fails the step naming it.
+func TestAnArtifactOfAnotherLanguageFailsTheStepNamingIt(t *testing.T) {
+	for artifact, want := range map[string]string{
+		`{"name":"tools","kind":"bundle","language":"typescript","entrypoints":["index.js"],"loads":["index.js"]}`: `language "typescript"`,
+		`{"name":"tools","kind":"archive","from":"files"}`:                                                         "archive",
+	} {
+		m, states, calls := buildModule(t, manifestWith(artifact), "x")
+		_, st := runBuild(t, m, states, "tools")
+		if st.State != delivery.StepFailed || !strings.Contains(st.Reason, "tools") || !strings.Contains(st.Reason, want) || len(*calls) != 0 {
+			t.Errorf("%s: the step ended %s: %q after %d commands", artifact, st.State, st.Reason, len(*calls))
+		}
+	}
+	m, states, _ := buildModule(t, manifestWith(goBundle), "x")
+	if _, st := runBuild(t, m, states, "nope"); st.State != delivery.StepFailed || !strings.Contains(st.Reason, `"nope"`) || !strings.Contains(st.Reason, "tools") {
+		t.Errorf("an unknown artifact: %s %q", st.State, st.Reason)
+	}
+}
+
+// The executable is named as the build agent names it: the artifact's binary, else the package's directory, else the
+// artifact.
+func TestTheBinaryIsNamedAsTheBuildAgentNamesIt(t *testing.T) {
+	for artifact, want := range map[string]string{
+		goBundle: "demo-bin",
+		`{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/demo-tools","loads":["demo-tools"]}`: "demo-tools",
+		`{"name":"tools","kind":"bundle","language":"go","system":"arch","from":".","loads":["tools"]}`:                   "tools",
+	} {
+		m, states, _ := buildModule(t, manifestWith(artifact), "x")
+		if res, st := runBuild(t, m, states, "tools"); res.Binary != want {
+			t.Errorf("%s: binary %q, want %q (%s %s)", artifact, res.Binary, want, st.State, st.Reason)
+		}
+	}
+}
+
+func TestAModuleDirectoryOutsideTheWorkspaceIsRefused(t *testing.T) {
+	m, states, calls := buildModule(t, manifestWith(goBundle), "x")
+	for _, tool := range m.Tools(m.StepTools()) {
+		if tool.Name == "build_go_build" {
+			tool.Run(map[string]any{"workspace": "ws1", "module": "../../..", "artifact": "tools"})
+		}
+	}
+	if st := states.waitEnd(t); st.State != delivery.StepFailed || len(*calls) != 0 {
+		t.Errorf("the step ended %s after %d commands", st.State, len(*calls))
+	}
+}

modules/mesh-build-go/cmd/mesh-build-go/fetch.go

@@ -0,0 +1,174 @@
+package main
+
+// The fetch step (novox/hq ADR 0405 §5 §6, ADR 0466 §4 §5, ADR 0469 §2; issue 1485): a commit of a repository of
+// the mesh's forge, fetched over SSH as the module's own account into a workspace of its own, and the Go modules every
+// go.mod in it names, downloaded into the module's cache and verified against go.sum.
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/buildstep"
+)
+
+// ForgeAlias is the ssh name the forge-account provision sets up in the account's ssh files (ADR 0409 §1); the
+// remote of a fetch is ssh://<it>/<owner>/<repository>.git.
+const ForgeAlias = "mesh-forge"
+
+// WorkspaceAge is how long a workspace is kept: a module removes the workspaces older than a day, each time it
+// fetches and every round it checks its toolchain. A workspace lies in the module's work directory, never in its
+// home (ADR 0466 §1).
+const WorkspaceAge = 24 * time.Hour
+
+var (
+	repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
+	workspaceOK  = regexp.MustCompile(`^[a-z0-9]+$`)
+)
+
+// FetchResult is what a passed fetch step says: the workspace it made, the commit in it and the directories whose
+// Go modules it downloaded.
+type FetchResult struct {
+	Workspace string   `json:"workspace"`
+	Commit    string   `json:"commit"`
+	Modules   []string `json:"modules"`
+}
+
+// fetchTool is the fetch step tool.
+func (m *Module) fetchTool() StepTool {
+	return StepTool{
+		Name: "build_go_fetch",
+		Description: "Fetch a commit of a repository of the mesh's forge over SSH, as this module's own account, into a " +
+			"workspace of its own, and download and verify the Go modules every go.mod in it names. Answers a step id at " +
+			"once; the step ends on its key in the steps state with the workspace's id.",
+		Input: map[string]any{
+			"type": "object",
+			"properties": map[string]any{
+				"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
+				"commit":     map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
+				"ref":        map[string]any{"type": "string", "description": "refs/pull/<n>/head when only a pull request's head carries the commit"},
+			},
+			"required": []string{"repository", "commit"},
+		},
+		Long: true,
+		Run:  m.fetch,
+	}
+}
+
+func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
+	repository, _ := args["repository"].(string)
+	commit, _ := args["commit"].(string)
+	ref, _ := args["ref"].(string)
+	if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") {
+		return nil, fmt.Errorf("%q is not a repository, <owner>/<name>", repository)
+	}
+	m.CleanWorkspaces(m.now())
+	id := buildstep.NewID()
+	root := filepath.Join(m.Work, id)
+	tree := filepath.Join(root, "tree")
+	if err := os.MkdirAll(tree, 0o700); err != nil {
+		return nil, err
+	}
+	remote := "ssh://" + ForgeAlias + "/" + repository + ".git"
+	if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref); err != nil {
+		return nil, err
+	}
+	dirs, err := m.goModDirs(ctx, tree)
+	if err != nil {
+		return nil, err
+	}
+	for _, d := range dirs {
+		if _, err := os.Stat(filepath.Join(tree, d, "vendor", "modules.txt")); err == nil {
+			continue // a module that vendors resolves nothing
+		}
+		for _, verb := range []string{"download", "verify"} {
+			r := m.Run(ctx, m.goEnv(), "go", "-C", filepath.Join(tree, d), "mod", verb)
+			if r.Status != 0 || r.Err != "" {
+				return nil, fmt.Errorf("go mod %s in %s: %s", verb, d, firstLine(r.Err+"\n"+r.Stdout+"\n"+r.Stderr))
+			}
+		}
+	}
+	return FetchResult{Workspace: id, Commit: commit, Modules: dirs}, nil
+}
+
+// goModDirs are the directories of the tree that hold a go.mod, relative to it, in order.
+func (m *Module) goModDirs(ctx context.Context, tree string) ([]string, error) {
+	r := m.Run(ctx, nil, "git", "-C", tree, "ls-files", "--", "go.mod", "*/go.mod")
+	if r.Status != 0 || r.Err != "" {
+		return nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr))
+	}
+	dirs := []string{}
+	for _, l := range strings.Split(r.Stdout, "\n") {
+		if l = strings.TrimSpace(l); l != "" {
+			dirs = append(dirs, filepath.ToSlash(filepath.Dir(l)))
+		}
+	}
+	sort.Strings(dirs)
+	return dirs, nil
+}
+
+// goEnv is what a go command of a step is given beyond the base environment: the installed toolchain and no other,
+// modules resolved by their go.mod and go.sum, the module's own caches, the mesh's own modules fetched directly (by
+// git, through the account's rewrite of the forge's address to ssh), and nothing that asks a question. For a directory
+// that vendors, see goEnvFor.
+func (m *Module) goEnv() []string {
+	return []string{
+		"GOTOOLCHAIN=local",
+		"GOFLAGS=-mod=mod",
+		"GOMODCACHE=" + filepath.Join(m.Cache, "gomod"),
+		"GOCACHE=" + filepath.Join(m.Cache, "gobuild"),
+		"GOPRIVATE=git.novox.be",
+		"GIT_TERMINAL_PROMPT=0",
+		"GIT_SSH_COMMAND=ssh -o BatchMode=yes",
+	}
+}
+
+// goEnvFor is goEnv for a command run in dir: a directory that vendors (vendor/modules.txt) is read from its vendor
+// directory, as fetch downloaded nothing for it, and every other by its go.mod and go.sum.
+func (m *Module) goEnvFor(dir string) []string {
+	env := m.goEnv()
+	if _, err := os.Stat(filepath.Join(dir, "vendor", "modules.txt")); err == nil {
+		env = append(env, "GOFLAGS=-mod=vendor") // the last value of a name is the one used
+	}
+	return env
+}
+
+// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory.
+func (m *Module) CleanWorkspaces(now time.Time) {
+	entries, err := os.ReadDir(m.Work)
+	if err != nil {
+		return
+	}
+	for _, e := range entries {
+		info, err := e.Info()
+		if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge {
+			continue
+		}
+		if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil {
+			m.Say("the workspace %s could not be removed: %v", e.Name(), err)
+		}
+	}
+}
+
+// clock is the time now: Now when a test set it, else the wall clock.
+func (m *Module) clock() time.Time {
+	if m.Now != nil {
+		return m.Now()
+	}
+	return time.Now()
+}
+func (m *Module) now() time.Time { return m.clock() }
+
+func firstLine(s string) string {
+	for _, l := range strings.Split(s, "\n") {
+		if l = strings.TrimSpace(l); l != "" {
+			return l
+		}
+	}
+	return ""
+}

modules/mesh-build-go/cmd/mesh-build-go/integration.go

@@ -0,0 +1,77 @@
+package main
+
+// The integration level (novox/hq ADR 0445 §2 §3, ADR 0470; issue 1486): `go test -tags integration` over the
+// workspace, against a throwaway postgres and a throwaway bus started for this run alone, handed to the tests as
+// MESH_TEST_POSTGRES and MESH_TEST_NATS. Its setup (compiling, the store, the bus) and its running time are measured and
+// judged apart, 30 s and 60 s; both servers are stopped and the run's directory removed whatever the tests did.
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/buildstep"
+)
+
+// The integration level's bounds (ADR 0445 §3).
+const (
+	IntegrationSetup = 30 * time.Second
+	IntegrationRun   = 60 * time.Second
+)
+
+func (m *Module) integrationBounds() Bounds {
+	b := m.IntegrationBounds
+	if b.Setup <= 0 {
+		b.Setup = IntegrationSetup
+	}
+	if b.Run <= 0 {
+		b.Run = IntegrationRun
+	}
+	return b
+}
+
+func (m *Module) integration(ctx context.Context, workspace, tree string, packages []string) (res *TestResult, err error) {
+	major, err := m.checkPostgres(ctx)
+	if err != nil {
+		return nil, err // a *buildstep.Refusal: this level only
+	}
+	run := filepath.Join(m.Work, workspace, "run-"+buildstep.NewID())
+	if err := os.MkdirAll(run, 0o700); err != nil {
+		return nil, err
+	}
+	// The run's directory goes, and the servers stop, however the level ends: passed, failed, a panic.
+	defer os.RemoveAll(run)
+
+	began := m.clock()
+	dirs, err := m.goModDirs(ctx, tree)
+	if err != nil {
+		return nil, err
+	}
+	postgres, stopStore, err := m.startStore(ctx, run)
+	if err != nil {
+		return nil, fmt.Errorf("the integration level's store: %w", err)
+	}
+	defer stopStore()
+	nats, stopBus, err := m.startBus(ctx, filepath.Join(run, "nats"))
+	if err != nil {
+		return nil, fmt.Errorf("the integration level's bus: %w", err)
+	}
+	defer stopBus()
+	env := []string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats}
+	// Compiling counts as setup: the tests are built once, here, with the addresses a TestMain may read, and the run
+	// below finds them built. Whether they compile is for the run to say; its failure is not read here.
+	for _, d := range dirs {
+		m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), env...), "go", "-C", filepath.Join(tree, d), "test", "-tags", "integration", "-run", "^$", "-count=1", "./...")
+	}
+	setup := m.clock().Sub(began)
+
+	return m.runLevel(ctx, tree, levelOpts{
+		level: LevelIntegration, tags: []string{"integration"}, packages: packages, bounds: m.integrationBounds(),
+		env:   env,
+		setup: setup, wall: true,
+		store: "postgres " + strconv.Itoa(major), bus: "nats-server " + m.Recipe.NatsServer,
+	})
+}