mesh-build-go: run go test, go build and go list in bubblewrap, so a change's code reaches no key, login or network (issue 1309, S1-S6, ADR 0448)

Each run step goes through mesh-sdk's buildstep.Isolated: no network, none of the account,
the workspace read-write, the Go caches beneath a throwaway layer unless the build step is
told publish (a trunk build). The integration level runs its store, its bus and its tests in
one shut-off process (this program again, level <spec>), so they reach each other and
nothing else. build_go_test takes a -run pattern (a try's tests). The module declares the
bubblewrap package and refuses a step, unhealthy, while bwrap is missing or older than 0.10.
The proof test runs a probe test through the real unit level.
This commit is contained in:
2026-10-22 18:12:01 +02:00
parent 51900367a0
commit eed6be0f4a
20 changed files with 791 additions and 24 deletions
@@ -49,6 +49,7 @@ func (m *Module) buildTool() StepTool {
"workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"},
"module": map[string]any{"type": "string", "description": "the module's directory in the workspace, . for the repository's root"},
"artifact": map[string]any{"type": "string", "description": "the artifact's name in the module's manifest"},
"publish": map[string]any{"type": "boolean", "description": "true for a merged trunk commit alone: the build writes the shared Go caches; without it (a candidate) they are read beneath a throwaway layer"},
},
"required": []string{"workspace", "module", "artifact"},
},
@@ -76,6 +77,7 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
workspace, _ := args["workspace"].(string)
module, _ := args["module"].(string)
name, _ := args["artifact"].(string)
publish, _ := args["publish"].(bool)
if !workspaceOK.MatchString(workspace) {
return nil, fmt.Errorf("%q is not a workspace id", workspace)
}
@@ -133,7 +135,13 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
linker += " -X " + recipeSystemStamp + "=" + system
}
cmd := []string{"-C", dir, "build", "-ldflags", linker, "-o", filepath.Join(out, binary), "./" + strings.Trim(art.From, "./")}
if r := m.Run(ctx, m.buildEnv(dir), "go", cmd...); r.Status != 0 || r.Err != "" {
// Shut off (ADR 0448): the shared caches are written by a trunk build alone (publish), a candidate's go build writes
// beneath a throwaway layer.
run, err := m.stepRunner(workspace, publish)
if err != nil {
return nil, err
}
if r := run(ctx, m.buildEnv(dir), "go", cmd...); r.Status != 0 || r.Err != "" {
return nil, fmt.Errorf("go build of %s: %s", art.Name, causeLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout))
}
if err := m.insideModule(workspace, dir, out); err != nil {
@@ -41,7 +41,7 @@ func buildModule(t *testing.T, manifest, content string) (*Module, *keptStates,
if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(manifest), 0o644); err != nil {
t.Fatal(err)
}
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
m.Run = throughBubblewrap(t, func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
*calls = append(*calls, cmdRecord{env: env, line: name + " " + strings.Join(args, " ")})
for i, a := range args {
if a == "-o" {
@@ -51,7 +51,7 @@ func buildModule(t *testing.T, manifest, content string) (*Module, *keptStates,
}
}
return forgekey.Ran{}
}
}, &[]shutOff{}) // the compile runs shut off (1309)
return m, states, calls
}
@@ -126,7 +126,7 @@ func (m *Module) digest(ctx context.Context, args map[string]any) (any, error) {
}
res := DigestResult{Level: level, Packages: map[string]string{}}
for _, d := range dirs {
pkgs, err := m.listPackages(ctx, tree, d, tags)
pkgs, err := m.listPackages(ctx, workspace, tree, d, tags)
if err != nil {
return nil, err
}
@@ -139,14 +139,18 @@ func (m *Module) digest(ctx context.Context, args map[string]any) (any, error) {
}
// listPackages is `go list -test -deps -json` of the directory's packages, by import path.
func (m *Module) listPackages(ctx context.Context, tree, dir string, tags []string) (map[string]*goPackage, error) {
func (m *Module) listPackages(ctx context.Context, workspace, tree, dir string, tags []string) (map[string]*goPackage, error) {
abs := filepath.Join(tree, dir)
args := []string{"-C", abs, "list", "-test", "-deps", "-json"}
if len(tags) > 0 {
args = append(args, "-tags", strings.Join(tags, ","))
}
args = append(args, "./...")
r := m.Run(ctx, m.goEnvFor(abs), "go", args...)
run, err := m.stepRunner(workspace, false)
if err != nil {
return nil, err
}
r := run(ctx, m.goEnvFor(abs), "go", args...)
if r.Status != 0 || r.Err != "" {
return nil, fmt.Errorf("go list in %s: %s", dir, causeLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout))
}
@@ -30,7 +30,7 @@ func digestFixture(t *testing.T) (*Module, string) {
m := moduleWith(newStates(), "go1.27.1")
m.Home, m.Work, m.Cache = t.TempDir(), work, t.TempDir()
real := StepRunner(m.Home)
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
m.Run = throughBubblewrap(t, func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
switch name {
case "git":
return forgekey.Ran{Stdout: "go.mod\n"}
@@ -38,7 +38,7 @@ func digestFixture(t *testing.T) (*Module, string) {
return forgekey.Ran{Stdout: "postgres (PostgreSQL) " + strconv.Itoa(m.Recipe.PostgresMajor) + ".1\n"}
}
return real(ctx, env, name, args...)
}
}, &[]shutOff{}) // go list runs shut off (1309); here the real go, unshut, reads the fixture
return m, tree
}
@@ -33,11 +33,22 @@ func (m *Module) integrationBounds() Bounds {
return b
}
func (m *Module) integration(ctx context.Context, workspace, tree string, packages []string) (res *TestResult, err error) {
func (m *Module) integration(ctx context.Context, workspace, tree string, packages []string, pattern string) (res *TestResult, err error) {
major, err := m.checkPostgres(ctx)
if err != nil {
return nil, err // a *buildstep.Refusal: this level only
}
if !m.Inside {
// The store, the bus and the tests run together in one shut-off process (level.go), so the tests reach the
// level's own store and bus and nothing else.
return m.integrationShutOff(ctx, workspace, packages, pattern)
}
return m.integrationHere(ctx, workspace, tree, packages, pattern, major)
}
// integrationHere is the level itself, in the shut-off process: the store and the bus started for the run, the tests
// compiled and run against them, both stopped and the run's directory removed however it ends.
func (m *Module) integrationHere(ctx context.Context, workspace, tree string, packages []string, pattern string, major int) (res *TestResult, err error) {
run := filepath.Join(m.Work, workspace, "run-"+buildstep.NewID())
if err := os.MkdirAll(run, 0o700); err != nil {
return nil, err
@@ -69,6 +80,7 @@ func (m *Module) integration(ctx context.Context, workspace, tree string, packag
setup := m.clock().Sub(began)
return m.runLevel(ctx, tree, levelOpts{
workspace: workspace, run: pattern,
level: LevelIntegration, tags: []string{"integration"}, packages: packages, bounds: m.integrationBounds(),
env: env,
setup: setup, wall: true,
@@ -107,6 +107,9 @@ func integModule(t *testing.T, f *integFake) (*Module, *keptStates) {
m := moduleWith(states, "go1.27.1")
m.Home, m.Work, m.Cache = t.TempDir(), t.TempDir(), t.TempDir()
m.Run, m.Now, m.StartBus = f.run, f.now, f.startBus
// These tests judge the level as it runs in its shut-off process (level.go, 1309-S5): the store, the bus and the
// tests side by side. That the step starts that one process shut off is isolate_test.go's.
m.Inside = true
if err := os.MkdirAll(filepath.Join(m.Work, "ws1", "tree"), 0o755); err != nil {
t.Fatal(err)
}
@@ -277,7 +280,7 @@ func TestAFailedOrPanickedRunStopsTheServersAndRemovesTheRunDirectory(t *testing
m, _ = integModule(t, g)
func() {
defer func() { recover() }()
m.integration(context.Background(), "ws1", filepath.Join(m.Work, "ws1", "tree"), nil)
m.integration(context.Background(), "ws1", filepath.Join(m.Work, "ws1", "tree"), nil, "")
}()
checkCleaned(g, m)
}
@@ -0,0 +1,123 @@
package main
// The run steps shut off (novox/hq ADR 0448; issue 1309, lead decision L2 of R4e): every command that runs a change's
// code (go test, go build, go list) runs through mesh-sdk's buildstep.Isolated, in bubblewrap: no network, none of the
// account (its home, its ssh key, its bus login, the runtime directory), the workspace read-write, the shared Go caches
// read-only under a throwaway writable layer, or read-write for a trunk build alone (publish). The fetch step (git, go
// mod download) holds the key and the network and runs none of the change's code, so it runs as before.
import (
"context"
"fmt"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
// BubblewrapLeast is the oldest bubblewrap a run step is shut off with: --overlay-src and --tmp-overlay, which keep the
// shared caches unchanged by a candidate, came in 0.10.
const BubblewrapLeast = "0.10"
// caches are the shared warm caches of the module: Go's module cache and its build cache.
func (m *Module) caches() []string {
return []string{filepath.Join(m.Cache, "gomod"), filepath.Join(m.Cache, "gobuild")}
}
// isolation is how a run step of the workspace is shut off: the workspace (its tree, its siblings, its run directories,
// its packed artifacts) read-write, the caches beneath a throwaway layer unless write, and an environment of a PATH and
// a locale alone (the step's HOME is its own, inside its /tmp).
func (m *Module) isolation(workspace string, write bool) buildstep.Isolation {
path := os.Getenv("PATH")
if path == "" {
path = defaultPath
}
return buildstep.Isolation{
Workspace: filepath.Join(m.Work, workspace),
Caches: m.caches(),
WriteCaches: write,
Env: []string{"PATH=" + path, "LC_ALL=C"},
}
}
// stepRunner is the runner of a run step of the workspace: buildstep.Isolated over m.Run, so every command it runs is
// shut off; write is true for a trunk build alone (the shared caches are written by trunk builds only). Inside is the
// integration level's own process, already running shut off (level.go): it runs its commands as they are.
func (m *Module) stepRunner(workspace string, write bool) (forgekey.Runner, error) {
if m.Inside {
return m.Run, nil
}
for _, c := range m.caches() {
// bubblewrap binds only a directory that is there; the fetch makes them, a step on a fresh node may come first.
if err := os.MkdirAll(c, 0o700); err != nil {
return nil, fmt.Errorf("the shared cache %s: %w", c, err)
}
}
return buildstep.Isolated(m.Run, m.isolation(workspace, write)), nil
}
// ProbeBubblewrap reads the installed bubblewrap's version (`bwrap --version`: "bubblewrap 0.13.0").
func ProbeBubblewrap(ctx context.Context) (string, error) {
ctx, cancel := context.WithTimeout(ctx, ProbeTimeout)
defer cancel()
r := forgekey.ExecRunnerClean(0)(ctx, []string{"PATH=" + defaultPath}, buildstep.Bubblewrap, "--version")
if r.Status != 0 || r.Err != "" {
return "", fmt.Errorf("`%s --version` failed: %s", buildstep.Bubblewrap, firstLine(r.Err+"\n"+r.Stderr))
}
return parseBubblewrapVersion(r.Stdout)
}
var bubblewrapVersion = regexp.MustCompile(`^bubblewrap (\d+)\.(\d+)(\.\d+)?\s*$`)
func parseBubblewrapVersion(out string) (string, error) {
f := bubblewrapVersion.FindStringSubmatch(strings.TrimSpace(out))
if f == nil {
return "", fmt.Errorf("`%s --version` answered %q", buildstep.Bubblewrap, strings.TrimSpace(out))
}
return f[1] + "." + f[2] + f[3], nil
}
// bubblewrapProblem says why the installed bubblewrap cannot shut a run step off, or "" when it can: it is missing, or
// older than BubblewrapLeast.
func bubblewrapProblem(version string, err error) string {
if err != nil {
return "bubblewrap cannot be run, and every build and test step runs in it (ADR 0448): " + err.Error()
}
if !atLeast(version, BubblewrapLeast) {
return fmt.Sprintf("bubblewrap %s installed, and a build or test step needs %s or later (ADR 0448)", version, BubblewrapLeast)
}
return ""
}
// atLeast says whether the dotted version v is least or later.
func atLeast(v, least string) bool {
a, b := strings.Split(v, "."), strings.Split(least, ".")
for i := 0; i < len(b); i++ {
var x, y int
if i < len(a) {
x, _ = strconv.Atoi(a[i])
}
y, _ = strconv.Atoi(b[i])
if x != y {
return x > y
}
}
return true
}
// checkBubblewrap refuses a step when bubblewrap cannot shut it off; nil Bubblewrap probes nothing (a module that only
// guards).
func (m *Module) checkBubblewrap(ctx context.Context) error {
if m.Bubblewrap == nil {
return nil
}
v, err := m.Bubblewrap(ctx)
if why := bubblewrapProblem(v, err); why != "" {
return &buildstep.Refusal{Text: why, Want: BubblewrapLeast + " or later", Have: v}
}
return nil
}
@@ -0,0 +1,295 @@
package main
import (
"context"
"encoding/json"
"errors"
"path/filepath"
"slices"
"strings"
"sync"
"testing"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
// shutOff is one command a step ran in bubblewrap: the bubblewrap words before "--", what it bound read-write, read-only
// and beneath a throwaway layer, and the command with its environment.
type shutOff struct {
flags []string
binds, roBinds, overlays []string
env []string
command string
chdir string
}
// bwrapArity is how many words each bubblewrap option the SDK writes takes.
var bwrapArity = map[string]int{"--unshare-all": 0, "--die-with-parent": 0, "--new-session": 0, "--clearenv": 0,
"--proc": 1, "--dev": 1, "--tmpfs": 1, "--dir": 1, "--chdir": 1, "--overlay-src": 1, "--tmp-overlay": 1,
"--ro-bind": 2, "--ro-bind-try": 2, "--bind": 2, "--symlink": 2, "--setenv": 2}
// throughBubblewrap is the fake runner of a step's tests: a bubblewrap command is read back (recorded in seen) and its
// command handed to run with the environment bubblewrap was told to set, the way it runs in the step; any other command
// (the fetch's git, postgres --version) goes to run as it is.
func throughBubblewrap(t *testing.T, run forgekey.Runner, seen *[]shutOff) forgekey.Runner {
t.Helper()
var mu sync.Mutex
return func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
if name != buildstep.Bubblewrap {
return run(ctx, env, name, args...)
}
var s shutOff
i := 0
for ; i < len(args) && args[i] != "--"; i++ {
n, ok := bwrapArity[args[i]]
if !ok || i+n >= len(args) {
t.Errorf("bubblewrap was given %q, which the test does not know: %v", args[i], args)
return forgekey.Ran{Status: 1, Err: "unknown bubblewrap option"}
}
s.flags = append(s.flags, args[i])
switch args[i] {
case "--bind":
s.binds = append(s.binds, args[i+1])
case "--ro-bind", "--ro-bind-try":
s.roBinds = append(s.roBinds, args[i+1])
case "--overlay-src":
s.overlays = append(s.overlays, args[i+1])
case "--setenv":
s.env = append(s.env, args[i+1]+"="+args[i+2])
case "--chdir":
s.chdir = args[i+1]
}
i += n
}
if i+1 >= len(args) {
t.Errorf("bubblewrap was given no command: %v", args)
return forgekey.Ran{Status: 1, Err: "no command"}
}
s.command = args[i+1] + " " + strings.Join(args[i+2:], " ")
mu.Lock()
*seen = append(*seen, s)
mu.Unlock()
return run(ctx, s.env, args[i+1], args[i+2:]...)
}
}
// checkShutOff fails the test unless s ran with no network and none of the account: every namespace unshared, the
// environment cleared, nothing of the home or the state bound, the caches beneath a throwaway layer unless write.
func checkShutOff(t *testing.T, m *Module, s shutOff, write bool) {
t.Helper()
for _, f := range []string{"--unshare-all", "--clearenv", "--die-with-parent"} {
if !slices.Contains(s.flags, f) {
t.Errorf("%s ran without %s: %v", s.command, f, s.flags)
}
}
for _, b := range append(append([]string{}, s.binds...), s.roBinds...) {
if m.Home != "" && (b == m.Home || strings.HasPrefix(b, m.Home+"/")) {
t.Errorf("%s was given the account's home: %s", s.command, b)
}
if strings.HasPrefix(b, "/run") {
t.Errorf("%s was given the runtime directory: %s", s.command, b)
}
}
for _, e := range s.env {
if strings.HasPrefix(e, "HOME="+m.Home) && m.Home != "" {
t.Errorf("%s was told the account's home: %s", s.command, e)
}
}
caches := m.caches()
for _, c := range caches {
bound, layered := slices.Contains(s.binds, c), slices.Contains(s.overlays, c)
switch {
case write && (!bound || layered):
t.Errorf("a trunk build's %s does not write the shared cache %s (bound %v, layered %v)", s.command, c, bound, layered)
case !write && (bound || !layered):
t.Errorf("%s may write the shared cache %s (bound %v, layered %v): only a trunk build does", s.command, c, bound, layered)
}
}
if want := filepath.Join(m.Work, "ws1"); !slices.Contains(s.binds, want) {
t.Errorf("%s was not given its workspace %s read-write: %v", s.command, want, s.binds)
}
}
// 1309-S2: every go test of the unit level runs shut off in bubblewrap: no network, none of the account, the caches
// beneath a throwaway layer.
func TestTheUnitLevelRunsEveryGoTestShutOff(t *testing.T) {
m, states, ws, calls := workspaceFor(t, "failing")
var seen []shutOff
m.Run = throughBubblewrap(t, m.Run, &seen)
if _, st := runUnit(t, m, states, ws); st.State != delivery.StepPassed {
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
}
ran := 0
for _, c := range *calls {
if strings.HasPrefix(c.line, "go ") {
ran++
}
}
if ran == 0 || len(seen) != ran {
t.Fatalf("%d go commands ran, %d of them shut off: %v", ran, len(seen), *calls)
}
for _, s := range seen {
checkShutOff(t, m, s, false)
}
}
// 1309-S2: a try's tests: build_go_test's run pattern reaches go test as one -run word; a pattern that is a flag is refused.
func TestTheRunPatternReachesGoTestAndAFlagIsRefused(t *testing.T) {
m, states, ws, calls := workspaceFor(t, "failing")
var seen []shutOff
m.Run = throughBubblewrap(t, m.Run, &seen)
run := stepTool(t, m, "build_go_test")
if _, err := run(map[string]any{"workspace": ws, "level": "unit", "run": "TestA|TestB"}); err != nil {
t.Fatal(err)
}
states.waitEnd(t)
found := false
for _, c := range *calls {
found = found || strings.Contains(c.line, " -run TestA|TestB ")
}
if !found {
t.Errorf("no go test was given -run TestA|TestB: %v", *calls)
}
for _, bad := range []string{"-exec=/bin/sh", "a b", "(unclosed"} {
if runPatternOK(bad) {
t.Errorf("the run pattern %q was taken", bad)
}
}
}
// 1309-S3 and S4: a candidate's go build runs shut off with the caches beneath a throwaway layer; a trunk build
// (publish) writes them.
func TestABuildRunsShutOffAndOnlyATrunkBuildWritesTheCaches(t *testing.T) {
for _, publish := range []bool{false, true} {
m, states, _ := buildModule(t, manifestWith(goBundle), "binary")
var seen []shutOff
m.Run = throughBubblewrap(t, m.Run, &seen)
run := stepTool(t, m, "build_go_build")
args := map[string]any{"workspace": "ws1", "module": "modules/demo", "artifact": "tools"}
if publish {
args["publish"] = true
}
if _, err := run(args); err != nil {
t.Fatal(err)
}
if st := states.waitEnd(t); st.State != delivery.StepPassed {
t.Fatalf("publish %v: the step ended %s: %s", publish, st.State, st.Reason)
}
if len(seen) != 1 || !strings.HasPrefix(seen[0].command, "go ") || !strings.Contains(seen[0].command, " build ") {
t.Fatalf("publish %v: shut off %v, want the one go build", publish, seen)
}
checkShutOff(t, m, seen[0], publish)
}
}
// 1309-S3: the digest's go list runs shut off.
func TestTheDigestListsPackagesShutOff(t *testing.T) {
m, _ := digestFixture(t)
var seen []shutOff
m.Run = throughBubblewrap(t, m.Run, &seen)
digests(t, m, LevelUnit)
if len(seen) == 0 {
t.Fatal("the digest ran no command shut off")
}
for _, s := range seen {
if !strings.HasPrefix(s.command, "go ") || !strings.Contains(s.command, " list ") {
t.Errorf("the digest shut off %q, want go list", s.command)
}
checkShutOff(t, m, s, false)
}
}
// 1309-S5: the integration level runs its store, its bus and its tests in ONE shut-off process (this program again,
// `level <spec>`), so they reach each other and nothing else; its answer is the level's result. Nothing of the level
// runs outside but reading postgres's version.
func TestTheIntegrationLevelRunsInOneShutOffProcess(t *testing.T) {
m, states, ws, _ := workspaceFor(t, "failing")
m.Executable = "/opt/bundle/mesh-build-go"
want := TestResult{Level: LevelIntegration, Passed: true, Tests: 3, Packages: []PackageResult{}, Store: "postgres 18"}
var outside []string
var seen []shutOff
m.Run = throughBubblewrap(t, func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
switch {
case name == "postgres" && len(args) == 1 && args[0] == "--version":
return forgekey.Ran{Stdout: "postgres (PostgreSQL) 18.6\n"}
case name == m.Executable && len(args) == 2 && args[0] == levelArg:
var spec levelSpec
if err := json.Unmarshal([]byte(args[1]), &spec); err != nil || spec.Workspace != ws || spec.Work != m.Work {
t.Errorf("the level's spec %s: %v", args[1], err)
}
raw, _ := json.Marshal(levelAnswer{Result: &want})
return forgekey.Ran{Stdout: string(raw) + "\n"}
}
outside = append(outside, name+" "+strings.Join(args, " "))
return forgekey.Ran{Status: 1, Err: "not expected here"}
}, &seen)
run := stepTool(t, m, "build_go_test")
if _, err := run(map[string]any{"workspace": ws, "level": LevelIntegration}); err != nil {
t.Fatal(err)
}
st := states.waitEnd(t)
if st.State != delivery.StepPassed {
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
}
var got TestResult
if err := json.Unmarshal(st.Result, &got); err != nil || got.Tests != 3 || !got.Passed || got.Store != "postgres 18" {
t.Errorf("the result %s (%v), want the shut-off process's", st.Result, err)
}
if len(outside) > 0 {
t.Errorf("ran outside the shut-off process: %v", outside)
}
if len(seen) != 1 {
t.Fatalf("shut off %d processes, want one: %v", len(seen), seen)
}
checkShutOff(t, m, seen[0], false)
if !slices.Contains(seen[0].roBinds, m.Executable) || slices.Contains(seen[0].roBinds, filepath.Dir(m.Executable)) {
t.Errorf("the program is bound %v; want the program alone, read-only", seen[0].roBinds)
}
}
// 1309-S1: a step is refused, and the module is unhealthy, while bubblewrap is missing or older than 0.10, naming it.
func TestAnOldOrMissingBubblewrapRefusesTheStepsAndIsUnhealthy(t *testing.T) {
for _, c := range []struct {
version string
err error
ok bool
}{{"0.13.0", nil, true}, {"0.10.0", nil, true}, {"0.9.0", nil, false}, {"", errors.New("bwrap: not found"), false}} {
m := moduleWith(newStates(), "go1.27.1")
m.Bubblewrap = func(context.Context) (string, error) { return c.version, c.err }
err := m.Check(context.Background())
h := m.Health(context.Background())
raw, _ := json.Marshal(h)
if c.ok {
if err != nil || strings.Contains(string(raw), "unhealthy") {
t.Errorf("bubblewrap %q: refused %v, health %s", c.version, err, raw)
}
continue
}
var ref *buildstep.Refusal
if !errors.As(err, &ref) || !strings.Contains(err.Error(), "bubblewrap") {
t.Errorf("bubblewrap %q %v: Check answered %v, want a refusal naming bubblewrap", c.version, c.err, err)
}
if !strings.Contains(string(raw), "unhealthy") || !strings.Contains(string(raw), "bubblewrap") {
t.Errorf("bubblewrap %q %v: health %s, want unhealthy naming bubblewrap", c.version, c.err, raw)
}
}
for out, want := range map[string]string{"bubblewrap 0.13.0\n": "0.13.0", "bubblewrap 0.10": "0.10"} {
if got, err := parseBubblewrapVersion(out); err != nil || got != want {
t.Errorf("%q read as %q, %v", out, got, err)
}
}
}
// stepTool is the Run of the named tool.
func stepTool(t *testing.T, m *Module, name string) func(map[string]any) (any, error) {
t.Helper()
for _, tool := range m.Tools(m.StepTools()) {
if tool.Name == name {
return tool.Run
}
}
t.Fatalf("no tool %s", name)
return nil
}
@@ -0,0 +1,84 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"net"
"os"
"path/filepath"
"testing"
)
// 1309-S6 (ADR 0448's check): a test of the change, run by the unit level, reads no key and no login of the account,
// resolves no name, reaches no socket of the machine (a TCP port on its loopback, a unix socket) and leaves the shared
// caches unchanged. The probe is testdata/repos/probe.
func TestARunStepReachesNoKeyLoginNameOrSocketAndLeavesTheCachesUnchanged(t *testing.T) {
needTools(t, "the shut-off unit level", "go", "git", "bwrap")
tree := repository(t, "probe")
work, err := os.MkdirTemp("", "w")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(work) })
if err := os.MkdirAll(filepath.Join(work, "ws1"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.Rename(tree, filepath.Join(work, "ws1", "tree")); err != nil {
t.Fatal(err)
}
home := t.TempDir()
key, login := filepath.Join(home, ".ssh", "mesh_ed25519"), filepath.Join(home, "broker")
for _, f := range []string{key, login} {
if err := os.MkdirAll(filepath.Dir(f), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(f, []byte("secret"), 0o600); err != nil {
t.Fatal(err)
}
}
tcp, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer tcp.Close()
sockDir := t.TempDir()
unix, err := net.Listen("unix", filepath.Join(sockDir, "s"))
if err != nil {
t.Fatal(err)
}
defer unix.Close()
m := &Module{Recipe: Current(), Home: home, Work: work, Cache: t.TempDir()}
m.Run = StepRunner(home)
for _, c := range m.caches() {
if err := os.MkdirAll(c, 0o700); err != nil {
t.Fatal(err)
}
}
res, err := m.runLevel(context.Background(), filepath.Join(work, "ws1", "tree"), levelOpts{workspace: "ws1", level: LevelUnit,
bounds: Bounds{Run: UnitBound * 100}, env: []string{"PROBE_KEY=" + key, "PROBE_LOGIN=" + login,
"PROBE_TCP=" + tcp.Addr().String(), "PROBE_SOCKET=" + filepath.Join(sockDir, "s"), "PROBE_CACHE=" + m.caches()[1]}})
if err != nil {
t.Fatal(err)
}
if !res.Passed {
t.Fatalf("the probe did not run: %+v", res)
}
raw, err := os.ReadFile(filepath.Join(work, "ws1", "tree", "seen.json"))
if err != nil {
t.Fatalf("the probe wrote nothing: %v", err)
}
var seen map[string]bool
if err := json.Unmarshal(raw, &seen); err != nil {
t.Fatal(err)
}
for what, reached := range seen {
if reached && what != "wroteCache" {
t.Errorf("the run step reached %s: %s", what, raw)
}
}
if _, err := os.Stat(filepath.Join(m.caches()[1], "written-by-a-candidate")); err == nil {
t.Errorf("a candidate's step changed the shared cache")
}
}
@@ -0,0 +1,121 @@
package main
// The integration level shut off (novox/hq ADR 0448, ADR 0470; issue 1309 S5): its store, its bus and its tests run in
// one bubblewrap of their own, so the tests reach the level's postgres (a socket in the run's directory) and its bus (a
// loopback port of the step's own network) and nothing of the machine. The process in it is this binary started again
// as `mesh-build-go level <spec>`: it runs the level as integrationHere does and prints its answer as one line of JSON.
// The tools bundle never runs a level in its own process, and a level never serves tools.
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/buildstep"
)
// levelArg is the first argument that makes this binary an integration level's process.
const levelArg = "level"
// levelSpec is what the shut-off process is asked: the workspace, the packages and the -run pattern, the level's bounds.
type levelSpec struct {
Work string `json:"work"`
Cache string `json:"cache"`
Workspace string `json:"workspace"`
Packages []string `json:"packages,omitempty"`
Run string `json:"run,omitempty"`
Setup time.Duration `json:"setup"`
Bound time.Duration `json:"bound"`
}
// levelAnswer is the one line the process prints: the level's result, or why it could not run.
type levelAnswer struct {
Result *TestResult `json:"result,omitempty"`
Error string `json:"error,omitempty"`
}
// executable is this program's path, bound read-only into the step: Executable when a test set it.
func (m *Module) executable() (string, error) {
if m.Executable != "" {
return m.Executable, nil
}
exe, err := os.Executable()
if err != nil {
return "", err
}
return filepath.EvalSymlinks(exe)
}
// integrationShutOff runs the integration level in one shut-off process and reads its answer.
func (m *Module) integrationShutOff(ctx context.Context, workspace string, packages []string, pattern string) (*TestResult, error) {
exe, err := m.executable()
if err != nil {
return nil, fmt.Errorf("the integration level's program: %w", err)
}
b := m.integrationBounds()
spec, err := json.Marshal(levelSpec{Work: m.Work, Cache: m.Cache, Workspace: workspace, Packages: packages, Run: pattern, Setup: b.Setup, Bound: b.Run})
if err != nil {
return nil, err
}
for _, c := range m.caches() {
if err := os.MkdirAll(c, 0o700); err != nil {
return nil, fmt.Errorf("the shared cache %s: %w", c, err)
}
}
iso := m.isolation(workspace, false)
iso.ReadOnly = append(iso.ReadOnly, exe) // the program alone, not the directory it lies in
r := buildstep.Isolated(m.Run, iso)(ctx, nil, exe, levelArg, string(spec))
var answer levelAnswer
line := lastLine(r.Stdout)
if err := json.Unmarshal([]byte(line), &answer); err != nil || (answer.Result == nil && answer.Error == "") {
return nil, fmt.Errorf("the integration level's shut-off process answered nothing it says (status %d %s): %s", r.Status, r.Err, firstLine(r.Stderr+"\n"+r.Stdout))
}
if answer.Error != "" {
return nil, fmt.Errorf("%s", answer.Error)
}
return answer.Result, nil
}
func lastLine(s string) string {
lines := strings.Split(strings.TrimSpace(s), "\n")
return strings.TrimSpace(lines[len(lines)-1])
}
// runLevelProcess is the shut-off process: args is the spec. It prints its answer and returns the exit status, 0 when it
// printed one.
func runLevelProcess(args []string) int {
say := func(a levelAnswer) int {
raw, _ := json.Marshal(a)
fmt.Println(string(raw))
return 0
}
if len(args) != 1 {
fmt.Fprintln(os.Stderr, "usage: mesh-build-go level <spec>")
return 2
}
var spec levelSpec
if err := json.Unmarshal([]byte(args[0]), &spec); err != nil {
return say(levelAnswer{Error: "the level's spec: " + err.Error()})
}
if !workspaceOK.MatchString(spec.Workspace) {
return say(levelAnswer{Error: fmt.Sprintf("%q is not a workspace id", spec.Workspace)})
}
m := &Module{Inside: true, Recipe: Current(), Work: spec.Work, Cache: spec.Cache, Home: os.Getenv("HOME"),
IntegrationBounds: Bounds{Setup: spec.Setup, Run: spec.Bound}, Say: func(string, ...any) {}}
m.Run = StepRunner(m.Home)
ctx := context.Background()
major, err := m.checkPostgres(ctx)
if err != nil {
return say(levelAnswer{Error: err.Error()})
}
tree := filepath.Join(spec.Work, spec.Workspace, "tree")
res, err := m.integrationHere(ctx, spec.Workspace, tree, spec.Packages, spec.Run, major)
if err != nil {
return say(levelAnswer{Error: err.Error()})
}
return say(levelAnswer{Result: res})
}
@@ -29,6 +29,9 @@ func main() {
if len(os.Args) > 1 && os.Args[1] == busArg {
os.Exit(runBus(os.Args[2:]))
}
if len(os.Args) > 1 && os.Args[1] == levelArg {
os.Exit(runLevelProcess(os.Args[2:]))
}
ctx := context.Background()
m := &Module{
Node: strings.TrimSpace(os.Getenv("MESH_NODE")),
@@ -48,6 +51,7 @@ func main() {
}
m.Run = StepRunner(m.Home)
m.Probe = ProbeGo(m.Home, m.Cache)
m.Bubblewrap = ProbeBubblewrap
// Outside a launch by the runtime — a build, a check — there is no node and no state to write.
if m.Node != "" {
go m.Start(ctx)
@@ -64,7 +64,7 @@ func TestManifestIsValidAndDeclaresTheAccountTheCapabilityAndThePackages(t *test
}
}
slices.Sort(packages)
if want := []string{"git", "openssh", "postgresql"}; !slices.Equal(packages, want) {
if want := []string{"bubblewrap", "git", "openssh", "postgresql"}; !slices.Equal(packages, want) {
t.Errorf("packages = %v, want %v", packages, want)
}
}
@@ -148,3 +148,15 @@ func TestGiteaReadsTheBuildModulesKeysAndTheCatalogueAcceptsThemTogether(t *test
t.Errorf("a read names a state its owner does not keep: %v", problems)
}
}
// 1309-S1: every run step runs in bubblewrap, so the module declares its package (the lead's decision: each build module
// that runs a step through it declares it), beside the user-namespaces capability bubblewrap needs.
func TestTheModuleDeclaresBubblewrapAndTheCapabilityItNeeds(t *testing.T) {
m := readManifest(t)
if r := resource(t, m, "bubblewrap"); r["type"] != "package" || r["package"] != "bubblewrap" {
t.Errorf("the bubblewrap resource is %v, want the package bubblewrap", r)
}
if !slices.Contains(m.Capabilities, "user-namespaces") {
t.Errorf("capabilities %v lack user-namespaces", m.Capabilities)
}
}
@@ -50,6 +50,14 @@ type Module struct {
IntegrationBounds Bounds
// Now is the clock the levels' times are read from; time.Now when nil.
Now func() time.Time
// Bubblewrap reads the installed bubblewrap's version (ProbeBubblewrap): a step is refused, and the module is
// unhealthy, while it is missing or older than BubblewrapLeast. Nil probes nothing.
Bubblewrap func(ctx context.Context) (string, error)
// Inside is the integration level's own process, started shut off in bubblewrap by a test step (level.go): its
// commands run as they are, never shut off a second time.
Inside bool
// Executable is this program's path, which the integration level starts again shut off; os.Executable when empty.
Executable string
// StartBus starts the throwaway bus of a run in dir and answers its address and what stops it (the module's own
// program, StartBus in bus.go, when nil).
StartBus func(ctx context.Context, dir string) (url string, stop func(), err error)
@@ -69,7 +77,10 @@ func (m *Module) guard(installed string, probeErr error) buildstep.Guard {
// CheckRound.
func (m *Module) Check(ctx context.Context) error {
installed, err := m.Probe(ctx)
return m.guard(installed, err).Check(ctx)
if err := m.guard(installed, err).Check(ctx); err != nil {
return err
}
return m.checkBubblewrap(ctx)
}
// Start checks once, as the module starts, and then every CheckRound until ctx ends.
@@ -95,6 +106,9 @@ func (m *Module) Health(ctx context.Context) health.Answer {
if err := m.guard(installed, perr).Check(ctx); err != nil {
return health.Unhealthy(err.Error())
}
if err := m.checkBubblewrap(ctx); err != nil {
return health.Unhealthy(err.Error())
}
return health.Healthy(fmt.Sprintf("%s installed, the version the module builds with (recipe %s)", installed, m.Recipe.Version()))
}
@@ -173,7 +173,7 @@ func TestSiblingsArePlacedBesideAndTheUnitLevelIsGivenTheirDirectory(t *testing.
}
calls := &[]cmdRecord{}
m.Run = capturedRunner(t, "failing", calls)
m.Run = throughBubblewrap(t, capturedRunner(t, "failing", calls), &[]shutOff{})
if err := os.MkdirAll(filepath.Join(m.Work, res.Workspace, "tree"), 0o755); err != nil {
t.Fatal(err)
}
@@ -33,6 +33,7 @@ func (m *Module) testTool() StepTool {
"level": map[string]any{"type": "string", "enum": []string{LevelUnit, LevelIntegration}, "description": "the level"},
"packages": map[string]any{"type": "array", "items": map[string]any{"type": "string"},
"description": "package patterns of each module, ./... when none"},
"run": map[string]any{"type": "string", "description": "optional go test -run pattern: only the tests it matches run (a try's tests)"},
},
"required": []string{"workspace", "level"},
},
@@ -56,6 +57,10 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
if err != nil {
return nil, err
}
pattern, _ := args["run"].(string)
if !runPatternOK(pattern) {
return nil, fmt.Errorf("%q is not a go test -run pattern", pattern)
}
if !workspaceOK.MatchString(workspace) {
return nil, fmt.Errorf("%q is not a workspace id", workspace)
}
@@ -65,9 +70,9 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
}
switch level {
case LevelUnit:
return m.runLevel(ctx, tree, levelOpts{level: LevelUnit, packages: packages, bounds: Bounds{Run: m.unitBound()}, env: m.besideEnv(workspace)})
return m.runLevel(ctx, tree, levelOpts{workspace: workspace, level: LevelUnit, packages: packages, run: pattern, bounds: Bounds{Run: m.unitBound()}, env: m.besideEnv(workspace)})
case LevelIntegration:
return m.integration(ctx, workspace, tree, packages)
return m.integration(ctx, workspace, tree, packages, pattern)
}
return nil, fmt.Errorf("the level %q is neither unit nor integration", level)
}
@@ -103,6 +108,10 @@ func packagesArg(v any) ([]string, error) {
// levelOpts are what a level asks of runLevel.
type levelOpts struct {
// workspace is the workspace's id: its commands run shut off in it (stepRunner).
workspace string
// run is go test's -run pattern, "" for every test.
run string
level string
tags []string
packages []string
@@ -129,6 +138,10 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR
if err != nil {
return nil, err
}
run, err := m.stepRunner(o.workspace, false)
if err != nil {
return nil, err
}
began := m.clock()
for _, d := range dirs {
packages := byDir[d]
@@ -139,8 +152,11 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR
if len(o.tags) > 0 {
args = append(args, "-tags", strings.Join(o.tags, ","))
}
if o.run != "" {
args = append(args, "-run", o.run)
}
args = append(args, packages...)
r := m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), o.env...), "go", args...)
r := run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), o.env...), "go", args...)
rd := newReader(res, d)
noise := rd.read(r.Stdout)
rd.finish()
@@ -202,3 +218,22 @@ func modulePath(gomod string) string {
}
return ""
}
// runPatternOK says whether a -run pattern is one go test takes as a single argument: no flag, no space, no control
// character, and short ("" is every test).
func runPatternOK(p string) bool {
if len(p) > 512 || strings.HasPrefix(p, "-") {
return false
}
for _, r := range p {
if r <= ' ' || r == 0x7f {
return false
}
}
if p != "" {
if _, err := regexp.Compile(strings.Split(p, "/")[0]); err != nil {
return false
}
}
return true
}
@@ -21,6 +21,9 @@ func TestMain(m *testing.M) {
if len(os.Args) > 1 && os.Args[1] == busArg {
os.Exit(runBus(os.Args[2:]))
}
if len(os.Args) > 1 && os.Args[1] == levelArg {
os.Exit(runLevelProcess(os.Args[2:])) // and the integration level's shut-off process (level.go)
}
os.Exit(m.Run())
}
@@ -54,7 +57,7 @@ func repository(t *testing.T, name string) string {
// test_test.go read the output captured from them, and this one proves the capture is what the real go says today. It
// starts go test, so it carries the tag.
func TestUnitLevelRunsTheRealGoOnTheFixtureRepositories(t *testing.T) {
needTools(t, "the unit level on fixtures", "go", "git")
needTools(t, "the unit level on fixtures", "go", "git", "bwrap")
want := map[string]struct {
passed, noTests bool
failed string
@@ -66,11 +69,18 @@ func TestUnitLevelRunsTheRealGoOnTheFixtureRepositories(t *testing.T) {
"broken": {failed: "fixture/broken"},
}
for name, w := range want {
tree := repository(t, name)
home := t.TempDir()
m := &Module{Home: home, Work: t.TempDir(), Cache: t.TempDir()}
// The tree lies in a workspace of the work directory, as a fetch leaves it: the level runs shut off in it (1309).
tree := filepath.Join(m.Work, "ws1", "tree")
if err := os.MkdirAll(filepath.Dir(tree), 0o755); err != nil {
t.Fatal(err)
}
if err := os.Rename(repository(t, name), tree); err != nil {
t.Fatal(err)
}
m.Run = StepRunner(home)
res, err := m.runLevel(context.Background(), tree, levelOpts{level: LevelUnit, bounds: Bounds{Run: UnitBound}})
res, err := m.runLevel(context.Background(), tree, levelOpts{workspace: "ws1", level: LevelUnit, bounds: Bounds{Run: UnitBound}})
if err != nil {
t.Fatalf("%s: %v", name, err)
}
@@ -96,7 +106,7 @@ func processesNamed(text string) []string {
// integration level passes, and afterwards no postgres or bus process of the run is left and its directory is gone.
// It starts postgres and a bus, so it runs in the merge check and in a try, never on the way.
func TestIntegrationLevelRunsAgainstARealStoreAndBusAndLeavesNothing(t *testing.T) {
needTools(t, "the integration level", "go", "git", "initdb", "pg_ctl", "postgres")
needTools(t, "the integration level", "go", "git", "initdb", "pg_ctl", "postgres", "bwrap")
tree := repository(t, "store")
// A short work directory: a unix socket's path is bounded at 108 bytes.
work, err := os.MkdirTemp("/tmp", "w")
@@ -113,7 +123,7 @@ func TestIntegrationLevelRunsAgainstARealStoreAndBusAndLeavesNothing(t *testing.
home := t.TempDir()
m := &Module{Recipe: Current(), Home: home, Work: work, Cache: t.TempDir()}
m.Run = StepRunner(home)
res, err := m.integration(context.Background(), "ws1", filepath.Join(work, "ws1", "tree"), nil)
res, err := m.integration(context.Background(), "ws1", filepath.Join(work, "ws1", "tree"), nil, "")
if err != nil {
t.Fatalf("the integration level: %v", err)
}
@@ -44,7 +44,7 @@ func workspaceFor(t *testing.T, captured string) (*Module, *keptStates, string,
states := newStates()
m := moduleWith(states, "go1.27.1")
m.Home, m.Work, m.Cache = t.TempDir(), t.TempDir(), t.TempDir()
m.Run = capturedRunner(t, captured, calls)
m.Run = throughBubblewrap(t, capturedRunner(t, captured, calls), &[]shutOff{}) // the step's go runs shut off (1309)
if err := os.MkdirAll(filepath.Join(m.Work, "ws1", "tree"), 0o755); err != nil {
t.Fatal(err)
}
@@ -185,12 +185,12 @@ func TestUnitLevelRunsGoTestJsonInTheModulesDirectoryWithTheStepsEnvironment(t *
// A go test that said nothing a test says (it could not run) ends the step failed, not the level.
func TestAGoTestThatCouldNotRunEndsTheStepFailed(t *testing.T) {
m, states, ws, _ := workspaceFor(t, "empty")
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
m.Run = throughBubblewrap(t, func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
if name == "go" {
return forgekey.Ran{Status: 1, Stderr: "go: cannot find main module\n"}
}
return forgekey.Ran{Stdout: "go.mod\n"}
}
}, &[]shutOff{})
_, st := runUnit(t, m, states, ws)
if st.State != delivery.StepFailed || !strings.Contains(st.Reason, "go: cannot find main module") {
t.Errorf("the step ended %s: %q", st.State, st.Reason)
@@ -0,0 +1,3 @@
module fixture/probe
go 1.25.0
@@ -0,0 +1,34 @@
package probe
import (
"encoding/json"
"net"
"os"
"testing"
"time"
)
// TestProbe tries what a change's code must not reach, named by PROBE_* (the test that runs it sets them through
// the environment of the step), and writes what it saw to seen.json beside it.
func TestProbe(t *testing.T) {
seen := map[string]bool{}
read := func(p string) bool { _, err := os.ReadFile(p); return err == nil }
seen["key"] = read(os.Getenv("PROBE_KEY"))
seen["login"] = read(os.Getenv("PROBE_LOGIN"))
_, err := net.LookupHost("git.novox.be")
seen["resolved"] = err == nil
reach := func(network, addr string) bool {
c, err := net.DialTimeout(network, addr, 2*time.Second)
if err == nil {
c.Close()
}
return err == nil
}
seen["tcp"] = reach("tcp", os.Getenv("PROBE_TCP"))
seen["socket"] = reach("unix", os.Getenv("PROBE_SOCKET"))
seen["wroteCache"] = os.WriteFile(os.Getenv("PROBE_CACHE")+"/written-by-a-candidate", []byte("x"), 0o644) == nil
raw, _ := json.Marshal(seen)
if err := os.WriteFile("seen.json", raw, 0o644); err != nil {
t.Fatal(err)
}
}
+5
View File
@@ -119,6 +119,11 @@
"type": "package",
"package": "openssh"
},
{
"id": "bubblewrap",
"type": "package",
"package": "bubblewrap"
},
{
"id": "postgresql",
"type": "package",