Files
mesh-catalog/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go
T
jschoubben 19994c1817
mesh/delivery merged: source.main moved to novox/mesh-catalog@ea74be9e2e10
mesh/check checking 19994c1817bb on ea74be9e2e10
The fetch fails when the shut-off package load cannot run at all, naming the directory and go list's error (issue 1966, AC182-1)
Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ
2026-11-10 16:37:40 +01:00

377 lines
14 KiB
Go

package main
import (
"context"
"encoding/json"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"testing"
"time"
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
const commitSHA = "0123456789abcdef0123456789abcdef01234567"
// cmdRecord is one command a step ran.
type cmdRecord struct {
env []string
line string
}
// fakeStep answers the commands of a fetch: git succeeds, rev-parse answers the commit, ls-files answers the
// go.mod files, and a go command answers by rule.
type fakeStep struct {
mu sync.Mutex
calls []cmdRecord
gomods string
gate chan struct{} // when set, the first git command waits for it
goRule func(line string) forgekey.Ran
files map[string]string // files (relative to the tree) written at checkout: go.mod and go.sum of the fixture
vendors []string // directories (relative to the tree) that get a vendor/modules.txt at checkout
rule func(line string) (forgekey.Ran, bool) // answers a command before the defaults do
}
func (f *fakeStep) run(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
line := name + " " + strings.Join(args, " ")
f.mu.Lock()
f.calls = append(f.calls, cmdRecord{env: env, line: line})
f.mu.Unlock()
if f.gate != nil && strings.Contains(line, " init ") {
<-f.gate
}
if f.rule != nil {
if r, ok := f.rule(line); ok {
return r
}
}
switch {
case name == "git" && strings.Contains(line, " rev-parse HEAD"):
return forgekey.Ran{Stdout: commitSHA + "\n"}
case name == "git" && strings.Contains(line, " ls-files "):
return forgekey.Ran{Stdout: f.gomods}
case name == "git" && strings.Contains(line, " checkout "):
for i, a := range args {
if a == "-C" {
for name, text := range f.files {
os.MkdirAll(filepath.Dir(filepath.Join(args[i+1], name)), 0o755)
os.WriteFile(filepath.Join(args[i+1], name), []byte(text), 0o644)
}
for _, v := range f.vendors {
os.MkdirAll(filepath.Join(args[i+1], v, "vendor"), 0o755)
os.WriteFile(filepath.Join(args[i+1], v, "vendor", "modules.txt"), nil, 0o644)
}
}
}
case name == "go" && f.goRule != nil:
return f.goRule(line)
}
return forgekey.Ran{}
}
func (f *fakeStep) lines() []string {
f.mu.Lock()
defer f.mu.Unlock()
out := []string{}
for _, c := range f.calls {
out = append(out, c.line)
}
return out
}
func fetchModule(t *testing.T, f *fakeStep) (*Module, *keptStates) {
t.Helper()
states := newStates()
m := moduleWith(states, "go1.27.1")
m.Run, m.Home, m.Work, m.Cache = f.run, t.TempDir(), t.TempDir(), t.TempDir()
return m, states
}
func fetchTool(t *testing.T, m *Module) func(map[string]any) (any, error) {
t.Helper()
for _, tool := range m.Tools(m.StepTools()) {
if tool.Name == "build_go_fetch" {
return tool.Run
}
}
t.Fatal("no tool build_go_fetch")
return nil
}
var fetchArgs = map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA}
// 1485/AC1: git is asked through FetchCommit with the ssh remote, and go mod download and verify run in each go.mod
// directory (not in one that vendors) with GOTOOLCHAIN=local, GOFLAGS=-mod=mod and the module's cache paths.
func TestFetchAsksGitForTheSshRemoteAndDownloadsEachModule(t *testing.T) {
f := &fakeStep{gomods: "go.mod\nsub/dir/go.mod\nvend/go.mod\n", vendors: []string{"vend"}}
m, states := fetchModule(t, f)
if _, err := fetchTool(t, m)(fetchArgs); err != nil {
t.Fatal(err)
}
st := states.waitEnd(t)
if st.State != sdkstep.StepPassed {
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
}
lines := strings.Join(f.lines(), "\n")
for _, want := range []string{
"remote add origin ssh://mesh-forge/novox/mesh-sdk.git",
"fetch --no-tags --depth 1 origin " + commitSHA,
"checkout -q --detach " + commitSHA,
} {
if !strings.Contains(lines, want) {
t.Errorf("git was not asked %q:\n%s", want, lines)
}
}
downloads, verifies := 0, 0
for _, c := range f.calls {
switch {
case strings.HasPrefix(c.line, "go -C ") && strings.HasSuffix(c.line, " mod download"):
downloads++
case strings.HasPrefix(c.line, "go -C ") && strings.HasSuffix(c.line, " mod verify"):
verifies++
default:
continue
}
if strings.Contains(c.line, "vend") {
t.Errorf("a module that vendors was downloaded: %s", c.line)
}
env := strings.Join(c.env, " ")
for _, want := range []string{"GOTOOLCHAIN=local", "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), "GOCACHE=" + filepath.Join(m.Cache, "gobuild"), "GOPRIVATE=git.novox.be"} {
if !strings.Contains(env, want) {
t.Errorf("%s: environment lacks %s: %s", c.line, want, env)
}
}
}
// CAT-BG3: with the Go seat bound, the fetch goes through it first, Go's public proxy second and the forge's git
// last, with no GOPRIVATE and the checksum database off for the forge's paths.
seat := filepath.Join(t.TempDir(), "go-package-registry.json")
if err := os.WriteFile(seat, []byte(`{"at":"forge.mesh","serves":{"scheme":"http","port":3000,"go-path":"/api/packages/novox/go"}}`), 0o600); err != nil {
t.Fatal(err)
}
m.GoSeatFile = seat
env := strings.Join(m.goEnv(), " ")
for _, want := range []string{"GOPROXY=http://forge.mesh:3000/api/packages/novox/go,https://proxy.golang.org|direct", "GONOSUMDB=git.novox.be", "GOPRIVATE= "} {
if !strings.Contains(env+" ", want) {
t.Errorf("with the Go seat bound the environment lacks %q: %s", want, env)
}
}
m.GoSeatFile = ""
if downloads != 2 || verifies != 2 {
t.Errorf("%d downloads and %d verifies, want 2 each (go.mod and sub/dir/go.mod):\n%s", downloads, verifies, lines)
}
var res FetchResult
if err := json.Unmarshal(st.Result, &res); err != nil {
t.Fatal(err)
}
if res.Commit != commitSHA || strings.Join(res.Modules, ",") != ".,sub/dir,vend" || res.Workspace == "" {
t.Errorf("result = %s", st.Result)
}
if _, err := os.Stat(filepath.Join(m.Work, res.Workspace, "tree")); err != nil {
t.Errorf("the workspace is not in the work directory: %v", err)
}
}
// 1485/AC2: a variable such as MESH_REGISTRY_PASSWORD or a *_TOKEN of the process is seen by no command of the step:
// the runner builds the environment from scratch, and the step passes it nothing of the kind.
func TestNoCommandOfTheStepSeesAPasswordOrAToken(t *testing.T) {
t.Setenv("MESH_REGISTRY_PASSWORD", "hunter2-registry")
t.Setenv("FORGE_TOKEN", "hunter2-token")
t.Setenv("HOME", "/not/the/account")
r := StepRunner("/home/account")
got := r(context.Background(), []string{"EXTRA=1"}, "env")
if got.Status != 0 {
t.Fatalf("env: %+v", got)
}
for _, secret := range []string{"hunter2", "MESH_REGISTRY_PASSWORD", "FORGE_TOKEN", "/not/the/account"} {
if strings.Contains(got.Stdout, secret) {
t.Errorf("a command saw %q:\n%s", secret, got.Stdout)
}
}
for _, want := range []string{"HOME=/home/account", "EXTRA=1", "PATH="} {
if !strings.Contains(got.Stdout, want) {
t.Errorf("a command lacks %q:\n%s", want, got.Stdout)
}
}
f := &fakeStep{gomods: "go.mod\n"}
m, states := fetchModule(t, f)
fetchTool(t, m)(fetchArgs)
states.waitEnd(t)
for _, c := range f.calls {
for _, w := range c.env {
if k, _, _ := strings.Cut(w, "="); strings.HasSuffix(k, "TOKEN") || strings.HasSuffix(k, "PASSWORD") || strings.HasSuffix(k, "SECRET") {
t.Errorf("%s was given %s", c.line, w)
}
}
}
}
// 1485/AC3: go mod verify failing ends the step failed naming the directory and the first line of its output.
func TestAVerifyThatFailsEndsTheStepNamingTheDirectoryAndTheLine(t *testing.T) {
const seatPkg = "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
for _, c := range []struct {
name, ran, out string
fatal bool
want []string
notWant string
}{
{name: "go mod verify fails", ran: "tools mod verify", out: "\nexample.org/x v1.0.0: dir has been modified\nsecond line\n",
want: []string{"tools", "example.org/x v1.0.0: dir has been modified"}, notWant: "second line"},
// 1966/AC178-2: what the fetch downloaded misses a package the directory's code imports: the shut-off package load
// fails the fetch naming the package in Go's words, not a shut-off step later.
{name: "the shut-off package load would reach the network", ran: "tools list -e -deps -test",
out: "fixture/tools/cmd: \n" + seatPkg + ": module lookup disabled by GOPROXY=off\n",
want: []string{"tools", "shut off", seatPkg + ": module lookup disabled by GOPROXY=off"}, notWant: "fixture/tools/cmd"},
// 1966/AC182-1: a package load that could not run at all (a fatal exit, no GOPROXY=off line) fails the fetch too.
{name: "the shut-off package load cannot run", ran: "tools list -e -deps -test", fatal: true,
out: "go: go.mod file not found\n", want: []string{"tools", "go: go.mod file not found"}},
} {
t.Run(c.name, func(t *testing.T) {
f := &fakeStep{gomods: "go.mod\ntools/go.mod\n", goRule: func(line string) forgekey.Ran {
if strings.Contains(line, "/"+c.ran) {
if strings.Contains(line, " list ") {
if c.fatal {
return forgekey.Ran{Status: 1, Stderr: c.out}
}
return forgekey.Ran{Stdout: c.out} // go list -e says a package's error and exits 0
}
return forgekey.Ran{Status: 1, Stdout: c.out}
}
return forgekey.Ran{}
}}
m, states := fetchModule(t, f)
if _, err := fetchTool(t, m)(fetchArgs); err != nil {
t.Fatal(err)
}
st := states.waitEnd(t)
if st.State != sdkstep.StepFailed {
t.Fatalf("the step ended %s, want failed", st.State)
}
for _, w := range c.want {
if !strings.Contains(st.Reason, w) {
t.Errorf("reason = %q, want it to hold %q", st.Reason, w)
}
}
if c.notWant != "" && strings.Contains(st.Reason, c.notWant) {
t.Errorf("reason = %q holds %q", st.Reason, c.notWant)
}
if c.ran == "tools list -e -deps -test" {
for _, call := range f.calls {
if strings.Contains(call.line, " list -e -deps") && !slices.Contains(call.env, "GOPROXY=off") {
t.Errorf("the package load ran with the network: %s", call.env)
}
}
}
})
}
}
// 1485/AC4: the call answers a step id before the fetch ends, and the key then ends passed with the workspace.
func TestTheCallAnswersAStepIdBeforeTheFetchEnds(t *testing.T) {
f := &fakeStep{gomods: "go.mod\n", gate: make(chan struct{})}
m, states := fetchModule(t, f)
out, err := fetchTool(t, m)(fetchArgs)
if err != nil {
t.Fatal(err)
}
answer := out.(map[string]any)
id, _ := answer["step"].(string)
if id == "" || answer["key"] != "ace."+id {
t.Fatalf("answer = %v, want a step id and its key", answer)
}
select {
case <-states.ended:
t.Fatal("the step ended while git was held")
case <-time.After(50 * time.Millisecond):
}
states.mu.Lock()
running := states.steps["ace."+id]
states.mu.Unlock()
if len(running) != 1 || running[0].State != sdkstep.StepRunning {
t.Errorf("the key holds %+v, want one running step", running)
}
close(f.gate)
select {
case key := <-states.ended:
if key != "ace."+id {
t.Errorf("ended %s", key)
}
case <-time.After(30 * time.Second):
t.Fatal("the step never ended")
}
states.mu.Lock()
last := states.steps["ace."+id][len(states.steps["ace."+id])-1]
states.mu.Unlock()
var res FetchResult
if last.State != sdkstep.StepPassed || json.Unmarshal(last.Result, &res) != nil || res.Workspace == "" {
t.Errorf("the key ends %+v", last)
}
}
func TestARepositoryThatIsNotOwnerSlashNameIsRefusedBeforeAnythingRuns(t *testing.T) {
for _, repo := range []string{"", "mesh-sdk", "../etc/passwd", "a/b/c", "a/..", "-o/x", "https://x/y"} {
f := &fakeStep{}
m, states := fetchModule(t, f)
fetchTool(t, m)(map[string]any{"repository": repo, "commit": commitSHA})
st := states.waitEnd(t)
if st.State != sdkstep.StepFailed || len(f.lines()) != 0 {
t.Errorf("%q: the step ended %s after %d commands", repo, st.State, len(f.lines()))
}
}
}
// Unheld workspaces last used more than two hours ago are removed (issue 1839, ADR 0509); younger ones, and anything
// that is not a workspace, stay.
func TestUnheldWorkspacesPastTwoHoursAreRemoved(t *testing.T) {
m := &Module{Work: t.TempDir()}
for name, age := range map[string]time.Duration{"old1": 3 * time.Hour, "young": time.Hour, "Keep-This": 48 * time.Hour} {
dir := filepath.Join(m.Work, name)
os.MkdirAll(filepath.Join(dir, "tree"), 0o755)
when := time.Now().Add(-age)
os.Chtimes(dir, when, when)
}
m.CleanWorkspaces(time.Now())
for name, want := range map[string]bool{"old1": false, "young": true, "Keep-This": true} {
_, err := os.Stat(filepath.Join(m.Work, name))
if (err == nil) != want {
t.Errorf("%s present = %v, want %v", name, err == nil, want)
}
}
}
// The work and cache directories the code uses are the ones the manifest places, and neither is in the home.
func TestWorkAndCacheAreTheDirectoriesTheManifestPlacesOutsideTheHome(t *testing.T) {
m := readManifest(t)
home := resource(t, m, "account")["home"].(string)
for id, path := range map[string]string{"work": WorkDir, "cache": CacheDir} {
if resource(t, m, id)["path"] != path {
t.Errorf("%s: the manifest places %v, the code uses %s", id, resource(t, m, id)["path"], path)
}
if path == home || strings.HasPrefix(path, home+"/") {
t.Errorf("%s %s is in the home %s", id, path, home)
}
}
}
// issue 1703: a go command's reason is its cause, not Go's progress nor go build's `# <package>` heading.
func TestCauseLineSkipsProgressAndThePackageHeading(t *testing.T) {
for in, want := range map[string]string{
"go: downloading x v1\n# git.example/m/cmd\n./main.go:3:2: undefined: y\n": "./main.go:3:2: undefined: y",
"\ngo: example.com/x@v1: reading: 404\n": "go: example.com/x@v1: reading: 404",
"go: downloading x v1\n": "go: downloading x v1",
// issue 1966: a module graph error goes on, indented, on the next line; a later error is not taken in.
"a.go:3:8: x@v1 requires\n\ty@v2: module lookup disabled by GOPROXY=off\nb.go:1:1: other\n": "a.go:3:8: x@v1 requires y@v2: module lookup disabled by GOPROXY=off",
} {
if got := causeLine(in); got != want {
t.Errorf("causeLine(%q) = %q, want %q", in, got, want)
}
if got := compileError(in); got != want {
t.Errorf("compileError(%q) = %q, want %q", in, got, want)
}
}
}