From 837cd2570a14917e33cffc8c858a5af6bfcd65bf Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 20:56:03 +0100 Subject: [PATCH] mesh-build-go: take mesh packages from the Go package registry alone The public proxy serves the forge's paths from git tags and direct reaches the forge's git, so a version missing from the registry was found elsewhere instead of failing loudly. The fetch now downloads in two passes over every directory: first each pinned mesh package with GOPROXY=,off (a miss says "the Go package registry holds no @"; no binding refuses "no Go package registry bound"), and every zero-version requirement of the graph is taken or refused then; second the rest of the build list with the public proxy alone, no direct. Before either, a go.mod naming a mesh package at a version other than the zero version is refused, so no forge path can reach the public proxy; a pull request head is placed only for an ask's commit, and the placement of go.mod-named pins goes. Every other go command (test, lint, compile) runs with GOPROXY=off; GOPRIVATE and the account's ssh settings leave goEnv, as repository fetches set their own. Builds: design 47 H2 H-A-494, H-A-494 addendum (two downloads), H-A-494 second addendum (a forge path at a real version is refused), H-A-494 third addendum (a pull request head places only an ask's commit pin); design 51 Go packages (the recipe test refuses direct) --- modules/mesh-build-go/README.md | 18 +- .../mesh-build-go/cmd/mesh-build-go/fetch.go | 160 ++++++++++++------ .../cmd/mesh-build-go/fetch_test.go | 94 +++++++--- .../mesh-build-go/installed_toolchain_test.go | 7 + .../mesh-build-go/cmd/mesh-build-go/module.go | 6 +- .../mesh-build-go/cmd/mesh-build-go/pins.go | 124 ++++++++------ .../mesh-build-go/pins_integration_test.go | 8 +- .../cmd/mesh-build-go/pins_test.go | 105 ++++++------ .../cmd/mesh-build-go/versions.go | 23 ++- .../cmd/mesh-build-go/versions_test.go | 29 +++- .../cmd/mesh-build-go/workspace.go | 3 +- 11 files changed, 367 insertions(+), 210 deletions(-) diff --git a/modules/mesh-build-go/README.md b/modules/mesh-build-go/README.md index ceee31df6..bd6f2bc30 100644 --- a/modules/mesh-build-go/README.md +++ b/modules/mesh-build-go/README.md @@ -37,7 +37,11 @@ Three read tools answer what the module stored in its own states, without the gu forge-account provision's ssh name, so the account's own ssh files and git rewrite are used) into a workspace of the work directory, then runs `go mod download` and `go mod verify` in every directory with a `go.mod` that does not vendor, with `GOTOOLCHAIN=local`, `GOFLAGS=-mod=mod` and the module's cache directory. Every command runs with an -environment built from scratch (`StepRunner`): `PATH`, `HOME` and what the step passes. +environment built from scratch (`StepRunner`): `PATH`, `HOME` and what the step passes. The fetch downloads in two +passes: each pinned mesh package from the Go package registry seat alone (`GOPROXY=,off`; a version the seat does +not hold fails as `the Go package registry holds no @`, and with no binding of the seat the fetch +refuses: `no Go package registry bound`), then the rest of the build list through Go's public proxy alone, never `direct`; +every other go command runs with `GOPROXY=off` (design 47 H, H-A-494). ## Room to work and the workspace's end @@ -53,11 +57,13 @@ The programs this module builds from a commit and keeps in its cache (composers asks) are read by `build_go_kept` (kind, key, size, last use) and removed by `build_go_kept_drop {kind, key}`; one no ask used for 30 days is swept with the workspaces (issue 1921). -A pin on a pull request's head (issue 1148): when `go mod download` fails on a `git.novox.be/novox/*` pseudo-version, the -fetch looks for that commit among the forge's `refs/pull/*/head`, places the repository at it under the workspace's -`pinned/beside/` as a plain directory, and hands it to Go (the fetch's commands, and the build with `GOPROXY=off`) through a -temporary `-modfile` in the workspace's `pins/` that replaces the module by that directory. A pin Go can fetch is left as -it is, a pin on no ref fails naming the module and the commit, and a directory that vendors is never resolved. +A pin on a pull request's head (issue 1148): when the ask pins a mesh package at a commit the Go package registry does not +hold, the fetch looks for that commit among the forge's `refs/pull/*/head`, places the repository at it under the +workspace's `pinned/beside/` as a plain directory, and hands it to Go (the fetch's commands, and the build with +`GOPROXY=off`) through a temporary `-modfile` in the workspace's `pins/` that replaces the module by that directory. A pin +on no ref fails naming the module and the commit, and a directory that vendors is never resolved. A go.mod names a mesh +package at the zero version only: one it names at another version is refused before any download (` names +@; a mesh package is required at the zero version`), never placed (design 47 H, H-A-494). The asked versions of the mesh's packages (novox/hq ADR 0573 §2, §4; issue 1585): a fetch given `packages` (a full commit by Go module path, which the builder and the checker fill from the indexer's trunk values or a group's heads) writes each diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index b41150326..539790977 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -194,40 +194,63 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, id str } // The directories download side by side, up to the bound (ADR 0536 §2); the first failure in directory order is the // step's. Resolving a pin places a repository beside the workspace, which two directories may share: one at a time. + // Two passes (design 47 H, H-A-494 addendum): every directory's mesh packages from the Go package registry first, + // then every directory's rest through the public proxy, so the module cache holds each mesh package before any + // download of the rest. var pins sync.Mutex errs := make([]error, len(dirs)) taken := make([][]TakenVersion, len(dirs)) slots := make(chan struct{}, m.parallel()) - var wg sync.WaitGroup - for i, d := range dirs { - if _, err := os.Stat(filepath.Join(tree, d, "vendor", "modules.txt")); err == nil { - continue // a module that vendors resolves nothing + sideBySide := func(pass func(i int, d string)) { + var wg sync.WaitGroup + for i, d := range dirs { + if _, err := os.Stat(filepath.Join(tree, d, "vendor", "modules.txt")); err == nil { + continue // a module that vendors resolves nothing + } + if errs[i] != nil { + continue // failed in the first pass + } + wg.Add(1) + slots <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-slots }() + pass(i, d) + }() } - wg.Add(1) - slots <- struct{}{} - go func() { - defer wg.Done() - defer func() { <-slots }() - // The checked repository's own modules come from the fetched tree, at whatever version go.mod names them - // (pins.go takeOwnRequirements; design 47 H, S31). - if taken[i], errs[i] = m.takeOwnRequirements(root, repository, filepath.Join(tree, d), d); errs[i] != nil { - return - } - // The asked versions of the mesh's packages go into the modfile before anything is downloaded (versions.go). - more, err := m.takeVersions(ctx, root, repository, filepath.Join(tree, d), d, req.Packages, &pins) - if taken[i], errs[i] = append(taken[i], more...), err; errs[i] != nil { - return - } - // Then what only a module the directory replaces names, which the download would not miss (pins.go). - more, errs[i] = m.takeReplacedZeroPins(ctx, root, repository, filepath.Join(tree, d), d, req.Packages, taken[i], &pins) - if taken[i] = append(taken[i], more...); errs[i] != nil { - return - } - more, errs[i] = m.downloadModules(ctx, root, repository, filepath.Join(tree, d), d, &pins, req.Packages) - taken[i] = append(taken[i], more...) - }() + wg.Wait() } - wg.Wait() + sideBySide(func(i int, d string) { + // A mesh package is required at the zero version only: one a go.mod names at another version is refused + // before either download, so the second can never carry a forge path to the public proxy (H-A-494 second addendum). + if errs[i] = namedForgeVersion(repository, filepath.Join(tree, d)); errs[i] != nil { + return + } + // The checked repository's own modules come from the fetched tree, at whatever version go.mod names them + // (pins.go takeOwnRequirements; design 47 H, S31). + if taken[i], errs[i] = m.takeOwnRequirements(root, repository, filepath.Join(tree, d), d); errs[i] != nil { + return + } + // The asked versions of the mesh's packages go into the modfile before anything is downloaded (versions.go). + more, err := m.takeVersions(ctx, root, repository, filepath.Join(tree, d), d, req.Packages, &pins) + if taken[i], errs[i] = append(taken[i], more...), err; errs[i] != nil { + return + } + // Then every requirement at the zero version of the directory's graph the asked versions did not take, its own + // and what only a module it replaces names, taken or refused before any download of the rest, so no forge + // path reaches the public proxy first (pins.go; H-A-494 third addendum). + more, errs[i] = m.takeZeroPinsFirst(ctx, root, repository, filepath.Join(tree, d), d, req.Packages, taken[i], &pins) + taken[i] = append(taken[i], more...) + }) + for _, err := range errs { + if err != nil { + return nil, err + } + } + sideBySide(func(i int, d string) { + more, err := m.downloadModules(ctx, root, repository, filepath.Join(tree, d), d, &pins, req.Packages) + taken[i], errs[i] = append(taken[i], more...), err + }) for _, err := range errs { if err != nil { return nil, err @@ -242,10 +265,10 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, id str return res, nil } -// downloadModules downloads and verifies the Go modules of the directory dir (d in the tree). A pin of a forge module -// that Go cannot fetch is looked for among the forge's pull request heads and handed to Go by a modfile (pins.go); a -// failure that names no such pin is the first failure. A requirement at the zero version is taken from the asked -// versions (pins.go takeZeroPins). +// downloadModules downloads and verifies the Go modules of the directory dir (d in the tree): the fetch's second +// download, after every mesh package was taken. A requirement at the zero version a failure still names is taken from +// the asked versions (pins.go takeZeroPins); any other failure is the first failure. A version a go.mod names is never +// placed from a pull request's head (H-A-494 third addendum): namedForgeVersion refused it before. func (m *Module) downloadModules(ctx context.Context, root, repository, dir, d string, pins *sync.Mutex, asked sdkstep.PackageVersions) ([]TakenVersion, error) { var taken []TakenVersion for { @@ -261,13 +284,7 @@ func (m *Module) downloadModules(ctx context.Context, root, repository, dir, d s if err != nil { return taken, err } - pins.Lock() - progress, err := m.resolvePins(ctx, root, dir, failure.Error()) - pins.Unlock() - if err != nil { - return taken, err - } - if !progress && len(took) == 0 { + if len(took) == 0 { return taken, failure } } @@ -444,10 +461,11 @@ func (m *Module) downloadAndVerify(ctx context.Context, dir, d string) error { } } } - cmd, env := []string{"-C", dir, "mod", verb}, m.goEnvFor(dir) + // The second download: the rest of the build list through Go's public proxy alone (publicEnv). + cmd, env := []string{"-C", dir, "mod", verb}, publicEnv(m.goEnvFor(dir)) if modfile, ok := m.pinModfile(dir); ok && verb == "verify" { // go mod verify takes no -modfile: it is run on the modfile itself, which names the placed repositories. - cmd, env = []string{"-C", filepath.Dir(modfile), "mod", verb}, m.goEnv() + cmd, env = []string{"-C", filepath.Dir(modfile), "mod", verb}, publicEnv(m.goEnv()) } r := m.Run(ctx, env, "go", cmd...) if r.Status != 0 || r.Err != "" { @@ -521,8 +539,10 @@ func inTestdata(path string) bool { } // goEnv is what a go command of a step is given beyond the base environment: the installed go alone (GOTOOLCHAIN=local, ADR 0477 §1: no toolchain is ever downloaded), -// modules resolved by their go.mod and go.sum, the module's own caches, the mesh's own modules fetched directly (by -// git, through the account's rewrite of the forge's address to ssh), and nothing that asks a question. For a directory +// modules resolved by their go.mod and go.sum, the module's own caches, the checksum database off for the forge's paths +// (go.sum and the builder's h1: digest guard them, ADR 0605 §6), and no proxy at all (GOPROXY=off): only the fetch's two +// downloads reach the network, each with its own proxy (registryEnv, publicEnv; design 47 H, H-A-494 and its addendum), +// and every other step (the test and lint steps, the compile) reads the module cache the fetch filled. For a directory // that vendors, see goEnvFor. func (m *Module) goEnv() []string { modcache := filepath.Join(m.Cache, "gomod") @@ -534,20 +554,48 @@ func (m *Module) goEnv() []string { "GOFLAGS=-mod=mod", "GOMODCACHE=" + modcache, "GOCACHE=" + filepath.Join(m.Cache, "gobuild"), + "GONOSUMDB=" + ForgeGoPaths, + "GOPROXY=off", } - if proxy := m.goSeat(); proxy != "" { - // The Go seat first, the public proxy second, the forge's git last (CAT-BG3, E11: direct goes in the first - // change after mesh-sdk's and mesh-contracts' first numbered versions are in the Go seat); no GOPRIVATE, and - // the checksum database off for the forge's paths, which go.sum and the builder's h1: digest guard instead. - // The seat goes on to the public proxy only on a 404 or 410 (",": a package it does not hold), so a seat that - // fails otherwise (5xx, refused, timeout) fails the fetch loudly; the public proxy goes on to direct on any - // failure ("|"): a public proxy the node cannot reach stopped the fetch before direct, and direct reaches the - // forge's git for a package the seat does not hold (issue 1966, ADR 0605 §6). - env = append(env, "GOPROXY="+proxy+","+PublicGoProxy+"|direct", "GONOSUMDB="+ForgeGoPaths, "GOPRIVATE=") - } else { - env = append(env, "GOPRIVATE="+ForgeGoPaths) + return append(env, m.threadEnv()...) +} + +// errNoRegistry is a fetch that needs a mesh package with no binding of the Go package registry seat: the package +// comes from that seat alone (design 47 H, H-A-494), never from the public proxy nor the forge's git. +var errNoRegistry = errors.New("no Go package registry bound") + +// registryEnv is env for the fetch's first download, of each pinned mesh package (takeVersion): the Go package registry +// seat alone and nothing behind it (GOPROXY=,off), so a version the seat does not hold fails as its miss +// (registryMiss), never reaching the public proxy, which serves the forge's paths from git tags, nor the forge's git +// (design 47 H, H-A-494 addendum). With no binding it refuses (errNoRegistry). +func (m *Module) registryEnv(env []string) ([]string, error) { + seat := m.goSeat() + if seat == "" { + return nil, errNoRegistry } - return append(append(env, "GIT_TERMINAL_PROMPT=0", "GIT_SSH_COMMAND=ssh -o BatchMode=yes"), m.threadEnv()...) + return withProxy(env, seat+",off"), nil +} + +// publicEnv is env for the fetch's second download, of the rest of the build list: Go's public proxy alone, no direct +// (design 47 H, H-A-494 addendum). The module cache already holds the mesh packages the first download took, so a forge +// path never reaches the public proxy. +func publicEnv(env []string) []string { return withProxy(env, PublicGoProxy) } + +// withProxy is env with its GOPROXY set to proxy, once. +func withProxy(env []string, proxy string) []string { + out := make([]string, 0, len(env)+1) + for _, e := range env { + if !strings.HasPrefix(e, "GOPROXY=") { + out = append(out, e) + } + } + return append(out, "GOPROXY="+proxy) +} + +// registryMiss says whether a failure of the first download is the Go package registry seat not holding the version: +// the seat answered 404 or 410, and Go, with nothing behind the seat but off, words it as the lookup off. +func registryMiss(failure string) bool { + return strings.Contains(failure, "disabled by GOPROXY=off") || strings.Contains(failure, "404 Not Found") || strings.Contains(failure, "410 Gone") } // PublicGoProxy is Go's public module proxy; ForgeGoPaths the module paths the forge serves. @@ -556,7 +604,7 @@ const ( ForgeGoPaths = "git.novox.be" ) -// goSeat is the Go seat's proxy URL from its binding (`at`, and `serves` scheme, port and go-path), empty when the +// goSeat is the Go package registry seat's proxy URL from its binding (`at`, and `serves` scheme, port and go-path), empty when the // module holds no binding of it or the binding names no address. func (m *Module) goSeat() string { if m.GoSeatFile == "" { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go index 396de3cb0..5086a427e 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go @@ -95,9 +95,33 @@ func fetchModule(t *testing.T, f *fakeStep) (*Module, *keptStates) { states := newStates() m := moduleWith(states, "go1.27.1") m.Run, m.Home, m.Work, m.Cache = f.run, t.TempDir(), t.TempDir(), t.TempDir() + bindRegistry(t, m) return m, states } +// testRegistry is the Go package registry seat's proxy a test binding names (bindRegistry). +const testRegistry = "http://forge.mesh:3000/api/packages/novox/go" + +// bindRegistry gives m a binding of the Go package registry seat, as every build node has (go-package-registry). +func bindRegistry(t *testing.T, m *Module) { + t.Helper() + m.GoSeatFile = filepath.Join(t.TempDir(), "go-package-registry.json") + if err := os.WriteFile(m.GoSeatFile, []byte(`{"at":"forge.mesh","serves":{"scheme":"http","port":3000,"go-path":"/api/packages/novox/go"}}`), 0o600); err != nil { + t.Fatal(err) + } +} + +// proxyValues are the GOPROXY settings of env. +func proxyValues(env []string) []string { + var out []string + for _, e := range env { + if strings.HasPrefix(e, "GOPROXY=") { + out = append(out, e) + } + } + return out +} + // fetchTool is the fetch tool as a build module is asked, always with its module's directory (A-447b): the root of the // tree unless the args name one. rawFetchTool is the tool as it is. func fetchTool(t *testing.T, m *Module) func(map[string]any) (any, error) { @@ -132,10 +156,20 @@ var fetchArgs = map[string]any{"repository": "novox/mesh-sdk", "commit": commitS // directory (not in one that vendors) with GOTOOLCHAIN=local, GOFLAGS=-mod=mod and the module's cache paths. func TestFetchAsksGitForTheSshRemoteAndDownloadsEachModule(t *testing.T) { // The root go.mod replaces the other two by directory: a fetch asked with a module's directory chooses what it replaces. + // Each Go module requires a mesh package at the zero version, which the ask pins (H-A-494 addendum: the first download). + const contracts = "git.novox.be/novox/mesh-contracts/go/x" f := &fakeStep{gomods: "go.mod\nsub/dir/go.mod\nvend/go.mod\n", vendors: []string{"vend"}, - files: map[string]string{"go.mod": "module fake\nreplace a => ./sub/dir\nreplace b => ./vend\n"}} + files: map[string]string{"go.mod": "module fake\nrequire " + contracts + " " + zeroVersion + "\nreplace a => ./sub/dir\nreplace b => ./vend\n", + "sub/dir/go.mod": "module fake/sub\nrequire " + contracts + " " + zeroVersion + "\n"}, + goRule: func(line string) forgekey.Ran { + if strings.Contains(line, " mod download -json "+contracts+"@") { + return forgekey.Ran{Stdout: `{"Path":"` + contracts + `","Version":"v0.1.85","Sum":"h1:zip=","GoModSum":"h1:mod="}`} + } + return forgekey.Ran{} + }} m, states := fetchModule(t, f) - if _, err := fetchTool(t, m)(fetchArgs); err != nil { + m.Parallel = 1 // one directory at a time, so the order of the commands is the fetch's own + if _, err := fetchTool(t, m)(map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA, "packages": map[string]any{contracts: "v0.1.85"}}); err != nil { t.Fatal(err) } st := states.waitEnd(t) @@ -165,27 +199,47 @@ func TestFetchAsksGitForTheSshRemoteAndDownloadsEachModule(t *testing.T) { if strings.Contains(c.line, "vend") { t.Errorf("a module that vendors was downloaded: %s", c.line) } + // The second download (H-A-494 addendum): the rest of the build list through Go's public proxy alone. env := strings.Join(c.env, " ") - for _, want := range []string{"GOTOOLCHAIN=local", "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), "GOCACHE=" + filepath.Join(m.Cache, "gobuild"), "GOPRIVATE=git.novox.be"} { + for _, want := range []string{"GOTOOLCHAIN=local", "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), "GOCACHE=" + filepath.Join(m.Cache, "gobuild"), "GOPROXY=" + PublicGoProxy} { if !strings.Contains(env, want) { t.Errorf("%s: environment lacks %s: %s", c.line, want, env) } } - } - // CAT-BG3: with the Go seat bound, the fetch goes through it first, Go's public proxy second and the forge's git - // last, with no GOPRIVATE and the checksum database off for the forge's paths. - seat := filepath.Join(t.TempDir(), "go-package-registry.json") - if err := os.WriteFile(seat, []byte(`{"at":"forge.mesh","serves":{"scheme":"http","port":3000,"go-path":"/api/packages/novox/go"}}`), 0o600); err != nil { - t.Fatal(err) - } - m.GoSeatFile = seat - env := strings.Join(m.goEnv(), " ") - for _, want := range []string{"GOPROXY=http://forge.mesh:3000/api/packages/novox/go,https://proxy.golang.org|direct", "GONOSUMDB=git.novox.be", "GOPRIVATE= "} { - if !strings.Contains(env+" ", want) { - t.Errorf("with the Go seat bound the environment lacks %q: %s", want, env) + if got := proxyValues(c.env); len(got) != 1 || got[0] != "GOPROXY="+PublicGoProxy { + t.Errorf("%s: GOPROXY settings %v, want the public proxy alone", c.line, got) } } - m.GoSeatFile = "" + // Design 51 *Go packages*, H-A-494: no go command of the fetch reaches the forge's git (Go's direct), and none is + // handed GOPRIVATE or the account's ssh; the checksum database stays off for the forge's paths (ADR 0605 §6), and a + // step's own environment has no proxy at all. + for _, c := range f.calls { + if !strings.HasPrefix(c.line, "go ") { + continue + } + for _, e := range c.env { + if strings.HasPrefix(e, "GOPROXY=") && strings.Contains(e, "direct") || strings.HasPrefix(e, "GOPRIVATE=") || strings.HasPrefix(e, "GIT_SSH_COMMAND=") { + t.Errorf("%s: the environment says %s", c.line, e) + } + } + } + // H-A-494 addendum: every download of the Go package registry seat comes before every download through the public proxy. + lastRegistry, firstPublic, registry := -1, -1, 0 + for i, c := range f.calls { + switch got := proxyValues(c.env); { + case len(got) == 1 && got[0] == "GOPROXY="+testRegistry+",off": + lastRegistry, registry = i, registry+1 + case len(got) == 1 && got[0] == "GOPROXY="+PublicGoProxy && firstPublic < 0: + firstPublic = i + } + } + if registry != 2 || firstPublic < 0 || lastRegistry > firstPublic { + t.Errorf("%d downloads from the seat, the last at %d, the first public at %d: want both directories' seat downloads before every public one:\n%s", registry, lastRegistry, firstPublic, lines) + } + env := strings.Join(m.goEnv(), " ") + if got := proxyValues(m.goEnv()); len(got) != 1 || got[0] != "GOPROXY=off" || !strings.Contains(env, "GONOSUMDB=git.novox.be") { + t.Errorf("a step's environment says GOPROXY %v, want off once, and GONOSUMDB=git.novox.be: %s", got, env) + } if downloads != 2 || verifies != 2 { t.Errorf("%d downloads and %d verifies, want 2 each (go.mod and sub/dir/go.mod):\n%s", downloads, verifies, lines) } @@ -265,18 +319,18 @@ func TestAVerifyThatFailsEndsTheStepNamingTheDirectoryAndTheLine(t *testing.T) { loadedModuleMark + "github.com/nats-io/nats.go v1.54.0\n", tagged: loadedModuleMark + "github.com/nats-io/nats-server/v2 v2.11.17\n", files: map[string]string{ - "tools/go.mod": "module fixture/tools\n\nrequire (\n\tgit.novox.be/novox/mesh-sdk/go " + zeroVersion + "\n\tgithub.com/nats-io/nats.go v1.54.0\n)\n", + "tools/go.mod": "module fixture/tools\n\nrequire (\n\tgit.novox.be/novox/mesh-sdk/go " + zeroVersion + "\n\tgithub.com/nats-io/nats.go v1.54.0\n)\n\nreplace git.novox.be/novox/mesh-sdk/go => ./sdk\n", "tools/go.sum": "github.com/nats-io/nats.go v1.54.0 h1:a=\ngithub.com/nats-io/nats.go v1.54.0/go.mod h1:b=\n", }, want: []string{"tools", "go.mod: require github.com/nats-io/nats-server/v2 v2.11.17", "go.sum: github.com/nats-io/nats-server/v2 v2.11.17 h1:", "go.sum: github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:"}, notWant: "nats.go"}, // Issue 1966's kind: a mesh module reached only through another, which go.mod does not require, is lacked at the zero - // version (ADR 0573 §2); one go.mod names at a commit (§4) is not. + // version (ADR 0573 §2); one go.mod requires is not. {name: "a mesh module the load reaches is not required", ran: "tools list -e -deps -test", out: loadedModuleMark + seatPkg + " " + zeroVersion + "\n" + - loadedModuleMark + "git.novox.be/novox/mesh-sdk/go v0.0.0-20261112000000-0123456789ab\n", - files: map[string]string{"tools/go.mod": "module fixture/tools\n\nrequire git.novox.be/novox/mesh-sdk/go v0.0.0-20261112000000-0123456789ab\n"}, + loadedModuleMark + "git.novox.be/novox/mesh-sdk/go " + zeroVersion + " /tree/sdk \n", + files: map[string]string{"tools/go.mod": "module fixture/tools\n\nrequire git.novox.be/novox/mesh-sdk/go " + zeroVersion + "\n\nreplace git.novox.be/novox/mesh-sdk/go => ./sdk\n"}, want: []string{"tools", "go.mod: require " + seatPkg + " " + zeroVersion}, notWant: "mesh-sdk"}, } { t.Run(c.name, func(t *testing.T) { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/installed_toolchain_test.go b/modules/mesh-build-go/cmd/mesh-build-go/installed_toolchain_test.go index e1fe33d8c..ec1b45bd9 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/installed_toolchain_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/installed_toolchain_test.go @@ -45,6 +45,13 @@ func TestEveryStepEnvironmentSaysGotoolchainLocal(t *testing.T) { if got := toolchainValues(env); len(got) != 1 || got[0] != "GOTOOLCHAIN=local" { t.Errorf("%s: GOTOOLCHAIN settings %v, want exactly GOTOOLCHAIN=local", name, got) } + // H-A-494: a step's own go commands reach no proxy (the test and lint steps, the compile); only the fetch's two + // downloads do, each with its own. + if strings.HasPrefix(name, "goEnv") { + if got := proxyValues(env); len(got) != 1 || got[0] != "GOPROXY=off" { + t.Errorf("%s: GOPROXY settings %v, want exactly GOPROXY=off", name, got) + } + } // Issue 1806: and its threads, its part of the node's processor cap (probeEnv runs no build). if procs := strings.Count(strings.Join(env, "\n"), "GOMAXPROCS="); name != "probeEnv" && procs != 1 { t.Errorf("%s: GOMAXPROCS said %d times, want once: %v", name, procs, env) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module.go b/modules/mesh-build-go/cmd/mesh-build-go/module.go index 949698e3b..7fa2380dd 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/module.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/module.go @@ -83,9 +83,9 @@ type Module struct { // Inside is the integration level's own process, started shut off in bubblewrap by a test step (level.go): its // commands run as they are, never shut off a second time. Inside bool - // GoSeatFile is the binding of the provision go-package-registry, the Go seat's read side (design 51 *Go - // packages*): bound, the mesh's Go modules are fetched through it first, Go's public proxy second, and the forge's - // git over the account's ssh last (Go's direct, kept for this milestone only); unbound, as before, directly. + // GoSeatFile is the binding of the provision go-package-registry, the Go package registry seat's read side (design + // 51 *Go packages*): the mesh's Go packages are fetched through it alone (design 47 H, H-A-494); unbound, a fetch + // that needs one refuses (errNoRegistry). GoSeatFile string // ArtifactStoreFile is the binding of the provision artifact-store, read for a group run's package members (§465). ArtifactStoreFile string diff --git a/modules/mesh-build-go/cmd/mesh-build-go/pins.go b/modules/mesh-build-go/cmd/mesh-build-go/pins.go index 2f55cd759..6f6e92514 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/pins.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/pins.go @@ -1,12 +1,13 @@ package main -// Pins on a pull request's head (novox/hq ADR 0482 §2, ADR 0500 §2; issue 1148): a group's Go members pin the head of -// the group's mesh-sdk pull request, a commit that only refs/pull//head of the forge carries until the merge. Go -// cannot fetch it by its pseudo-version, so a fetch that meets "unknown revision" for such a pin finds the commit among -// the forge's pull request heads, places that repository beside the workspace as a plain directory, and hands it to -// Go through a temporary -modfile, a copy of the directory's go.mod that replaces the module by that directory. The -// fetch's go commands and the build use the same modfile, and the build still runs with GOPROXY=off. A pin Go can -// fetch is not touched, and a directory that vendors is never resolved. +// Pins on a pull request's head (novox/hq ADR 0482 §2, ADR 0500 §2, ADR 0573 §4; issue 1148): a group's check asks a +// mesh package at the head of the group's pull request, a commit that only refs/pull//head of the forge carries until +// the merge. The Go package registry does not hold it, so the fetch finds the commit among the forge's pull request +// heads, places that repository beside the workspace as a plain directory, and hands it to Go through a temporary +// -modfile, a copy of the directory's go.mod that replaces the module by that directory. The fetch's go commands and +// the build use the same modfile, and the build still runs with GOPROXY=off. Only a pin the ask gives as a commit is +// placed; a version a go.mod names is refused, never placed (design 47 H, H-A-494 second and third addenda), and a +// directory that vendors is never resolved. import ( "context" @@ -111,44 +112,6 @@ func (m *Module) pinModfile(dir string) (string, bool) { return p, err == nil } -// resolvePins resolves the pins of dir that a go command failed on: those whose commit the failure names. It answers -// false when there is none, and the failure is then the caller's. -func (m *Module) resolvePins(ctx context.Context, root, dir, failure string) (bool, error) { - raw, err := os.ReadFile(filepath.Join(dir, "go.mod")) - if err != nil { - return false, nil - } - modfile, _ := m.pinModfile(dir) - done := "" - if modfile != "" { - if b, err := os.ReadFile(modfile); err == nil { - done = string(b) - } - } - progress := false - for _, p := range forgePins(string(raw)) { - if p.short == zeroShort { - continue // takeZeroPins': a requirement at the zero version names no commit to look for - } - if !strings.Contains(failure, p.short) || strings.Contains(done, "replace "+p.module+" =>") { - continue - } - where, err := m.placePin(ctx, root, p, failure) - if errors.Is(err, errNotAHead) { - // On no pull request head: go's failure is the caller's, in go's own words (issue 1748). - continue - } - if err != nil { - return false, err - } - if err := m.replaceInModfile(dir, p.module, where); err != nil { - return false, err - } - progress = true - } - return progress, nil -} - // errNotAHead is a pin whose commit no pull request head of the forge carries, when go's failure was not go knowing no // such revision: the failure is the network's or the forge's, and the caller says it in go's words (issue 1748). var errNotAHead = errors.New("on no pull request head") @@ -359,21 +322,72 @@ func (m *Module) takeZeroPins(ctx context.Context, root, repository, dir, d, fai return m.takeZeroRequirements(ctx, root, repository, dir, d, zeroRequirementsOf(dir), asked, pins) } -// takeReplacedZeroPins takes, before anything is downloaded, the requirements at the zero version that only a module -// dir replaces by a directory names (zeroRequirementsOf): go mod download loads only what dir's own go.mod names, so it -// passes without them, and the shut-off steps would look them up on the network (issue 1966). One the asked versions -// took (before, takeVersions) is left as it is. -func (m *Module) takeReplacedZeroPins(ctx context.Context, root, repository, dir, d string, asked sdkstep.PackageVersions, before []TakenVersion, pins *sync.Mutex) ([]TakenVersion, error) { - var replaced []zeroRequirement +// takeZeroPinsFirst takes or refuses, before anything else is downloaded, every requirement at the zero version of +// dir's module graph (zeroRequirementsOf) the asked versions did not take (before, takeVersions): dir's own, and those +// only a module dir replaces by a directory names, which go mod download would not miss and the shut-off steps would +// look up on the network (issue 1966). So no forge path reaches the second download, the public proxy's, first (design +// 47 H, H-A-494 third addendum). +func (m *Module) takeZeroPinsFirst(ctx context.Context, root, repository, dir, d string, asked sdkstep.PackageVersions, before []TakenVersion, pins *sync.Mutex) ([]TakenVersion, error) { + raw, _ := os.ReadFile(filepath.Join(dir, "go.mod")) + _, replaced := goModRequirements(string(raw)) + var reqs []zeroRequirement for _, r := range zeroRequirementsOf(dir) { - if r.by != "" && !slices.ContainsFunc(before, func(tv TakenVersion) bool { return tv.Package == r.module }) { - replaced = append(replaced, r) + // One dir's go.mod replaces is decided by its replace. + if !replaced[r.module] && !slices.ContainsFunc(before, func(tv TakenVersion) bool { return tv.Package == r.module }) { + reqs = append(reqs, r) } } - if len(replaced) == 0 { + if len(reqs) == 0 { return nil, nil } - return m.takeZeroRequirements(ctx, root, repository, dir, d, replaced, asked, pins) + return m.takeZeroRequirements(ctx, root, repository, dir, d, reqs, asked, pins) +} + +// namedForgeVersion refuses a mesh package (a module of the forge) that dir's module graph names at a version other +// than the zero version (ADR 0573 §4; design 47 H, H-A-494 second addendum), in the graph zeroRequirementsOf reads: dir's +// go.mod, then each module it replaces by a directory. A module of the checked repository (built from its tree) and one +// dir's go.mod replaces are left out. The first such requirement is named, before either download. +func namedForgeVersion(repository, dir string) error { + raw, err := os.ReadFile(filepath.Join(dir, "go.mod")) + if err != nil { + return nil + } + _, replaced := goModRequirements(string(raw)) + check := func(gomod string) error { + required, _ := goModRequirements(gomod) + for _, pkg := range slices.Sorted(maps.Keys(required)) { + v := required[pkg] + if !strings.HasPrefix(pkg, forgePrefix) || v == zeroVersion || replaced[pkg] || sameRepository(pkg, repository) { + continue + } + return fmt.Errorf("%s names %s@%s; a mesh package is required at the zero version", modulePathOf(gomod), pkg, v) + } + return nil + } + if err := check(string(raw)); err != nil { + return err + } + local := localReplaces(string(raw)) + for _, module := range slices.Sorted(maps.Keys(local)) { + b, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(local[module]), "go.mod")) + if err != nil { + continue // a replace to a directory with no go.mod: Go says so in its own words + } + if err := check(string(b)); err != nil { + return err + } + } + return nil +} + +// modulePathOf is the module path a go.mod declares, "" when it declares none. +func modulePathOf(gomod string) string { + for _, line := range strings.Split(gomod, "\n") { + if f := strings.Fields(line); len(f) >= 2 && f[0] == "module" { + return strings.Trim(f[1], `"`) + } + } + return "" } // takeZeroRequirements takes, for each requirement at the zero version, the version it is built against (ADR 0573 §2, diff --git a/modules/mesh-build-go/cmd/mesh-build-go/pins_integration_test.go b/modules/mesh-build-go/cmd/mesh-build-go/pins_integration_test.go index 998c89a94..f39b98be0 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/pins_integration_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/pins_integration_test.go @@ -17,9 +17,9 @@ import ( ) // 1148/AC2 as issue 1585 S15 has it (novox/hq ADR 0573 §4): given a forge with a mesh-sdk-like pull request whose branch -// is deleted (only refs/pull/7/head carries its commit) and a Go member whose go.mod names an older commit of it, when +// is deleted (only refs/pull/7/head carries its commit) and a Go member whose go.mod names it at the zero version, when // build_go_fetch is asked that head as the package's version and build_go_build runs on the member, then both pass, the -// build with GOPROXY=off against the head: the older commit holds no Go package at all. +// build with GOPROXY=off against the head (a go.mod names a mesh package at the zero version only, design 47 H, H-A-494). func TestRealFetchAndBuildOfAMemberAskedAPullHeadItsGoModDoesNotName(t *testing.T) { t.Parallel() // Not skipped when a tool is missing: a pass of the check must mean this test ran (the check prints no test log). @@ -36,7 +36,7 @@ func TestRealFetchAndBuildOfAMemberAskedAPullHeadItsGoModDoesNotName(t *testing. }, "refs/pull/7/head") git(t, work, "checkout", "-q", "main") // no branch of the repository carries the head member := map[string]string{ - "go.mod": "module fixture/member\n\ngo 1.25.0\n\nrequire git.novox.be/novox/dep/go v0.0.0-20261021120000-" + fork[:12] + "\n", + "go.mod": "module fixture/member\n\ngo 1.25.0\n\nrequire git.novox.be/novox/dep/go " + zeroVersion + "\n", "cmd/prog/main.go": "package main\n\nimport (\n\t\"fmt\"\n\n\tdep \"git.novox.be/novox/dep/go\"\n)\n\n" + "func main() { fmt.Println(dep.Name) }\n", "module.json": `{"module":"fixture-member","version":"1","build":{"artifacts":[{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/prog","binary":"prog-bin","loads":["prog-bin"]}]}}`, @@ -50,7 +50,7 @@ func TestRealFetchAndBuildOfAMemberAskedAPullHeadItsGoModDoesNotName(t *testing. if st.State != sdkstep.StepPassed { t.Fatalf("the fetch ended %s: %s", st.State, st.Reason) } - if len(res.Packages) != 1 || res.Packages[0].Built != head || res.Packages[0].Named != "v0.0.0-20261021120000-"+fork[:12] { + if len(res.Packages) != 1 || res.Packages[0].Built != head || res.Packages[0].Named != zeroVersion { t.Errorf("the fetch names what it built against: %+v, want the asked head %s", res.Packages, head) } if _, err := os.Stat(filepath.Join(m.Work, res.Workspace, "pinned", "dep@"+head, "beside", "dep", "go", "dep.go")); err != nil { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go b/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go index c9df8da62..105f302ca 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go @@ -18,7 +18,11 @@ import ( // pinShort is the 12 digits of commitSHA a pseudo-version ends in. const pinShort = "0123456789ab" -const pinnedGoMod = "module fixture/member\n\ngo 1.26\n\nrequire (\n\tgit.novox.be/novox/mesh-sdk/go v0.1.15-0.20261022120000-" + pinShort + "\n)\n" +// pinnedGoMod requires mesh-sdk at the zero version (ADR 0573 §4); the ask pins it (pinAsk). +const pinnedGoMod = "module fixture/member\n\ngo 1.26\n\nrequire (\n\tgit.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-" + zeroShort + "\n)\n" + +// pinAsk asks mesh-sdk at the head of a group's pull request, commitSHA: only refs/pull/7/head of pullList carries it. +var pinAsk = map[string]any{"git.novox.be/novox/mesh-sdk/go": commitSHA} // pinHarness runs a fetch of a tree whose only go.mod is gomod (a vendoring one when vendor is set) against a runner // that stands for the forge and for go: a pull request's head is listed by ls-remote when pulls says so, and go mod @@ -84,7 +88,7 @@ func (h *pinHarness) fetch(states *keptStates) (FetchResult, sdkstep.Step) { h.t.Helper() // A member of a repository other than mesh-sdk's, whose packages it pins: one of its own repository is built from the // tree (S31). - if _, err := fetchTool(h.t, h.m)(map[string]any{"repository": "novox/mesh-member", "commit": commitSHA}); err != nil { + if _, err := fetchTool(h.t, h.m)(map[string]any{"repository": "novox/mesh-member", "commit": commitSHA, "packages": pinAsk}); err != nil { h.t.Fatal(err) } st := states.waitEnd(h.t) @@ -149,26 +153,40 @@ func TestAPinOnAPullHeadIsPlacedBesideAndHandedToGoByAModfile(t *testing.T) { } } -// 1148/AC3: a pin go downloads as it is (on the trunk or a tag) behaves as today: no listing of pull requests, no -// repository placed, no -modfile in the fetch or the build. -func TestAPinGoCanDownloadIsLeftAsToday(t *testing.T) { - h, states := newPinHarness(t, pinnedGoMod, false, pullList) - h.onTrunk = true - res, st := h.fetch(states) - if st.State != sdkstep.StepPassed { - t.Fatalf("the step ended %s: %s", st.State, st.Reason) +// H-A-494 second addendum: a go.mod naming a mesh package at a version other than the zero version (a pseudo-version, a +// release) is refused before either download, naming the module and the version, whether its own go.mod names it or a +// module it replaces by a directory; a module of the checked repository, and one the go.mod replaces, are not refused. +func TestAMeshPackageNamedAtARealVersionIsRefusedBeforeEitherDownload(t *testing.T) { + const sdk = "git.novox.be/novox/mesh-sdk/go" + for _, c := range []struct{ name, gomod, sib, refused string }{ + {name: "a pseudo-version", gomod: "module fixture/member\n\nrequire " + sdk + " v0.1.15-0.20261022120000-" + pinShort + "\n", + refused: "fixture/member names " + sdk + "@v0.1.15-0.20261022120000-" + pinShort + "; a mesh package is required at the zero version"}, + {name: "a release named by a module it replaces", gomod: "module fixture/member\n\nrequire fixture/sib " + zeroVersion + "\n\nreplace fixture/sib => ./sib\n", + sib: "module fixture/sib\n\nrequire " + sdk + " v0.1.85\n", refused: "fixture/sib names " + sdk + "@v0.1.85"}, + {name: "replaced by the go.mod", gomod: "module fixture/member\n\nrequire " + sdk + " v0.1.85\n\nreplace " + sdk + " => ./sdk\n"}, + {name: "of the checked repository", gomod: "module fixture/member\n\nrequire git.novox.be/novox/mesh-member/go v0.1.85\n"}, + } { + t.Run(c.name, func(t *testing.T) { + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "go.mod"), []byte(c.gomod), 0o644) + if c.sib != "" { + os.MkdirAll(filepath.Join(dir, "sib"), 0o755) + os.WriteFile(filepath.Join(dir, "sib", "go.mod"), []byte(c.sib), 0o644) + } + err := namedForgeVersion("novox/mesh-member", dir) + if (err != nil) != (c.refused != "") || (err != nil && !strings.Contains(err.Error(), c.refused)) { + t.Fatalf("refused: %v, want %q", err, c.refused) + } + }) } - if h.ran("ls-remote") || h.ran("pinned") { - t.Errorf("a pin on the trunk made the fetch look for a pull request or place a repository:\n%s", strings.Join(h.f.lines(), "\n")) + // In a fetch: the step fails so, and runs no download at all. + h, states := newPinHarness(t, "module fixture/member\n\nrequire "+sdk+" v0.1.85\n", false, pullList) + _, st := h.fetch(states) + if st.State != sdkstep.StepFailed || !strings.Contains(st.Reason, "fixture/member names "+sdk+"@v0.1.85; a mesh package is required at the zero version") { + t.Fatalf("the step ended %s: %q", st.State, st.Reason) } - dir := filepath.Join(h.m.Work, res.Workspace, "tree") - for _, e := range append(h.m.goEnvFor(dir), h.m.buildEnv(dir)...) { - if strings.Contains(e, "-modfile") { - t.Errorf("a -modfile is handed to go: %s", e) - } - } - if !h.ran(" mod download") || !h.ran(" mod verify") { - t.Errorf("go mod download and verify did not run as before") + if h.ran(" mod download") || h.ran("ls-remote") { + t.Errorf("a download or a listing ran before the refusal:\n%s", strings.Join(h.f.lines(), "\n")) } } @@ -225,42 +243,6 @@ func TestTwoDirectoriesPinningOneHeadShareOneCopy(t *testing.T) { } } -// 1148/AC1: two directories pinning two different commits of one repository get two copies, each its own. -func TestTwoCommitsOfOneRepositoryGetTwoCopies(t *testing.T) { - other := "fedcba9876543210fedcba9876543210fedcba98" - h, states := newPinHarness(t, pinnedGoMod, false, commitSHA+"\trefs/pull/7/head\n"+other+"\trefs/pull/8/head\n") - h.f.gomods = "go.mod\nsub/go.mod\n" - base := h.m.Run - h.m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { - if name == "git" && strings.Contains(strings.Join(args, " "), "pinned/mesh-sdk@"+other+"/") && strings.Contains(strings.Join(args, " "), " rev-parse HEAD") { - return forgekey.Ran{Stdout: other + "\n"} - } - // the second directory pins the other commit - if name == "git" && strings.Contains(strings.Join(args, " "), " checkout ") && !strings.Contains(strings.Join(args, " "), "pinned") { - r := base(ctx, env, name, args...) - for i, a := range args { - if a == "-C" { - os.WriteFile(filepath.Join(args[i+1], "sub", "go.mod"), []byte(strings.ReplaceAll(pinnedGoMod, pinShort, other[:12])), 0o644) - } - } - return r - } - if name == "go" && strings.Contains(strings.Join(args, " "), "/sub mod download") && !strings.Contains(strings.Join(env, " "), "-modfile=") { - return forgekey.Ran{Status: 1, Stderr: "go: git.novox.be/novox/mesh-sdk/go@v0.1.15-0.20261022120000-" + other[:12] + ": invalid version: unknown revision " + other[:12] + "\n"} - } - return base(ctx, env, name, args...) - } - _, st := h.fetch(states) - if st.State != sdkstep.StepPassed { - t.Fatalf("the step ended %s: %s", st.State, st.Reason) - } - for _, c := range []string{commitSHA, other} { - if !h.ran("pinned/mesh-sdk@" + c) { - t.Errorf("no copy of mesh-sdk at %s", c) - } - } -} - // A relative replace of a go.mod (a package of the module's own repository in a sibling directory, ADR 0593 §1) must still // resolve in the temporary modfile, which `go mod verify` runs in its own directory: every relative target is made absolute // under the tree it came from. The route-proxy failure of 2026-11-02 13:53 was a ./ replace met there; this takes real go. @@ -340,8 +322,13 @@ func TestARequirementAtTheZeroVersionIsTakenFromItsPinOrRefused(t *testing.T) { } h, states := newPinHarness(t, gomod, false, pullList) base := h.m.Run + var mu sync.Mutex + var ran []string // every command, in order h.m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { line := name + " " + strings.Join(args, " ") + mu.Lock() + ran = append(ran, line) + mu.Unlock() switch { case c.replaced && name == "git" && strings.Contains(line, " checkout "): r := base(ctx, env, name, args...) @@ -381,6 +368,12 @@ func TestARequirementAtTheZeroVersionIsTakenFromItsPinOrRefused(t *testing.T) { if h.ran(" mod download -json " + sdk + "@") { t.Errorf("go was asked for a version of %s with no pin:\n%s", sdk, strings.Join(h.f.lines(), "\n")) } + // H-A-494 third addendum: refused before any download of the rest, so no forge path reaches the public proxy. + for _, line := range ran { + if strings.HasSuffix(line, " mod download") { + t.Errorf("the rest was downloaded before the refusal: %s", line) + } + } return } if st.State != sdkstep.StepPassed { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/versions.go b/modules/mesh-build-go/cmd/mesh-build-go/versions.go index 77332ac17..cdb6ca864 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/versions.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/versions.go @@ -127,9 +127,18 @@ func (m *Module) takeVersion(ctx context.Context, root, dir string, tv *TakenVer tagged := sdkstep.PackageSemver.MatchString(tv.Asked) var r forgekey.Ran var info downloadInfo - r = m.Run(ctx, m.goEnv(), "go", "-C", pinsDir, "mod", "download", "-json", tv.Package+"@"+tv.Asked) + // The first download: a mesh package from the Go package registry seat alone (design 47 H, H-A-494 and its addendum). + env, err := m.registryEnv(m.goEnv()) + if err != nil { + return fmt.Errorf("%s@%s: %w", tv.Package, tv.Asked, err) + } + r = m.Run(ctx, env, "go", "-C", pinsDir, "mod", "download", "-json", tv.Package+"@"+tv.Asked) _ = json.Unmarshal([]byte(r.Stdout), &info) failed := r.Status != 0 || r.Err != "" || info.Version == "" || info.Error != "" + if failed && registryMiss(info.Error+"\n"+r.Err+"\n"+r.Stderr) && !commitOK.MatchString(tv.Asked) { + // Only a commit may still be a pull request's head (placePin); any other version the seat does not hold is its miss. + return registryMissed(tv.Package, tv.Asked, info.Error+"\n"+r.Err+"\n"+r.Stderr) + } if failed && tagged { failure := strings.TrimSpace(info.Error + "\n" + r.Err + "\n" + r.Stderr) return downloadFailure(failure, "downloading the group's tagged version %s of %s through the Go seat failed", tv.Asked, tv.Package) @@ -148,6 +157,9 @@ func (m *Module) takeVersion(ctx context.Context, root, dir string, tv *TakenVer where, err := m.placePin(ctx, root, p, failure) if errors.Is(err, errNotAHead) { pins.Unlock() + if registryMiss(failure) { + return registryMissed(tv.Package, tv.Asked, failure) + } // On no head, and go failed for another cause (the forge or the resolver did not answer): it fails with go's own // words, and not as a kind the builder holds until the module changes (issue 1748). return downloadFailure(failure, "downloading the asked version %s of %s failed", tv.Asked, tv.Package) @@ -188,6 +200,12 @@ func (m *Module) takeVersion(ctx context.Context, root, dir string, tv *TakenVer return m.takeRequirementsOf(ctx, pinsDir, info.GoMod) } +// registryMissed is the first download's miss: the Go package registry seat holds no such version of the package, said +// in the module's words with Go's beside them (design 47 H, H-A-494: a missing version fails loudly as the registry's miss). +func registryMissed(pkg, version, failure string) error { + return fmt.Errorf("the Go package registry holds no %s@%s: %s", pkg, version, causeLine(strings.TrimSpace(failure))) +} + // appendSums writes the checksums of one download (the module's and its go.mod's) into the go.sum in pinsDir. func appendSums(pinsDir, module string, info downloadInfo) error { sums := "" @@ -232,7 +250,8 @@ func (m *Module) takeRequirementsOf(ctx context.Context, pinsDir, gomodPath stri if strings.HasPrefix(module, forgePrefix) || (named[module] != "" && compareVersions(named[module], version) >= 0) { continue } - r := m.Run(ctx, m.goEnv(), "go", "-C", pinsDir, "mod", "download", "-json", module+"@"+version) + // Never a module of the forge (above): the rest of the build list, through Go's public proxy alone (publicEnv). + r := m.Run(ctx, publicEnv(m.goEnv()), "go", "-C", pinsDir, "mod", "download", "-json", module+"@"+version) var info downloadInfo _ = json.Unmarshal([]byte(r.Stdout), &info) if r.Status != 0 || r.Err != "" || info.Version == "" || info.Error != "" { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/versions_test.go b/modules/mesh-build-go/cmd/mesh-build-go/versions_test.go index cabcbe84d..82285d33c 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/versions_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/versions_test.go @@ -32,6 +32,7 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) { passing bool // the step ends failed as a transport fault, which the bus gives again (ADR 0664 §1) downloads int // the downloads asked in all (0: not counted) requires string // the go.mod of the version taken, its requirements to be taken too + unbound bool // the module holds no binding of the Go package registry seat }{ {name: "a trunk commit newer than go.mod's", asked: trunk, answer: `{"Path":"` + sdk + `","Version":"v0.1.15-0.20261101000000-aaaaaaaaaaaa","Sum":"h1:zip=","GoModSum":"h1:mod="}`, @@ -42,9 +43,6 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) { answer: `{"Path":"` + sdk + `","Version":"v0.1.16-0.20261116170327-aaaaaaaaaaaa","Sum":"h1:zip=","GoModSum":"h1:mod=","GoMod":"GOMOD"}`, built: "v0.1.16-0.20261116170327-aaaaaaaaaaaa", edit: "-require=" + sdk + "@v0.1.16-0.20261116170327-aaaaaaaaaaaa", requires: "module " + sdk + "\n\ngo 1.26\n\nrequire (\n\tgithub.com/nats-io/nats.go v1.54.0\n\tgithub.com/klauspost/compress v1.20.0\n\tgit.novox.be/novox/mesh-contracts/go/x v0.1.0\n)\n"}, - {name: "go.mod names a newer one", asked: trunk, - answer: `{"Path":"` + sdk + `","Version":"v0.1.15-0.20261001000000-aaaaaaaaaaaa","Sum":"h1:zip="}`, - built: "v0.1.15-0.20261022120000-" + pinShort, said: "newer than the asked"}, {name: "a group's head Go cannot fetch", asked: commitSHA, failing: true, answer: `{"Path":"` + sdk + `","Error":"invalid version: unknown revision ` + pinShort + `"}`, built: commitSHA, said: "placed from the pull request's head"}, @@ -70,11 +68,17 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) { {name: "a group's tagged version", asked: "v0.1.15-feat-x.2", answer: `{"Path":"` + sdk + `","Version":"v0.1.15-feat-x.2","Sum":"h1:zip=","GoModSum":"h1:mod="}`, built: "v0.1.15-feat-x.2", edit: "-require=" + sdk + "@v0.1.15-feat-x.2"}, - {name: "a group's tagged version older than go.mod's", asked: "v0.1.14-feat-x.2", - answer: `{"Path":"` + sdk + `","Version":"v0.1.14-feat-x.2","Sum":"h1:zip=","GoModSum":"h1:mod="}`, - built: "v0.1.14-feat-x.2", said: "the group's tagged version", edit: "-require=" + sdk + "@v0.1.14-feat-x.2"}, - {name: "a tagged version the Go seat lacks", asked: "v0.1.15-feat-x.3", failing: true, failsWith: "the group's tagged version v0.1.15-feat-x.3", + // H-A-494: a version the Go package registry seat does not hold fails as its miss, naming @: the seat's 404, + // or, as Go words it with off behind the seat, the lookup off; a commit only once no pull request head carries it. + {name: "a tagged version the Go seat lacks", asked: "v0.1.15-feat-x.3", failing: true, failsWith: "the Go package registry holds no " + sdk + "@v0.1.15-feat-x.3", answer: `{"Path":"` + sdk + `","Error":"` + sdk + `@v0.1.15-feat-x.3: reading https://seat/go/@v/v0.1.15-feat-x.3.info: 404 Not Found"}`}, + {name: "a release the Go seat lacks", asked: "v0.1.85", failing: true, failsWith: "the Go package registry holds no " + sdk + "@v0.1.85", downloads: 1, + answer: `{"Path":"` + sdk + `","Error":"` + sdk + `@v0.1.85: module lookup disabled by GOPROXY=off"}`}, + {name: "a commit the Go seat lacks on no head", asked: offHead, failing: true, failsWith: "the Go package registry holds no " + sdk + "@" + offHead, downloads: 1, + answer: `{"Path":"` + sdk + `","Error":"` + sdk + `@` + offHead + `: module lookup disabled by GOPROXY=off"}`}, + // H-A-494: with no binding of the seat, a mesh package is fetched from nowhere else: the fetch refuses. + {name: "no Go package registry bound", asked: trunk, unbound: true, failsWith: "no Go package registry bound", + answer: `{"Path":"` + sdk + `","Version":"v0.1.15-0.20261101000000-aaaaaaaaaaaa","Sum":"h1:zip=","GoModSum":"h1:mod="}`}, // ADR 0664 §1 (issue 1748): one download; a failure of the network is a transport fault the bus gives the ask again for after a // delay, no retry here; another failure is not one. {name: "an unknown revision is not taken again", asked: offHead, failing: true, failsWith: "on no pull request head", downloads: 1, @@ -85,6 +89,9 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) { t.Run(c.name, func(t *testing.T) { h, states := newPinHarness(t, pinnedGoMod, false, pullList) h.f.gomods = "go.mod\nother/go.mod\n" + if c.unbound { + h.m.GoSeatFile = "" + } downloads := 0 base := h.m.Run h.m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { @@ -130,6 +137,14 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) { t.Fatal(err) } st := states.waitEnd(t) + // H-A-494 addendum: a mesh package is downloaded from the Go package registry seat alone, off behind it. + for _, cmd := range h.cmds { + if strings.Contains(cmd.line, " mod download -json "+sdk+"@") { + if got := proxyValues(cmd.env); len(got) != 1 || got[0] != "GOPROXY="+testRegistry+",off" { + t.Errorf("%s: GOPROXY settings %v, want the seat alone and off", cmd.line, got) + } + } + } if c.downloads != 0 && downloads != c.downloads { t.Errorf("go downloaded %d times, want %d: no retry here", downloads, c.downloads) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/workspace.go b/modules/mesh-build-go/cmd/mesh-build-go/workspace.go index 2afd8fd14..c7ff57717 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/workspace.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/workspace.go @@ -167,7 +167,8 @@ func (m *Module) downloadBeside(ctx context.Context, root, tree string, dirs []s if err := os.WriteFile(file, []byte(work), 0o600); err != nil { return err } - env := append(m.goEnv(), "GOWORK="+file, "GOFLAGS=-mod=readonly") // a workspace takes no -mod=mod + // The rest of a build list: through Go's public proxy alone (publicEnv; design 47 H, H-A-494 addendum). + env := publicEnv(append(m.goEnv(), "GOWORK="+file, "GOFLAGS=-mod=readonly")) // a workspace takes no -mod=mod if res := m.Run(ctx, env, "go", "-C", r, "mod", "download"); res.Status != 0 || res.Err != "" { return fmt.Errorf("go mod download of the graph of %s beside: %s", filepath.Base(r), causeLine(res.Err+"\n"+res.Stdout+"\n"+res.Stderr)) } -- 2.54.0