From 4b9ea0bf111725a5c21c3ddcfa4e1d04f0f2f183 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 5 Nov 2026 21:14:56 +0100 Subject: [PATCH 1/4] mesh-build-node builds and tests each TypeScript package at its own path, with its own lock (ADR 0634 section 8, issue 1966) A folder module (the contracts folders) declares several package artifacts under ts/seats//, each with its own package.json and lock and none at the module directory; the fetch asked for a lock at the module directory and failed the group's check. The fetch installs, the build packs and the test level runs at the artifact's path; an artifact naming no usable path keeps the module directory. The module's .gitignore anchors the binary's name so the command directory is not ignored. Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- modules/mesh-build-node/.gitignore | 4 +- .../cmd/mesh-build-node/build.go | 25 ++- .../cmd/mesh-build-node/check.go | 179 ++++++++++++------ .../cmd/mesh-build-node/fetch.go | 23 ++- .../cmd/mesh-build-node/package.go | 12 +- .../cmd/mesh-build-node/package_paths_test.go | 93 +++++++++ 6 files changed, 265 insertions(+), 71 deletions(-) create mode 100644 modules/mesh-build-node/cmd/mesh-build-node/package_paths_test.go diff --git a/modules/mesh-build-node/.gitignore b/modules/mesh-build-node/.gitignore index f7604aaaa..ce26533a5 100644 --- a/modules/mesh-build-node/.gitignore +++ b/modules/mesh-build-node/.gitignore @@ -1 +1,3 @@ -mesh-build-node +# the binary, built at the module or in its command directory; never the command directory itself +/mesh-build-node +/cmd/mesh-build-node/mesh-build-node diff --git a/modules/mesh-build-node/cmd/mesh-build-node/build.go b/modules/mesh-build-node/cmd/mesh-build-node/build.go index 4627c2c7a..086bd7c01 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/build.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/build.go @@ -13,6 +13,7 @@ import ( "encoding/json" "fmt" "os" + "path" "path/filepath" "sort" "strings" @@ -37,6 +38,27 @@ type artifact struct { Language string `json:"language"` Entrypoints []string `json:"entrypoints"` External []string `json:"external"` + // Paths are the directories of a package artifact, relative to the module's: one, holding its package.json and + // its package-lock.json (as a Go package's, mesh-build-go gopackage.go); none means the module's directory. + Paths []string `json:"paths"` +} + +// packageDir is the directory of a package artifact relative to its module's directory in dir: the one path it names +// when that is a directory of the module holding a package.json (the contracts folders: ts/seats//, each with its +// own lock, ADR 0634 section 8), else the module's directory itself (a package that names no path, or source paths +// beside the module's package.json: today's form). +func (a artifact) packageDir(dir string) string { + if len(a.Paths) != 1 { + return "." + } + p := path.Clean(strings.TrimSuffix(a.Paths[0], "/")) + if p == "." || path.IsAbs(p) || p == ".." || strings.HasPrefix(p, "../") { + return "." + } + if _, err := os.Stat(filepath.Join(dir, filepath.FromSlash(p), "package.json")); err != nil { + return "." + } + return p } // BuildResult is what a passed build step says. @@ -88,7 +110,8 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { return nil, err } if a.Kind == "package" { - return m.buildPackage(ctx, workspace, module, root, dir, a, publish) + at := a.packageDir(dir) + return m.buildPackage(ctx, workspace, module, root, filepath.Join(dir, filepath.FromSlash(at)), at, a, publish) } if why, err := lockProblem(dir, module); err != nil || why != "" { if err == nil { diff --git a/modules/mesh-build-node/cmd/mesh-build-node/check.go b/modules/mesh-build-node/cmd/mesh-build-node/check.go index 651688ac4..cda4b1ca5 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/check.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/check.go @@ -16,6 +16,7 @@ import ( "fmt" "io/fs" "os" + "path" "path/filepath" "strings" "time" @@ -99,78 +100,89 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) { if err != nil { return nil, err } - scripts, err := packageScripts(dir) - if err != nil { - return nil, err - } - sources, err := typeScriptFiles(dir) - if err != nil { - return nil, err - } res := &TestResult{Workspace: workspace, Module: module, Level: level, By: sdkstep.By("mesh-build-node")} script := "test" if level == "integration" { script = "test:integration" } - hasScript := scripts[script] != "" - if level == "integration" && !hasScript { - res.Passed, res.NoTests, res.Why = true, true, "no tests: the package has no test:integration script" - return res, nil - } - if !hasScript && len(sources) == 0 { - res.Passed, res.NoTests, res.Why = true, true, "no tests: the package has no test script and no .ts files" - return res, nil - } - if err := m.provide(root); err != nil { - return nil, fmt.Errorf("providing the toolchain's packages to the workspace: %w", err) - } where := module if where == "" { where = "." } + targets, err := testTargets(dir, where) + if err != nil { + return nil, err + } var check, run time.Duration var failed []Failure - // a command that could not start is not a verdict: the step fails. One that ran and said no is. - ran := func(name string, r forgekey.Ran, report string) error { - if r.Err != "" { - return fmt.Errorf("%s could not be run: status %d %s: %s", name, r.Status, r.Err, firstLine(r.Stderr)) - } - if r.Status != 0 { - failed = append(failed, testFailures(where, name, dir, r, report)...) - } - return nil - } - if len(sources) > 0 { - // A package that builds (a build or prepare script, which the fetch's --ignore-scripts never runs) is emitted - // by tsc into the workspace, shut off like every run step, so its tests find what they import of its own output - // (issue 1869: mesh-sdk's tests import ../dist); any other is type-checked without emitting. - name, line := "type-check", m.typeCheck(dir, sources) - if emit := m.emitBuild(dir, scripts); emit != nil { - name, line = "build", emit - } - start := m.now() - r := m.runIn(ctx, root, false, dir, nil, "node", line...) - check = m.now().Sub(start) - if err := ran(name, r, ""); err != nil { + tested := false + for _, tg := range targets { + scripts, err := packageScripts(tg.dir) + if err != nil { return nil, err } - res.Ran = append(res.Ran, name) - } - if hasScript && len(failed) == 0 { - start := m.now() - // node --test writes its TAP report to a file of the workspace (issue 1901), whatever the script prints itself - report := filepath.Join(root, ".mesh-node-test-"+level+".tap") - _ = os.Remove(report) - env := []string{"PATH=" + binDir(root) + ":" + pathOrDefault(), "npm_config_cache=" + filepath.Join(m.Toolchain.Paths.Cache, "npm"), - "NODE_OPTIONS=" + buildstep.NodeTestOptions(report)} - r := m.runIn(ctx, root, false, dir, env, "npm", scriptArgs(script)...) - run = m.now().Sub(start) - tap, _ := os.ReadFile(report) - _ = os.Remove(report) - if err := ran(script, r, string(tap)); err != nil { + sources, err := typeScriptFiles(tg.dir) + if err != nil { return nil, err } - res.Ran = append(res.Ran, script) + hasScript := scripts[script] != "" + if (level == "integration" && !hasScript) || (!hasScript && len(sources) == 0) { + continue + } + tested = true + if err := m.provide(root); err != nil { + return nil, fmt.Errorf("providing the toolchain's packages to the workspace: %w", err) + } + // a command that could not start is not a verdict: the step fails. One that ran and said no is. + ran := func(name string, r forgekey.Ran, report string) error { + if r.Err != "" { + return fmt.Errorf("%s could not be run: status %d %s: %s", name, r.Status, r.Err, firstLine(r.Stderr)) + } + if r.Status != 0 { + failed = append(failed, testFailures(tg.where, name, tg.dir, r, report)...) + } + return nil + } + if len(sources) > 0 { + // A package that builds (a build or prepare script, which the fetch's --ignore-scripts never runs) is emitted + // by tsc into the workspace, shut off like every run step, so its tests find what they import of its own output + // (issue 1869: mesh-sdk's tests import ../dist); any other is type-checked without emitting. + name, line := "type-check", m.typeCheck(tg.dir, sources) + if emit := m.emitBuild(tg.dir, scripts); emit != nil { + name, line = "build", emit + } + start := m.now() + r := m.runIn(ctx, root, false, tg.dir, nil, "node", line...) + check += m.now().Sub(start) + if err := ran(name, r, ""); err != nil { + return nil, err + } + res.Ran = appendOnce(res.Ran, name) + } + if hasScript && len(failed) == 0 { + start := m.now() + // node --test writes its TAP report to a file of the workspace (issue 1901), whatever the script prints itself + report := filepath.Join(root, ".mesh-node-test-"+level+".tap") + _ = os.Remove(report) + env := []string{"PATH=" + binDir(root) + ":" + pathOrDefault(), "npm_config_cache=" + filepath.Join(m.Toolchain.Paths.Cache, "npm"), + "NODE_OPTIONS=" + buildstep.NodeTestOptions(report)} + r := m.runIn(ctx, root, false, tg.dir, env, "npm", scriptArgs(script)...) + run += m.now().Sub(start) + tap, _ := os.ReadFile(report) + _ = os.Remove(report) + if err := ran(script, r, string(tap)); err != nil { + return nil, err + } + res.Ran = appendOnce(res.Ran, script) + } + } + if !tested { + res.Passed, res.NoTests = true, true + res.Why = "no tests: the package has no test script and no .ts files" + if level == "integration" { + res.Why = "no tests: the package has no test:integration script" + } + return res, nil } res.Failed = failed var why []string @@ -294,3 +306,56 @@ func typeScriptFiles(dir string) ([]string, error) { }) return out, err } + +// testTarget is a directory whose tests a level runs, and the name it is reported under. +type testTarget struct{ dir, where string } + +// testTargets are the directories a module's tests run in: each TypeScript package artifact's own path when every +// package the module declares names one (ADR 0634 §8: its package.json, its lock, its tests), else the module's +// directory. +func testTargets(dir, where string) ([]testTarget, error) { + whole := []testTarget{{dir, where}} + raw, err := os.ReadFile(filepath.Join(dir, "module.json")) + if err != nil { + return whole, nil + } + var mj struct { + Build struct { + Artifacts []artifact `json:"artifacts"` + } `json:"build"` + } + if json.Unmarshal(raw, &mj) != nil { + return whole, nil + } + var out []testTarget + seen := map[string]bool{} + for _, a := range mj.Build.Artifacts { + if a.Language != "typescript" { + continue + } + if a.Kind != "package" { + return whole, nil // a bundle is tested in its module's directory + } + at := a.packageDir(dir) + if at == "." { + return whole, nil + } + if !seen[at] { + seen[at] = true + out = append(out, testTarget{filepath.Join(dir, filepath.FromSlash(at)), path.Join(where, at)}) + } + } + if len(out) == 0 { + return whole, nil + } + return out, nil +} + +func appendOnce(list []string, s string) []string { + for _, x := range list { + if x == s { + return list + } + } + return append(list, s) +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/fetch.go b/modules/mesh-build-node/cmd/mesh-build-node/fetch.go index e758fab70..bd6343ecd 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/fetch.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/fetch.go @@ -17,6 +17,7 @@ import ( "fmt" "io/fs" "os" + "path" "path/filepath" "regexp" "sort" @@ -210,8 +211,8 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, group return res, nil } -// typeScriptModules are the directories of the tree whose module.json declares a typescript artifact, relative to -// it, in order. +// typeScriptModules are the directories of the tree that a typescript artifact of a module.json is built in, relative to +// it, in order: the module's directory for a bundle, the artifact's path for a package. func (m *Module) typeScriptModules(ctx context.Context, tree string) (dirs []string, packages map[string]bool, err error) { r := m.Run(ctx, m.baseEnv(), "git", "-C", tree, "ls-files", "--", "module.json", "*/module.json") if r.Status != 0 || r.Err != "" { @@ -237,13 +238,21 @@ func (m *Module) typeScriptModules(ctx context.Context, tree string) (dirs []str if json.Unmarshal(raw, &mj) != nil { continue // a module.json the build step will refuse by name, not a fetch's business } + // A bundle is built in its module's directory, a package in its own path (ADR 0634 §8: as at a repository's + // root, each package with its own lock); the first artifact to claim a directory decides it. for _, a := range mj.Build.Artifacts { - if a.Language == "typescript" { - d := filepath.ToSlash(filepath.Dir(l)) - dirs = append(dirs, d) - packages[d] = a.Kind == "package" - break + if a.Language != "typescript" { + continue } + d := filepath.ToSlash(filepath.Dir(l)) + if a.Kind == "package" { + d = path.Join(d, a.packageDir(filepath.Join(tree, filepath.FromSlash(d)))) + } + if _, seen := packages[d]; seen { + continue + } + dirs = append(dirs, d) + packages[d] = a.Kind == "package" } } sort.Strings(dirs) diff --git a/modules/mesh-build-node/cmd/mesh-build-node/package.go b/modules/mesh-build-node/cmd/mesh-build-node/package.go index 0663f180a..713d76c6c 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/package.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/package.go @@ -469,11 +469,13 @@ func fetchCommit(ctx context.Context, run forgekey.Runner, dir, remote, commit, // (publish) publishes it, from this module's own process with the registry's write credential, which no run step is // given: a version the registry lacks is published; the same packed file already there is done without a publish; // another packed file under that version fails `version taken:`. The reference is @, answered only when -// the registry holds this build's file; a candidate answers its digest alone. -func (m *Module) buildPackage(ctx context.Context, workspace, module, root, dir string, a artifact, publish bool) (any, error) { - rel := module - if rel == "" { - rel = "." +// the registry holds this build's file; a candidate answers its digest alone. dir is the package's directory, at its +// place in the module's. +func (m *Module) buildPackage(ctx context.Context, workspace, module, root, dir, at string, a artifact, publish bool) (any, error) { + // rel is the package's directory in the workspace, as the fetch installed it (the shims lie under it). + rel := path.Join(module, at) + if module == "" { + rel = path.Join(".", at) } p, err := m.buildAndPack(ctx, root, dir, rel, publish) if err != nil { diff --git a/modules/mesh-build-node/cmd/mesh-build-node/package_paths_test.go b/modules/mesh-build-node/cmd/mesh-build-node/package_paths_test.go new file mode 100644 index 000000000..98dd5f781 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/package_paths_test.go @@ -0,0 +1,93 @@ +package main + +import ( + "context" + "path/filepath" + "strings" + "testing" +) + +// A folder module (mesh-contracts' root, a domain folder) declares several package artifacts, each at its own path +// with its own package.json and package-lock.json and none at the module's directory (ADR 0634 section 8). +const folderManifest = `{"module":"c","build":{"artifacts":[` + + `{"name":"ts-a","kind":"package","language":"typescript","paths":["ts/seats/a"]},` + + `{"name":"ts-b","kind":"package","language":"typescript","paths":["ts/seats/b/"]}]}}` + +func folderFiles(prefix string) map[string]string { + files := map[string]string{prefix + "module.json": folderManifest} + for _, s := range []string{"a", "b"} { + files[prefix+"ts/seats/"+s+"/package.json"] = `{"name":"@x/` + s + `","version":"0.1.0","scripts":{"build":"tsc","test":"node --test"}}` + files[prefix+"ts/seats/"+s+"/package-lock.json"] = `{"lockfileVersion":3}` + files[prefix+"ts/seats/"+s+"/index.ts"] = "export const v = 1;\n" + } + return files +} + +// The fetch installs each package artifact at its path from the lock there, and finds no lock missing at the module. +func TestAFolderModulesPackagesAreInstalledEachAtItsPath(t *testing.T) { + m, f := fetchMachine(t, folderFiles("modules/c/"), nil) + out, err := m.fetch(context.Background(), fetchArgs) + if err != nil { + t.Fatal(err) + } + r := out.(FetchResult) + if len(r.Modules) != 2 || r.Modules[0] != "modules/c/ts/seats/a" || r.Modules[1] != "modules/c/ts/seats/b" || len(r.Installed) != 2 { + t.Errorf("result = %+v, want both package directories found and installed", r) + } + var prefixes []string + for _, c := range npmCalls(f) { + for i, a := range c.args { + if a == "--prefix" { + prefixes = append(prefixes, strings.TrimPrefix(c.args[i+1], filepath.Join(m.Work, r.Workspace, "tree"))) + } + } + } + if len(prefixes) != 2 || prefixes[0] != "/modules/c/ts/seats/a" || prefixes[1] != "/modules/c/ts/seats/b" { + t.Errorf("npm ci ran with --prefix %v, want each package's own directory", prefixes) + } +} + +// The build runs the package's script and pack in its own path, and the test level runs in each path. +func TestAFolderModulesPackageIsBuiltAndTestedInItsPath(t *testing.T) { + m, f, work := newMachine(t) + workspaceWith(t, work, folderFiles("")) + f.effect = func(c call) { + if c.name == "npm" && len(c.args) > 0 && c.args[0] == "pack" { + for i, a := range c.args { + if a == "--pack-destination" { + write(t, filepath.Join(c.args[i+1], "x.tgz"), "packed", 0o644) + } + } + } + } + if _, err := m.build(context.Background(), map[string]any{"workspace": "w1", "module": ".", "artifact": "ts-b"}); err != nil { + t.Fatal(err) + } + want := filepath.Join(work, "w1", "tree", "ts", "seats", "b") + built := 0 + for _, c := range f.calls { + if c.name == "npm" { + built++ + if c.dir != want { + t.Errorf("%s ran in %s, want %s", joined(c), c.dir, want) + } + } + } + if built != 2 { + t.Errorf("%d npm commands, want the build and the pack", built) + } + f.calls = nil + res, err := m.test(context.Background(), testArgs("unit")) + if err != nil { + t.Fatal(err) + } + tests := map[string]bool{} + for _, c := range f.calls { + if c.name == "npm" { + tests[c.dir] = true + } + } + if r := res.(*TestResult); !r.Passed || len(tests) != 2 || !tests[want] || !tests[filepath.Join(work, "w1", "tree", "ts", "seats", "a")] { + t.Errorf("unit level = %+v, ran tests in %v, want both package paths", r, tests) + } +} From 1feda1f533b820d1a2315fcaf4f51d2c3d107bd5 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 5 Nov 2026 21:17:21 +0100 Subject: [PATCH 2/4] mesh-build-go takes a zero-version requirement from the ask or the trunk, never from the pull request heads (ADR 0573 sections 2 and 4, issue 1966) A go.mod requirement of a forge module at v0.0.0-00010101000000-000000000000 names no commit; the fetch looked for the commit 000000000000 among the pull request heads and failed. It is now taken from the commit the ask names for the package, else from the head of its repository among the asked versions, else from the latest published version. Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- .../mesh-build-go/cmd/mesh-build-go/fetch.go | 25 +++++-- .../mesh-build-go/cmd/mesh-build-go/pins.go | 73 +++++++++++++++++++ .../cmd/mesh-build-go/pins_test.go | 68 +++++++++++++++++ .../cmd/mesh-build-go/versions.go | 4 + 4 files changed, 163 insertions(+), 7 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index 8f66a1a9b..d7e75b8f5 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -19,6 +19,7 @@ import ( "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" sdkstep "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/forgekey" ) @@ -177,7 +178,9 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, id str if taken[i], errs[i] = m.takeVersions(ctx, root, filepath.Join(tree, d), d, req.Packages, &pins); errs[i] != nil { return } - errs[i] = m.downloadModules(ctx, root, filepath.Join(tree, d), d, &pins) + var more []TakenVersion + more, errs[i] = m.downloadModules(ctx, root, filepath.Join(tree, d), d, &pins, req.Packages) + taken[i] = append(taken[i], more...) }() } wg.Wait() @@ -194,21 +197,29 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, id str // downloadModules downloads and verifies the Go modules of the directory dir (d in the tree). A pin of a forge module // that Go cannot fetch is looked for among the forge's pull request heads and handed to Go by a modfile (pins.go); a -// failure that names no such pin is the first failure. -func (m *Module) downloadModules(ctx context.Context, root, dir, d string, pins *sync.Mutex) error { +// failure that names no such pin is the first failure. A requirement at the zero version is taken from the asked +// versions (pins.go takeZeroPins). +func (m *Module) downloadModules(ctx context.Context, root, dir, d string, pins *sync.Mutex, asked delivery.PackageVersions) ([]TakenVersion, error) { + var taken []TakenVersion for { failure := m.downloadAndVerify(ctx, dir, d) if failure == nil { - return nil + return taken, nil + } + // A requirement at the zero version names no commit: it is taken from the asked versions or the trunk's latest. + took, err := m.takeZeroPins(ctx, root, dir, d, failure.Error(), asked, pins) + taken = append(taken, took...) + if err != nil { + return taken, err } pins.Lock() progress, err := m.resolvePins(ctx, root, dir, failure.Error()) pins.Unlock() if err != nil { - return err + return taken, err } - if !progress { - return failure + if !progress && len(took) == 0 { + return taken, failure } } } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/pins.go b/modules/mesh-build-go/cmd/mesh-build-go/pins.go index f51f52398..36105d5f6 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/pins.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/pins.go @@ -12,16 +12,23 @@ import ( "context" "errors" "fmt" + "maps" "os" "path/filepath" "regexp" + "slices" "strings" + "sync" + "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/forgekey" ) const forgePrefix = "git.novox.be/novox/" +// zeroShort is the 12 digits a requirement at the zero version v0.0.0-00010101000000-000000000000 ends in: no commit. +const zeroShort = "000000000000" + var pseudoVersion = regexp.MustCompile(`[-.][0-9]{14}-([0-9a-f]{12})$`) // pin is a requirement of a go.mod on a commit of a repository of the forge, by a pseudo-version. @@ -107,6 +114,9 @@ func (m *Module) resolvePins(ctx context.Context, root, dir, failure string) (bo } progress := false for _, p := range forgePins(string(raw)) { + if p.short == zeroShort { + continue // takeZeroPins': a requirement at the zero version names no commit to look for + } if !strings.Contains(failure, p.short) || strings.Contains(done, "replace "+p.module+" =>") { continue } @@ -241,3 +251,66 @@ func absoluteReplaces(body []byte, dir string) []byte { } return []byte(strings.Join(lines, "\n")) } + +// takeZeroPins takes, for each requirement of dir's go.mod at the zero version that a go failure names, the version it is +// built against (ADR 0573 §2, §4: whatever go.mod names): the commit the ask names for the package; else the head of +// the package's repository among the asked versions (the check's or try's members: another package of the same +// repository names it); else the latest published version, the trunk's head. It never looks for the commit +// 000000000000 among the pull request heads. It answers what it took; none when no zero requirement is named. +func (m *Module) takeZeroPins(ctx context.Context, root, dir, d, failure string, asked delivery.PackageVersions, pins *sync.Mutex) ([]TakenVersion, error) { + raw, err := os.ReadFile(filepath.Join(dir, "go.mod")) + if err != nil || !strings.Contains(failure, zeroShort) { + return nil, nil + } + resolved := map[string]string{} + if modfile, ok := m.pinModfile(dir); ok { + if b, err := os.ReadFile(modfile); err == nil { + resolved, _ = goModRequirements(string(b)) + for module := range resolved { + if strings.Contains(string(b), "replace "+module+" =>") { + resolved[module] = "replaced" + } + } + } + } + var taken []TakenVersion + for _, p := range forgePins(string(raw)) { + if p.short != zeroShort { + continue + } + if v := resolved[p.module]; v != "" && !strings.HasSuffix(v, zeroShort) { + continue // taken before + } + named := "v0.0.0-00010101000000-" + zeroShort + tv := TakenVersion{Dir: d, Package: p.module, Asked: askedFor(asked, p), Named: named} + why := "the ask names it" + if tv.Asked == "" { + tv.Asked, why = "latest", "the ask names no commit of its repository, so the trunk's latest published version" + } else if _, ok := asked[p.module]; !ok { + why = "the head of its repository among the asked versions" + } + if err := m.takeVersion(ctx, root, dir, &tv, pins); err != nil { + return taken, fmt.Errorf("the requirement of %s at the zero version: %w", p.module, err) + } + tv.Said = strings.TrimPrefix(tv.Said+"; "+why, "; ") + if m.Say != nil { + m.Say("%s: %s names no commit; built at %s (%s)", d, p.module, tv.Built, why) + } + taken = append(taken, tv) + } + return taken, nil +} + +// askedFor is the commit the ask names for the package p: its own, else that of the first package of the same repository +// in order (the members' heads are by repository), else none. +func askedFor(asked delivery.PackageVersions, p pin) string { + if c := asked[p.module]; c != "" { + return c + } + for _, pkg := range slices.Sorted(maps.Keys(asked)) { + if pkg == forgePrefix+p.repo || strings.HasPrefix(pkg, forgePrefix+p.repo+"/") { + return asked[pkg] + } + } + return "" +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go b/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go index 48829ce5a..85e28c646 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go @@ -290,3 +290,71 @@ func TestARelativeReplaceStillResolvesInTheTemporaryModfile(t *testing.T) { t.Fatalf("go mod verify in the pins directory: %v\n%s", err, out) } } + +// A go.mod requirement at the zero version v0.0.0-00010101000000-000000000000 names no commit (ADR 0573 §2, §4: whatever +// go.mod names): it is taken from the commit the ask names for the package, else from the head of its repository among +// the asked versions, else from the trunk's latest published version, and never looked for as the commit 000000000000 +// among the pull request heads. +func TestARequirementAtTheZeroVersionIsTakenFromTheAskOrTheTrunk(t *testing.T) { + const sdk = "git.novox.be/novox/mesh-sdk/go" + zeroMod := "module fixture/member\n\ngo 1.26\n\nrequire (\n\t" + sdk + " v0.0.0-00010101000000-" + zeroShort + "\n)\n" + trunk := strings.Repeat("a", 40) + for _, c := range []struct { + name string + packages map[string]any + download string // what go mod download -json answers + want string // the version asked of go + head bool // the asked commit is on no trunk: only a pull request head carries it + }{ + {name: "the ask names the package", packages: map[string]any{sdk: trunk}, want: sdk + "@" + trunk, + download: `{"Version":"v0.1.15-0.20261101000000-aaaaaaaaaaaa","Sum":"h1:zip="}`}, + {name: "the ask names another package of its repository", packages: map[string]any{"git.novox.be/novox/mesh-sdk/other": trunk}, want: sdk + "@" + trunk, + download: `{"Version":"v0.1.15-0.20261101000000-aaaaaaaaaaaa","Sum":"h1:zip="}`}, + {name: "the ask names the head of a pull request", packages: map[string]any{sdk: commitSHA}, want: sdk + "@" + commitSHA, head: true}, + {name: "the ask names nothing of its repository", packages: map[string]any{"git.novox.be/novox/mesh-tools/go": trunk}, want: sdk + "@latest", + download: `{"Version":"v0.1.20-0.20261104000000-bbbbbbbbbbbb","Sum":"h1:zip="}`}, + {name: "no ask", want: sdk + "@latest", download: `{"Version":"v0.1.20-0.20261104000000-bbbbbbbbbbbb","Sum":"h1:zip="}`}, + } { + t.Run(c.name, func(t *testing.T) { + h, states := newPinHarness(t, zeroMod, false, pullList) + base := h.m.Run + h.m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + line := name + " " + strings.Join(args, " ") + switch { + case name == "go" && strings.Contains(line, " mod download -json "): + h.mu.Lock() + h.cmds = append(h.cmds, cmdRecord{env: env, line: line}) + h.mu.Unlock() + if c.head { + return forgekey.Ran{Status: 1, Stdout: `{"Error":"invalid version: unknown revision ` + commitSHA[:12] + `"}`} + } + return forgekey.Ran{Stdout: c.download} + case name == "go" && strings.Contains(line, " mod download") && !strings.Contains(strings.Join(env, " "), "-modfile="): + return forgekey.Ran{Status: 1, Stderr: "go: " + sdk + "@v0.0.0-00010101000000-" + zeroShort + ": invalid version: unknown revision " + zeroShort + "\n"} + } + return base(ctx, env, name, args...) + } + args := map[string]any{"repository": "novox/mesh-delivery", "commit": commitSHA} + if c.packages != nil { + args["packages"] = c.packages + } + if _, err := fetchTool(t, h.m)(args); err != nil { + t.Fatal(err) + } + st := states.waitEnd(t) + if st.State != delivery.StepPassed { + t.Fatalf("the step ended %s: %s", st.State, st.Reason) + } + if !h.ran(" mod download -json " + c.want) { + t.Errorf("go was not asked for %s:\n%s", c.want, strings.Join(h.f.lines(), "\n")) + } + // The pull request heads are listed only to place an asked head, never to look for the commit of zeros. + if listed := h.ran("ls-remote"); listed != c.head { + t.Errorf("the pull request heads were listed: %v, want %v", listed, c.head) + } + if h.ran("pinned/mesh-sdk@" + zeroShort) { + t.Error("a repository was placed at the commit of zeros") + } + }) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/versions.go b/modules/mesh-build-go/cmd/mesh-build-go/versions.go index 726319b66..104a511bb 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/versions.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/versions.go @@ -105,6 +105,10 @@ func (m *Module) takeVersion(ctx context.Context, root, dir string, tv *TakenVer // Not a commit Go can fetch: the head of a group's pull request (ADR 0573 §4), placed from the forge's heads. // Whether it is one is the forge's fact (placePin), whatever go said. failure := strings.TrimSpace(info.Error + "\n" + r.Err + "\n" + r.Stderr) + if !commitOK.MatchString(tv.Asked) { + // "latest" (the trunk's head) is no commit of a pull request: there is nothing to place. + return fmt.Errorf("the %s version of %s could not be taken (a download failure that may be retaken): %s", tv.Asked, tv.Package, causeLine(failure)) + } repo, sub, _ := strings.Cut(strings.TrimPrefix(tv.Package, forgePrefix), "/") p := pin{module: tv.Package, repo: repo, sub: sub, short: tv.Asked[:12]} pins.Lock() From cbea2470d225ab00c6ece043e506d12fccb91952 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 5 Nov 2026 21:55:04 +0100 Subject: [PATCH 3/4] mesh-vault starts its passes once Serve holds the runtime's channel: started earlier every watch is refused and waits for the minute's pass (issue 774) Checked by TestTheVaultWholeRotatesANodesLoginWithAStandInEngine: the race it met is what this closes. Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- modules/mesh-vault/cmd/mesh-vault/main.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/modules/mesh-vault/cmd/mesh-vault/main.go b/modules/mesh-vault/cmd/mesh-vault/main.go index 3333b3c0b..63ba03a1b 100644 --- a/modules/mesh-vault/cmd/mesh-vault/main.go +++ b/modules/mesh-vault/cmd/mesh-vault/main.go @@ -302,9 +302,20 @@ func readPlanRaw(key string) ([]byte, error) { // assignmentKeys is every key of the controller's assignments mirror, read the same way. func assignmentKeys() ([]string, error) { return stdio.State(AssignmentsRead).Keys() } +// Checked by TestTheVaultWholeRotatesANodesLoginWithAStandInEngine: the race it met (a pass before Serve, every watch +// refused, the rotation waiting for the minute's pass) is what this closes. +// afterServed runs f once served is closed, that is once stdio.Serve holds the runtime's channel: every function of the +// bus refuses before that ("before Serve"), and a watch refused then waits for the minute's pass. +func afterServed(served <-chan struct{}, f func()) { + go func() { + <-served + f() + }() +} + func main() { env := build() - go func() { + afterServed(stdio.Served(), func() { tick := time.NewTicker(time.Minute) defer tick.Stop() // At start, every covered account it holds whose hash is not of its current generation (ADR 0547 clause 9). @@ -361,7 +372,7 @@ func main() { } <-tick.C } - }() + }) if err := stdio.Serve("", tools(env)); err != nil { say("%v", err) os.Exit(1) From 7a5eef32e7721fbf32d68a560aa930c25fb3300c Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 5 Nov 2026 23:34:46 +0100 Subject: [PATCH 4/4] A zero-version requirement of the checked repository's own module is built from its fetched tree, and a failed latest says what failed (issue 1921) Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- .../mesh-build-go/cmd/mesh-build-go/fetch.go | 6 +- .../mesh-build-go/cmd/mesh-build-go/pins.go | 80 ++++++++++++++++++- .../cmd/mesh-build-go/pins_test.go | 41 ++++++++++ .../cmd/mesh-build-go/versions.go | 2 +- 4 files changed, 123 insertions(+), 6 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index d7e75b8f5..b733df269 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -179,7 +179,7 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, id str return } var more []TakenVersion - more, errs[i] = m.downloadModules(ctx, root, filepath.Join(tree, d), d, &pins, req.Packages) + more, errs[i] = m.downloadModules(ctx, root, repository, filepath.Join(tree, d), d, &pins, req.Packages) taken[i] = append(taken[i], more...) }() } @@ -199,7 +199,7 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, id str // that Go cannot fetch is looked for among the forge's pull request heads and handed to Go by a modfile (pins.go); a // failure that names no such pin is the first failure. A requirement at the zero version is taken from the asked // versions (pins.go takeZeroPins). -func (m *Module) downloadModules(ctx context.Context, root, dir, d string, pins *sync.Mutex, asked delivery.PackageVersions) ([]TakenVersion, error) { +func (m *Module) downloadModules(ctx context.Context, root, repository, dir, d string, pins *sync.Mutex, asked delivery.PackageVersions) ([]TakenVersion, error) { var taken []TakenVersion for { failure := m.downloadAndVerify(ctx, dir, d) @@ -207,7 +207,7 @@ func (m *Module) downloadModules(ctx context.Context, root, dir, d string, pins return taken, nil } // A requirement at the zero version names no commit: it is taken from the asked versions or the trunk's latest. - took, err := m.takeZeroPins(ctx, root, dir, d, failure.Error(), asked, pins) + took, err := m.takeZeroPins(ctx, root, repository, dir, d, failure.Error(), asked, pins) taken = append(taken, took...) if err != nil { return taken, err diff --git a/modules/mesh-build-go/cmd/mesh-build-go/pins.go b/modules/mesh-build-go/cmd/mesh-build-go/pins.go index 36105d5f6..aec9e034b 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/pins.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/pins.go @@ -12,6 +12,7 @@ import ( "context" "errors" "fmt" + "io/fs" "maps" "os" "path/filepath" @@ -257,7 +258,7 @@ func absoluteReplaces(body []byte, dir string) []byte { // the package's repository among the asked versions (the check's or try's members: another package of the same // repository names it); else the latest published version, the trunk's head. It never looks for the commit // 000000000000 among the pull request heads. It answers what it took; none when no zero requirement is named. -func (m *Module) takeZeroPins(ctx context.Context, root, dir, d, failure string, asked delivery.PackageVersions, pins *sync.Mutex) ([]TakenVersion, error) { +func (m *Module) takeZeroPins(ctx context.Context, root, repository, dir, d, failure string, asked delivery.PackageVersions, pins *sync.Mutex) ([]TakenVersion, error) { raw, err := os.ReadFile(filepath.Join(dir, "go.mod")) if err != nil || !strings.Contains(failure, zeroShort) { return nil, nil @@ -282,10 +283,27 @@ func (m *Module) takeZeroPins(ctx context.Context, root, dir, d, failure string, continue // taken before } named := "v0.0.0-00010101000000-" + zeroShort + if "novox/"+p.repo == repository { + // A module of the checked repository itself: it is in the fetched tree and nowhere else, since nothing of the + // repository is published before its merge. Every module of the tree goes in, as one may require another. + own, err := m.useOwnModules(dir, root, p.repo) + if err != nil { + return taken, fmt.Errorf("the requirement of %s at the zero version: %w", p.module, err) + } + if !slices.Contains(own, p.module) { + return taken, fmt.Errorf("the requirement of %s at the zero version: no module of that path is in the fetched tree of %s, and nothing of a repository is published before its merge", p.module, repository) + } + tv := TakenVersion{Dir: d, Package: p.module, Named: named, Built: "the fetched tree", Said: "a module of the checked repository: built from its tree"} + if m.Say != nil { + m.Say("%s: %s names no commit; built from the fetched tree of %s", d, p.module, repository) + } + taken = append(taken, tv) + continue + } tv := TakenVersion{Dir: d, Package: p.module, Asked: askedFor(asked, p), Named: named} why := "the ask names it" if tv.Asked == "" { - tv.Asked, why = "latest", "the ask names no commit of its repository, so the trunk's latest published version" + tv.Asked, why = "latest", "the ask names no commit of its repository, so its latest published version" } else if _, ok := asked[p.module]; !ok { why = "the head of its repository among the asked versions" } @@ -314,3 +332,61 @@ func askedFor(asked delivery.PackageVersions, p pin) string { } return "" } + +// useOwnModules replaces, in dir's temporary modfile, every module of the fetched tree whose path lies in the repository +// repo by its directory in the tree (once), and answers the paths of those modules. A replace of a module dir does not +// require is harmless to Go; one module of a repository requires another of it at the zero version. +func (m *Module) useOwnModules(dir, root, repo string) ([]string, error) { + tree := filepath.Join(root, "tree") + prefix := forgePrefix + repo + type found struct{ module, where string } + var mods []found + err := filepath.WalkDir(tree, func(path string, e fs.DirEntry, err error) error { + if err != nil { + return err + } + if e.IsDir() { + if n := e.Name(); n == ".git" || n == "vendor" || n == "node_modules" { + return filepath.SkipDir + } + return nil + } + if e.Name() != "go.mod" { + return nil + } + raw, err := os.ReadFile(path) + if err != nil { + return err + } + for _, line := range strings.Split(string(raw), "\n") { + if f := strings.Fields(line); len(f) == 2 && f[0] == "module" { + if mod := strings.Trim(f[1], `"`); (mod == prefix || strings.HasPrefix(mod, prefix+"/")) && filepath.Dir(path) != dir { + mods = append(mods, found{mod, filepath.Dir(path)}) + } + break + } + } + return nil + }) + if err != nil { + return nil, err + } + modfile, _ := m.pinModfile(dir) + done := "" + if modfile != "" { + if b, err := os.ReadFile(modfile); err == nil { + done = string(b) + } + } + var paths []string + for _, f := range mods { + paths = append(paths, f.module) + if strings.Contains(done, "replace "+f.module+" =>") { + continue + } + if err := m.replaceInModfile(dir, f.module, f.where); err != nil { + return nil, err + } + } + return paths, nil +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go b/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go index 85e28c646..529294287 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go @@ -310,6 +310,8 @@ func TestARequirementAtTheZeroVersionIsTakenFromTheAskOrTheTrunk(t *testing.T) { download: `{"Version":"v0.1.15-0.20261101000000-aaaaaaaaaaaa","Sum":"h1:zip="}`}, {name: "the ask names another package of its repository", packages: map[string]any{"git.novox.be/novox/mesh-sdk/other": trunk}, want: sdk + "@" + trunk, download: `{"Version":"v0.1.15-0.20261101000000-aaaaaaaaaaaa","Sum":"h1:zip="}`}, + {name: "the ask names the root of its repository, which no package of is published", packages: map[string]any{"git.novox.be/novox/mesh-sdk": trunk}, want: sdk + "@" + trunk, + download: `{"Version":"v0.1.15-0.20261101000000-aaaaaaaaaaaa","Sum":"h1:zip="}`}, {name: "the ask names the head of a pull request", packages: map[string]any{sdk: commitSHA}, want: sdk + "@" + commitSHA, head: true}, {name: "the ask names nothing of its repository", packages: map[string]any{"git.novox.be/novox/mesh-tools/go": trunk}, want: sdk + "@latest", download: `{"Version":"v0.1.20-0.20261104000000-bbbbbbbbbbbb","Sum":"h1:zip="}`}, @@ -358,3 +360,42 @@ func TestARequirementAtTheZeroVersionIsTakenFromTheAskOrTheTrunk(t *testing.T) { }) } } + +// A requirement at the zero version of a module of the checked repository itself (a nested module not yet published, which +// another module of the repository requires) is built from the fetched tree, and never asked of Go or the forge (issue 1921). +func TestARequirementAtTheZeroVersionOfTheCheckedRepositorysOwnModuleIsTakenFromItsTree(t *testing.T) { + const own = "git.novox.be/novox/mesh-delivery/nested/go" + zeroMod := "module fixture/member\n\ngo 1.26\n\nrequire (\n\t" + own + " v0.0.0-00010101000000-" + zeroShort + "\n)\n" + h, states := newPinHarness(t, zeroMod, false, pullList) + base := h.m.Run + h.m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + line := name + " " + strings.Join(args, " ") + if name == "go" && strings.Contains(line, " mod download") && !strings.Contains(strings.Join(env, " "), "-modfile=") { + trees, _ := filepath.Glob(filepath.Join(h.m.Work, "*", "tree")) + for _, tree := range trees { + os.MkdirAll(filepath.Join(tree, "nested", "go"), 0o755) + os.WriteFile(filepath.Join(tree, "nested", "go", "go.mod"), []byte("module "+own+"\n\ngo 1.26\n"), 0o644) + } + return forgekey.Ran{Status: 1, Stderr: "go: " + own + "@v0.0.0-00010101000000-" + zeroShort + ": invalid version: unknown revision " + zeroShort + "\n"} + } + return base(ctx, env, name, args...) + } + if _, err := fetchTool(t, h.m)(map[string]any{"repository": "novox/mesh-delivery", "commit": commitSHA}); err != nil { + t.Fatal(err) + } + st := states.waitEnd(t) + if st.State != delivery.StepPassed { + t.Fatalf("the step ended %s: %s", st.State, st.Reason) + } + modfiles, _ := filepath.Glob(filepath.Join(h.m.Work, "*", "pins", "*", "go.mod")) + if len(modfiles) != 1 { + t.Fatalf("modfiles = %v", modfiles) + } + body, _ := os.ReadFile(modfiles[0]) + if !strings.Contains(string(body), "replace "+own+" => "+filepath.Join(filepath.Dir(filepath.Dir(filepath.Dir(modfiles[0]))), "tree", "nested", "go")) { + t.Errorf("the module of the tree was not replaced into the modfile:\n%s", body) + } + if h.ran(" mod download -json ") || h.ran("ls-remote") { + t.Error("a module of the checked repository was asked of Go or looked for among the pull request heads") + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/versions.go b/modules/mesh-build-go/cmd/mesh-build-go/versions.go index 104a511bb..eb00ca5e2 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/versions.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/versions.go @@ -107,7 +107,7 @@ func (m *Module) takeVersion(ctx context.Context, root, dir string, tv *TakenVer failure := strings.TrimSpace(info.Error + "\n" + r.Err + "\n" + r.Stderr) if !commitOK.MatchString(tv.Asked) { // "latest" (the trunk's head) is no commit of a pull request: there is nothing to place. - return fmt.Errorf("the %s version of %s could not be taken (a download failure that may be retaken): %s", tv.Asked, tv.Package, causeLine(failure)) + return fmt.Errorf("go cannot fetch the latest version of %s, and the ask names no commit of its repository to build it at: either no version of it is published, or the forge did not answer: %s", tv.Package, causeLine(failure)) } repo, sub, _ := strings.Cut(strings.TrimPrefix(tv.Package, forgePrefix), "/") p := pin{module: tv.Package, repo: repo, sub: sub, short: tv.Asked[:12]}