diff --git a/checks/deps/go.mod b/checks/deps/go.mod new file mode 100644 index 00000000..898f815d --- /dev/null +++ b/checks/deps/go.mod @@ -0,0 +1,3 @@ +module git.novox.be/novox/mesh-catalog/checks/deps + +go 1.22 diff --git a/checks/deps/main.go b/checks/deps/main.go new file mode 100644 index 00000000..dae84e1e --- /dev/null +++ b/checks/deps/main.go @@ -0,0 +1,255 @@ +// Command deps makes a module's dependencies available to the catalogue's check, or says why it cannot +// (novox/hq issue 385). +// +// A module's tests run only when its dependencies can be fetched. The build seat clones what a check +// needs with the forge credential and forgets that credential before the check runs, so a module pinning +// an SDK commit the seat's module cache does not hold cannot fetch it from the forge. The SDK is cloned +// beside the check all the same (MESH_CHECK_BESIDE, mesh-controller's besideRefs): every requirement of the +// mesh's own forge is taken from the clone of its repository there, at the commit or the tag its version +// names, through a go.mod of its own (-modfile) — the module's own files are not changed, and no forge +// credential is added. +// +// Usage: deps -beside -work . It prints the go.mod to build the module with, or +// nothing when the module fetches what it needs; when it cannot, it prints why and exits 1, and the check +// fails: a module whose tests could not run is a failure, never a note. +package main + +import ( + "archive/tar" + "bytes" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" +) + +// forge is the prefix of the Go modules of the mesh's own forge: [/]. +const forge = "git.novox.be/novox/" + +func main() { + beside := flag.String("beside", "", "where the repositories beside the check are cloned (MESH_CHECK_BESIDE)") + work := flag.String("work", os.TempDir(), "where the go.mod and the sources taken from beside are written") + flag.Parse() + if flag.NArg() != 1 { + fmt.Println("usage: deps -beside -work ") + os.Exit(2) + } + modfile, err := resolve(flag.Arg(0), *beside, *work) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + if modfile != "" { + fmt.Println(modfile) + } +} + +// resolve is the go.mod the module at dir builds with: "" when its own fetches everything, else one +// beside it in work that takes every requirement of the mesh's forge from its clone beside the check. +func resolve(dir, beside, work string) (string, error) { + out, err := goIn(dir, "mod", "download") + if err == nil { + return "", nil + } + fetching := lastLine(out) + required, err := forgeRequirements(dir) + if err != nil { + return "", err + } + if len(required) == 0 { + return "", fmt.Errorf("its dependencies cannot be fetched: %s", fetching) + } + if beside == "" { + return "", fmt.Errorf("its dependencies cannot be fetched (%s), and nothing was cloned beside the check to "+ + "take %s from", fetching, repositories(required)) + } + into, err := os.MkdirTemp(work, filepath.Base(dir)+"-") + if err != nil { + return "", err + } + modfile := filepath.Join(into, "go.mod") + if err := copyFile(filepath.Join(dir, "go.mod"), modfile); err != nil { + return "", err + } + if err := copyFile(filepath.Join(dir, "go.sum"), filepath.Join(into, "go.sum")); err != nil && + !errors.Is(err, os.ErrNotExist) { + return "", err + } + for _, r := range required { + source, err := export(beside, r, into) + if err != nil { + return "", fmt.Errorf("its dependencies cannot be fetched (%s), and %s@%s cannot be taken from the "+ + "clone beside the check: %w", fetching, r.Path, r.Version, err) + } + if out, err := goIn(dir, "mod", "edit", "-replace", r.Path+"@"+r.Version+"="+source, modfile); err != nil { + return "", fmt.Errorf("%s@%s cannot be taken from the clone beside the check: %s", r.Path, r.Version, + lastLine(out)) + } + } + if out, err := goIn(dir, "mod", "download", "-modfile="+modfile); err != nil { + return "", fmt.Errorf("its dependencies cannot be fetched even with %s taken from the clone beside the "+ + "check: %s", repositories(required), lastLine(out)) + } + return modfile, nil +} + +// requirement is one module the module requires. +type requirement struct { + Path string + Version string +} + +// forgeRequirements is every requirement of the module at dir that comes from the mesh's own forge. +func forgeRequirements(dir string) ([]requirement, error) { + out, err := goIn(dir, "mod", "edit", "-json") + if err != nil { + return nil, fmt.Errorf("its go.mod cannot be read: %s", lastLine(out)) + } + var mod struct{ Require []requirement } + if err := json.Unmarshal(out, &mod); err != nil { + return nil, fmt.Errorf("its go.mod cannot be read: %w", err) + } + var on []requirement + for _, r := range mod.Require { + if strings.HasPrefix(r.Path, forge) { + on = append(on, r) + } + } + return on, nil +} + +// repositoryOf splits a module path of the forge into its repository and the directory in it. +func repositoryOf(path string) (string, string) { + repo, sub, _ := strings.Cut(strings.TrimPrefix(path, forge), "/") + return repo, sub +} + +func repositories(rs []requirement) string { + var names []string + for _, r := range rs { + repo, _ := repositoryOf(r.Path) + names = append(names, repo) + } + return strings.Join(names, ", ") +} + +// pseudo is a pseudo-version's commit: v0.1.14-0.20261012082113-9747f3e1b1ac → 9747f3e1b1ac. +var pseudo = regexp.MustCompile(`[-.]\d{14}-([0-9a-f]{12})$`) + +// refOf is the ref a version names in its repository: a pseudo-version's commit, else the tag a module in +// a directory is released under (/). +func refOf(sub, version string) string { + version = strings.TrimSuffix(version, "+incompatible") + if m := pseudo.FindStringSubmatch(version); m != nil { + return m[1] + } + if sub != "" { + return sub + "/" + version + } + return version +} + +// export writes the module r names, at its version, from its repository's clone beside the check into +// into, and answers its directory. +func export(beside string, r requirement, into string) (string, error) { + repo, sub := repositoryOf(r.Path) + clone := filepath.Join(beside, repo) + if _, err := os.Stat(filepath.Join(clone, ".git")); err != nil { + return "", fmt.Errorf("no clone of %s is beside the check", repo) + } + ref := refOf(sub, r.Version) + commit, err := exec.Command("git", "-C", clone, "rev-parse", "--verify", "--quiet", ref+"^{commit}").Output() + if err != nil { + return "", fmt.Errorf("the clone of %s holds no %s (a commit or tag not pushed to the forge?)", repo, ref) + } + at := strings.TrimSpace(string(commit)) + args := []string{"-C", clone, "archive", "--format=tar", at} + if sub != "" { + args = append(args, "--", sub) + } + var stderr bytes.Buffer + archive := exec.Command("git", args...) + archive.Stderr = &stderr + raw, err := archive.Output() + if err != nil { + return "", fmt.Errorf("git archive of %s at %s: %v %s", repo, ref, err, strings.TrimSpace(stderr.String())) + } + root := filepath.Join(into, repo+"@"+at[:12]) + if err := untar(raw, root); err != nil { + return "", err + } + source := filepath.Join(root, filepath.FromSlash(sub)) + if _, err := os.Stat(filepath.Join(source, "go.mod")); err != nil { + return "", fmt.Errorf("%s at %s has no go.mod in %q", repo, ref, sub) + } + return source, nil +} + +// untar writes a tar's directories and regular files under root. +func untar(raw []byte, root string) error { + r := tar.NewReader(bytes.NewReader(raw)) + for { + h, err := r.Next() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + name := filepath.Clean(filepath.FromSlash(h.Name)) + if name == "." || strings.HasPrefix(name, "..") || filepath.IsAbs(name) { + continue + } + path := filepath.Join(root, name) + switch h.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(path, 0o755); err != nil { + return err + } + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + body, err := io.ReadAll(r) + if err != nil { + return err + } + if err := os.WriteFile(path, body, 0o644); err != nil { + return err + } + } + } +} + +func copyFile(from, to string) error { + raw, err := os.ReadFile(from) + if err != nil { + return err + } + return os.WriteFile(to, raw, 0o644) +} + +// goIn runs the go command in dir. +func goIn(dir string, args ...string) ([]byte, error) { + cmd := exec.Command("go", args...) + cmd.Dir = dir + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return append(out, stderr.Bytes()...), err + } + return out, nil +} + +// lastLine is the last line a command said, which is where go says what failed. +func lastLine(out []byte) string { + lines := strings.Split(strings.TrimSpace(string(out)), "\n") + return strings.TrimSpace(lines[len(lines)-1]) +} diff --git a/checks/deps/main_test.go b/checks/deps/main_test.go new file mode 100644 index 00000000..3328fbc4 --- /dev/null +++ b/checks/deps/main_test.go @@ -0,0 +1,177 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// offline is a Go that fetches nothing: what the build seat is once the forge credential is gone, for a pin +// its module cache does not hold (novox/hq issue 385). +func offline(t *testing.T) { + t.Helper() + t.Setenv("GOPROXY", "off") + t.Setenv("GOFLAGS", "") + t.Setenv("GOWORK", "off") + // Not GOPRIVATE: a private module is fetched directly by git, around GOPROXY, with whatever credential + // the machine running the test holds. + t.Setenv("GOPRIVATE", "") + t.Setenv("GONOPROXY", "") + t.Setenv("GOSUMDB", "off") + t.Setenv("GOMODCACHE", t.TempDir()) +} + +func git(t *testing.T, dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@t", "GIT_COMMITTER_NAME=t", + "GIT_COMMITTER_EMAIL=t@t", "GIT_AUTHOR_DATE=2026-10-12T08:21:13Z", "GIT_COMMITTER_DATE=2026-10-12T08:21:13Z") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } + return strings.TrimSpace(string(out)) +} + +func write(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +// sdkBeside is the SDK's repository cloned beside a check, its Go module under go/, one commit tagged +// go/v0.1.7. It answers the beside root and the commit. +func sdkBeside(t *testing.T) (string, string) { + t.Helper() + beside := t.TempDir() + repo := filepath.Join(beside, "mesh-sdk") + write(t, filepath.Join(repo, "go", "go.mod"), "module git.novox.be/novox/mesh-sdk/go\n\ngo 1.22\n") + write(t, filepath.Join(repo, "go", "sdk", "sdk.go"), "package sdk\n\nfunc Hello() string { return \"hello\" }\n") + write(t, filepath.Join(repo, "README.md"), "the SDK\n") + git(t, repo, "init", "--quiet") + git(t, repo, "add", ".") + git(t, repo, "commit", "--quiet", "-m", "sdk") + git(t, repo, "tag", "go/v0.1.7") + return beside, git(t, repo, "rev-parse", "HEAD") +} + +// moduleRequiring is a catalogue module whose code and test use the SDK at the version given. +func moduleRequiring(t *testing.T, version string) string { + t.Helper() + dir := filepath.Join(t.TempDir(), "messenger") + write(t, filepath.Join(dir, "go.mod"), "module git.novox.be/novox/mesh-catalog/modules/messenger\n\ngo 1.22\n\n"+ + "require git.novox.be/novox/mesh-sdk/go "+version+"\n") + write(t, filepath.Join(dir, "main.go"), "package main\n\nimport \"git.novox.be/novox/mesh-sdk/go/sdk\"\n\n"+ + "func main() { println(sdk.Hello()) }\n") + return dir +} + +func builds(t *testing.T, dir, modfile string) { + t.Helper() + args := []string{"vet"} + if modfile != "" { + args = append(args, "-modfile="+modfile) + } + cmd := exec.Command("go", append(args, "./...")...) + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("the module does not build with %q: %v\n%s", modfile, err, out) + } +} + +// A module that needs nothing it cannot fetch is left as it is: no other go.mod. +func TestAModuleThatFetchesItsDependenciesNeedsNothingBeside(t *testing.T) { + offline(t) + dir := filepath.Join(t.TempDir(), "plain") + write(t, filepath.Join(dir, "go.mod"), "module example.com/plain\n\ngo 1.22\n") + write(t, filepath.Join(dir, "main.go"), "package main\n\nfunc main() {}\n") + modfile, err := resolve(dir, "", t.TempDir()) + if err != nil || modfile != "" { + t.Fatalf("resolve = %q, %v; want nothing", modfile, err) + } +} + +// The case of issue 385: the module pins an SDK commit the build seat cannot fetch. It is taken from the +// SDK's clone beside the check, and the module builds against it. +func TestAPinnedSDKCommitIsTakenFromTheCloneBeside(t *testing.T) { + offline(t) + beside, commit := sdkBeside(t) + dir := moduleRequiring(t, "v0.1.14-0.20261012082113-"+commit[:12]) + modfile, err := resolve(dir, beside, t.TempDir()) + if err != nil { + t.Fatalf("resolve: %v", err) + } + if modfile == "" { + t.Fatal("resolve said the module fetches its dependencies, with nothing to fetch them from") + } + builds(t, dir, modfile) + if raw, _ := os.ReadFile(filepath.Join(dir, "go.mod")); strings.Contains(string(raw), "replace") { + t.Fatalf("the module's own go.mod was changed:\n%s", raw) + } +} + +// A released SDK is found by the tag it is released under: go/. +func TestATaggedSDKIsTakenFromTheCloneBeside(t *testing.T) { + offline(t) + beside, _ := sdkBeside(t) + dir := moduleRequiring(t, "v0.1.7") + modfile, err := resolve(dir, beside, t.TempDir()) + if err != nil { + t.Fatalf("resolve: %v", err) + } + builds(t, dir, modfile) +} + +// A commit the clone beside does not hold (a branch never pushed) cannot be tested, and says which. +func TestACommitTheCloneBesideDoesNotHoldIsSaid(t *testing.T) { + offline(t) + beside, _ := sdkBeside(t) + dir := moduleRequiring(t, "v0.1.14-0.20261010132341-4f3efc98e0c0") + _, err := resolve(dir, beside, t.TempDir()) + if err == nil { + t.Fatal("resolve found a commit nothing holds") + } + for _, want := range []string{"git.novox.be/novox/mesh-sdk/go@v0.1.14-0.20261010132341-4f3efc98e0c0", "4f3efc98e0c0"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the reason %q does not name %s", err, want) + } + } +} + +// With nothing cloned beside (a check by hand without the forge), the reason says that. +func TestNoCloneBesideIsSaid(t *testing.T) { + offline(t) + _, commit := sdkBeside(t) + dir := moduleRequiring(t, "v0.1.14-0.20261012082113-"+commit[:12]) + for _, beside := range []string{"", t.TempDir()} { + _, err := resolve(dir, beside, t.TempDir()) + if err == nil { + t.Fatalf("resolve with %q beside found what is not there", beside) + } + if !strings.Contains(err.Error(), "mesh-sdk") { + t.Errorf("the reason %q does not name the repository it needed", err) + } + } +} + +func TestTheRefOfAVersion(t *testing.T) { + cases := []struct{ sub, version, want string }{ + {"go", "v0.1.14-0.20261012082113-9747f3e1b1ac", "9747f3e1b1ac"}, + {"go", "v0.0.0-20261012082113-9747f3e1b1ac", "9747f3e1b1ac"}, + {"go", "v0.1.7", "go/v0.1.7"}, + {"", "v1.2.0", "v1.2.0"}, + {"", "v2.0.0+incompatible", "v2.0.0"}, + } + for _, c := range cases { + if got := refOf(c.sub, c.version); got != c.want { + t.Errorf("refOf(%q, %q) = %q, want %q", c.sub, c.version, got, c.want) + } + } +} diff --git a/merge-check-typescript.sh b/merge-check-typescript.sh index 4a66f9da..3b0ed856 100644 --- a/merge-check-typescript.sh +++ b/merge-check-typescript.sh @@ -9,14 +9,17 @@ # # 1. its dependencies are the toolchain's own (/app/node_modules: the mesh's SDK, Node's types and what the # runtime carries), copied beside it, never fetched: the check holds no credential for the package -# registry. A module that depends on a package the toolchain does not hold is said NOT CHECKED, by name; +# registry. A module that depends on a package the toolchain does not hold **fails the check**, by name: +# it could not be type-checked or tested, and what could not run is a failure, never a note (novox/hq +# issue 385); # 2. every source that is not a test is type-checked with the toolchain's compiler and the flags a build # compiles with — the same check the build makes, before the merge rather than in the walk; # 3. its tests (test/*.test.ts) run under Node's own runner with types stripped, as its own `npm test` does # after its build: the sources are compiled to dist/ and beside themselves first, so a test that imports # the compiled module, or a source that imports its sibling by its .js name, finds it. # -# Said, never passed silently: a module with no tests is said NOT TESTED on every run that touches it. +# Said, never passed silently: a module with no tests is said NOT TESTED on every run that touches it — +# nothing of it could fail. A module whose tests exist and could not run fails (novox/hq issue 385). # # By hand: MESH_CHECK_MODULES=gitea sh merge-check-typescript.sh, in the toolchain image (it reads # /app/node_modules), or with TOOLCHAIN_MODULES pointing at a node_modules that holds the same. @@ -54,7 +57,8 @@ for m in $touched; do require("fs").existsSync(process.argv[2] + "/" + n)); console.log(names.filter((n) => n !== "typescript" && !held(n)).join(" "));' "$copy/package.json" "$deps") if [ -n "$missing" ]; then - echo "NOT CHECKED: $d depends on $missing, which the toolchain does not hold and the check cannot fetch" + echo "FAILED: $d could not be checked — it depends on $missing, which the toolchain does not hold and the check cannot fetch (novox/hq issue 385)" + failed="$failed $m" continue fi mkdir -p "$copy/node_modules" diff --git a/merge-check.sh b/merge-check.sh index 1e6fcdb8..b2bddd63 100755 --- a/merge-check.sh +++ b/merge-check.sh @@ -14,8 +14,11 @@ # it shares a rule with refused (MESH_GATE is the controller the mesh runs) — and the count of # long-running resources without `health` held to the number in health-undeclared, which only goes down; # 2. the Go tests of every module the change touches that has them, under the race detector when the -# toolchain has a C compiler — and a module whose dependencies cannot be fetched here is said as not -# tested, never passed silently. A module written in TypeScript is type-checked and tested by +# toolchain has a C compiler. A requirement of the mesh's own forge the module cache does not hold (a +# pinned SDK commit) is taken from its repository's clone beside the check (checks/deps); a module whose +# dependencies still cannot be had **fails the check**, by name and why — its tests could not run, and a +# test that could not run is a failure, never a note (novox/hq issue 385). A module written in +# TypeScript is type-checked and tested by # merge-check-typescript.sh, declared above, in the TypeScript toolchain (novox/hq issues 302, 311); # 3. the words (novox/hq ADR 0244): no word the glossary retired for the tools, as copied in # retired-words, in any text a module's tools can show an agent — every string in its own code that is @@ -77,16 +80,26 @@ if [ -n "${MESH_CHECK_MODULES:-}" ]; then else touched=$(printf '%s\n' "${MESH_CHECK_CHANGED:-}" | tr ',' '\n' | sed -n 's#^modules/\([^/]*\)/.*#\1#p' | sort -u) fi +# A module's dependencies: fetched, or a forge requirement taken from its clone beside the check +# (MESH_CHECK_BESIDE); its tests run against them (checks/deps, novox/hq issue 385). The helper's own test +# runs first. +(cd checks/deps && go test -count=1 .) +deps=$(mktemp -d) +trap 'rm -rf "$deps"' EXIT +(cd checks/deps && go build -o "$deps/deps" .) +untestable="" for m in $touched; do [ -d "modules/$m" ] || continue if [ ! -f "modules/$m/go.mod" ]; then [ -f "modules/$m/package.json" ] && echo "modules/$m is TypeScript: merge-check-typescript.sh type-checks and tests it" continue fi - if ! (cd "modules/$m" && GOPRIVATE=git.novox.be go mod download >/dev/null 2>&1); then - echo "NOT TESTED: modules/$m — its dependencies cannot be fetched by the build seat" + if ! modfile=$(GOPRIVATE=git.novox.be "$deps/deps" -beside "${MESH_CHECK_BESIDE:-}" -work "$deps" "modules/$m"); then + echo "FAILED: modules/$m could not be tested — $modfile" + untestable="$untestable modules/$m" continue fi + [ -n "$modfile" ] && echo "modules/$m takes its forge dependencies from the clones beside the check" echo "testing modules/$m" unformatted=$(cd "modules/$m" && gofmt -l .) if [ -n "$unformatted" ]; then @@ -95,6 +108,10 @@ for m in $touched; do fi cgo=0 [ -n "$race" ] && cgo=1 - (cd "modules/$m" && CGO_ENABLED=$cgo GOPRIVATE=git.novox.be go vet ./... && - CGO_ENABLED=$cgo GOPRIVATE=git.novox.be go test $race -count=1 ./...) + (cd "modules/$m" && CGO_ENABLED=$cgo GOPRIVATE=git.novox.be go vet ${modfile:+"-modfile=$modfile"} ./... && + CGO_ENABLED=$cgo GOPRIVATE=git.novox.be go test ${modfile:+"-modfile=$modfile"} $race -count=1 ./...) done +if [ -n "$untestable" ]; then + echo "the check fails: the tests of$untestable could not run (novox/hq issue 385)" + exit 1 +fi