From 350aca05fb819f42eda901146f36b3bbfdf9ba27 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 18:27:33 +0100 Subject: [PATCH] =?UTF-8?q?The=20Go=20build=20module=20says=20at=20a=20bui?= =?UTF-8?q?ld's=20end=20which=20mesh=20packages=20its=20programs=20import,?= =?UTF-8?q?=20so=20a=20fingerprint=20can=20take=20their=20content=20(ADR?= =?UTF-8?q?=200593=20=C2=A72)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A build ask whose job built an artifact asks the new step build_go_imports once: `go list -deps` of each Go bundle's program, no tests, the forge's packages outside the module's own directory (a sibling package of the same repository counts), sorted and each once. The answer carries them as BuildAnswer.Imports. The step is also a tool, to read them by hand. Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- modules/mesh-build-go/buildstep/askrun.go | 24 ++++ .../mesh-build-go/buildstep/askrun_test.go | 12 +- .../buildstep/buildcpu/buildcpu_agree_test.go | 14 +++ .../mesh-build-go/cmd/mesh-build-go/asks.go | 1 + .../cmd/mesh-build-go/imports.go | 115 ++++++++++++++++++ .../cmd/mesh-build-go/imports_test.go | 63 ++++++++++ .../mesh-build-go/cmd/mesh-build-go/tools.go | 4 +- modules/mesh-build-go/module.json | 1 + 8 files changed, 232 insertions(+), 2 deletions(-) create mode 100644 modules/mesh-build-go/buildstep/buildcpu/buildcpu_agree_test.go create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/imports.go create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/imports_test.go diff --git a/modules/mesh-build-go/buildstep/askrun.go b/modules/mesh-build-go/buildstep/askrun.go index 2a0836b2..60bc385e 100644 --- a/modules/mesh-build-go/buildstep/askrun.go +++ b/modules/mesh-build-go/buildstep/askrun.go @@ -25,6 +25,10 @@ type StepFunc func(ctx context.Context, args map[string]any) (any, error) // AskSteps are the step tools a build module runs an ask with. Copy is the image module's alone; Digest the Go module's. type AskSteps struct { Fetch, Build, Copy, Digest, Test StepFunc + // Imports reads the mesh packages the built programs import (the Go module's alone, ADR 0593 §2): it answers + // {"imports": [...]}, sorted, from the workspace the build ran in. A build that built an artifact asks it once, at + // its end; nil reports none. + Imports StepFunc // Toolchain is the toolchain the module builds with now; a build ask naming another is declined (Declining). Nil checks nothing // (an image or an archive names none). Toolchain func(ctx context.Context) (delivery.ToolchainUsed, error) @@ -190,6 +194,26 @@ func (s AskSteps) build(ctx context.Context, job delivery.BuildJob) (any, error) } answer.Artifacts = append(answer.Artifacts, delivery.AskBuilt{Artifact: a.Artifact, Digest: built.Digest, Reference: built.Reference}) } + if s.Imports != nil && f.Workspace != "" { + out, err := s.Imports(ctx, map[string]any{"workspace": f.Workspace, "module": job.Path}) + if err != nil { + return nil, fmt.Errorf("the imports of %s: %w", job.Module, err) + } + r, err := raw(out) + if err != nil { + return nil, err + } + var read struct { + Imports []string `json:"imports"` + } + if err := json.Unmarshal(r, &read); err != nil { + return nil, fmt.Errorf("the imports of %s answered what is not a list: %s", job.Module, r) + } + if err := delivery.CheckImports(read.Imports); err != nil { + return nil, err + } + answer.Imports = read.Imports + } return answer, nil } diff --git a/modules/mesh-build-go/buildstep/askrun_test.go b/modules/mesh-build-go/buildstep/askrun_test.go index 7fe442c3..b0fce397 100644 --- a/modules/mesh-build-go/buildstep/askrun_test.go +++ b/modules/mesh-build-go/buildstep/askrun_test.go @@ -76,6 +76,12 @@ func TestRunAskBuilds(t *testing.T) { steps := r.steps() var released []string steps.Release = func(ws string) error { released = append(released, ws); return nil } + // ADR 0593 §2: a Go module's build ends by saying which mesh packages its programs import. + steps.Imports = func(ctx context.Context, args map[string]any) (any, error) { + b, _ := json.Marshal(args) + r.calls = append(r.calls, "imports "+string(b)) + return map[string]any{"imports": []string{"git.novox.be/novox/mesh-sdk/go/announce", "git.novox.be/novox/mesh-sdk/go/delivery"}}, nil + } // Issue 1585: the ask's package versions reach the fetch, and the ask's own fetch arguments stay as they were. a := ask(delivery.BuildJob{Module: "m", Path: "modules/m", Publish: true, Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}, @@ -92,7 +98,8 @@ func TestRunAskBuilds(t *testing.T) { want := `copy {"artifact":"base","from":"registry/b@sha256:00","module":"m"} fetch {"commit":"` + runCommit + `","packages":{"git.novox.be/novox/mesh-sdk/go":"` + runCommit + `"},"repository":"novox/x"} build {"artifact":"bundle","module":"modules/m","publish":true,"workspace":"/w/1"} -build {"artifact":"lib","module":"modules/m","publish":true,"workspace":"/w/1"}` +build {"artifact":"lib","module":"modules/m","publish":true,"workspace":"/w/1"} +imports {"module":"modules/m","workspace":"/w/1"}` if got != want { t.Fatalf("calls:\n%s\nwant:\n%s", got, want) } @@ -103,6 +110,9 @@ build {"artifact":"lib","module":"modules/m","publish":true,"workspace":"/w/1"}` if len(answer.Artifacts) != 3 || answer.Artifacts[1].Reference != "artifact-store://bundle" { t.Fatalf("answer %+v", answer) } + if !slices.Equal(answer.Imports, []string{"git.novox.be/novox/mesh-sdk/go/announce", "git.novox.be/novox/mesh-sdk/go/delivery"}) { + t.Errorf("the answer's imports are %v", answer.Imports) + } } // A build ask for another toolchain is declined (ADR 0584 clause 2), so the node sits out and the asker puts it again. diff --git a/modules/mesh-build-go/buildstep/buildcpu/buildcpu_agree_test.go b/modules/mesh-build-go/buildstep/buildcpu/buildcpu_agree_test.go new file mode 100644 index 00000000..bf61988d --- /dev/null +++ b/modules/mesh-build-go/buildstep/buildcpu/buildcpu_agree_test.go @@ -0,0 +1,14 @@ +package buildcpu + +import ( + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// The share's names the composer gives are the build modules' (catalogue writes them too, so it does not import this). +func TestTheShareNamesAgreeWithTheCatalogues(t *testing.T) { + if ShareSetting != catalogue.BuildCPUShareSetting || ShareWord != catalogue.BuildCPUShareWord || DefaultPercent != catalogue.BuildCPUDefaultPercent { + t.Fatal("buildcpu's share names differ from catalogue's") + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/asks.go b/modules/mesh-build-go/cmd/mesh-build-go/asks.go index 0465c45d..5579cc6c 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/asks.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/asks.go @@ -37,6 +37,7 @@ func (m *Module) askSteps(steps []StepTool) buildstep.AskSteps { } return buildstep.AskSteps{ Fetch: byName["build_go_fetch"], Build: byName["build_go_build"], Digest: byName["build_go_digest"], Test: byName["build_go_test"], + Imports: byName[ImportsTool], Toolchain: func(ctx context.Context) (delivery.ToolchainUsed, error) { installed, err := m.Probe(ctx) if err != nil { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/imports.go b/modules/mesh-build-go/cmd/mesh-build-go/imports.go new file mode 100644 index 00000000..a0995d10 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/imports.go @@ -0,0 +1,115 @@ +package main + +// The imports step (novox/hq ADR 0593 §2): at a build's end the module says which mesh packages the module's Go programs +// import, so the indexer can take the fingerprint over those packages' content and no other. It is `go list -deps` of +// each Go bundle artifact's program, the tests left out, kept to the packages of the mesh's forge that lie outside the +// module's own directory (a sibling package of the module's own repository counts, a package under the module itself +// comes with its tree). + +import ( + "context" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// ImportsResult is what the imports step answers; a BuildAnswer carries Imports as it is. +type ImportsResult struct { + Imports []string `json:"imports"` +} + +func (m *Module) importsTool() StepTool { + return StepTool{ + Name: ImportsTool, + Description: "Read the mesh packages the Go programs of a module of a fetched workspace import: `go list -deps` of each Go " + + "bundle artifact, without tests, kept to the forge's packages outside the module's own directory, sorted and each once. " + + "It is what a build says at its end for the fingerprint (ADR 0593 §2). Read-only.", + Input: map[string]any{ + "type": "object", + "properties": map[string]any{ + "workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"}, + "module": map[string]any{"type": "string", "description": "the module's directory in the workspace, . for the repository's root"}, + }, + "required": []string{"workspace", "module"}, + }, + Run: m.imports, + } +} + +func (m *Module) imports(ctx context.Context, args map[string]any) (any, error) { + workspace, _ := args["workspace"].(string) + module, _ := args["module"].(string) + if !workspaceOK.MatchString(workspace) { + return nil, fmt.Errorf("%q is not a workspace id", workspace) + } + tree := filepath.Join(m.Work, workspace, "tree") + dir := filepath.Join(tree, filepath.FromSlash(module)) + if rel, err := filepath.Rel(tree, dir); err != nil || strings.HasPrefix(rel, "..") { + return nil, fmt.Errorf("the module directory %q lies outside the workspace", module) + } + raw, err := os.ReadFile(filepath.Join(dir, "module.json")) + if err != nil { + return nil, fmt.Errorf("no module in %q of workspace %s: %w", module, workspace, err) + } + manifest, err := catalogue.Decode(raw) + if err != nil { + return nil, fmt.Errorf("the manifest in %q: %w", module, err) + } + run, err := m.stepRunner(workspace, false) + if err != nil { + return nil, err + } + return ModuleImports(ctx, run, m.buildEnv(dir), dir, manifest) +} + +// ModuleImports are the mesh packages the Go bundle artifacts of manifest, in the module directory dir, import. +func ModuleImports(ctx context.Context, run forgekey.Runner, env []string, dir string, manifest catalogue.Manifest) (ImportsResult, error) { + seen := map[string]bool{} + if manifest.Build != nil { + for _, a := range manifest.Build.Artifacts { + if a.Kind != "bundle" || a.Language != "go" || strings.TrimSpace(a.From) == "" { + continue + } + r := run(ctx, env, "go", "-C", dir, "list", "-deps", "-f", "{{.ImportPath}} {{.Dir}}", "./"+strings.Trim(a.From, "./")) + if r.Status != 0 || r.Err != "" { + return ImportsResult{}, fmt.Errorf("go list of %s: %s", a.Name, causeLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout)) + } + for _, l := range strings.Split(r.Stdout, "\n") { + pkg, where, ok := strings.Cut(strings.TrimSpace(l), " ") + if !ok || !meshPackage(pkg) || ownedBy(dir, where) { + continue + } + seen[pkg] = true + } + } + } + out := []string{} + for p := range seen { + out = append(out, p) + } + slices.Sort(out) + if err := delivery.CheckImports(out); err != nil { + return ImportsResult{}, err + } + return ImportsResult{Imports: out}, nil +} + +func meshPackage(pkg string) bool { + _, _, ok := delivery.PackageRepository(pkg) + return ok +} + +// ownedBy says the package directory where lies under the module directory dir and is not a vendored copy. +func ownedBy(dir, where string) bool { + rel, err := filepath.Rel(dir, where) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return false + } + return rel != "vendor" && !strings.HasPrefix(rel, "vendor"+string(filepath.Separator)) +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/imports_test.go b/modules/mesh-build-go/cmd/mesh-build-go/imports_test.go new file mode 100644 index 00000000..249f0587 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/imports_test.go @@ -0,0 +1,63 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// ADR 0593 §2 (C3): a fixture whose program imports announce, a sibling package of its own repository and a package of its +// own directory reports the first two; a test's import never appears (go list is run without -test), and the standard +// library, a package of another host and the module's own packages are left out. +func TestModuleImportsAreTheMeshPackagesOutsideTheModule(t *testing.T) { + dir := "/w/ws1/tree/modules/demo" + var line string + run := func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + line = name + " " + strings.Join(args, " ") + return forgekey.Ran{Stdout: strings.Join([]string{ + "fmt /usr/lib/go/src/fmt", + "git.novox.be/novox/mesh-sdk/go/announce /cache/mesh-sdk@v1/go/announce", + "git.novox.be/novox/mesh-sdk/go/announce /cache/mesh-sdk@v1/go/announce", + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep /w/ws1/tree/modules/mesh-build-go/buildstep", + "git.novox.be/novox/mesh-catalog/modules/demo/internal/own " + dir + "/internal/own", + "git.novox.be/novox/mesh-sdk/go/vendored " + dir + "/vendor/git.novox.be/novox/mesh-sdk/go/vendored", + "github.com/nats-io/nats.go /cache/nats.go", + }, "\n")} + } + m, err := catalogue.Decode([]byte(`{"module":"demo","version":"1","build":{"artifacts":[` + goBundle + `,{"name":"files","kind":"archive","from":"files"}]}}`)) + if err != nil { + t.Fatal(err) + } + got, err := ModuleImports(context.Background(), run, nil, dir, m) + if err != nil { + t.Fatal(err) + } + want := []string{"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep", "git.novox.be/novox/mesh-sdk/go/announce", "git.novox.be/novox/mesh-sdk/go/vendored"} + if !slices.Equal(got.Imports, want) { + t.Errorf("imports %v, want %v", got.Imports, want) + } + if strings.Contains(line, "-test") || !strings.Contains(line, "list -deps") || !strings.HasSuffix(line, "./cmd/demo-tools") { + t.Errorf("go ran as %q: it lists the program's dependencies, without tests", line) + } +} + +// The step tool reads the manifest of the fetched workspace and answers the same list. +func TestImportsStepReadsTheWorkspace(t *testing.T) { + m, _, _ := buildModule(t, manifestWith(goBundle), "x") + if err := os.WriteFile(filepath.Join(m.Work, "ws1", "tree", "modules", "demo", "go.mod"), []byte("module demo\n"), 0o644); err != nil { + t.Fatal(err) + } + out, err := m.imports(context.Background(), map[string]any{"workspace": "ws1", "module": "modules/demo"}) + if err != nil { + t.Fatal(err) + } + if got := out.(ImportsResult).Imports; len(got) != 0 { + t.Errorf("a fake compiler's go list names no package, and the answer was %v", got) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/tools.go b/modules/mesh-build-go/cmd/mesh-build-go/tools.go index 9a3552de..dbc20eab 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/tools.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/tools.go @@ -11,6 +11,8 @@ import ( const ( StatusTool = "build_go_status" HealthTool = "build_go_health" + // ImportsTool reads the mesh packages a module's programs import (ADR 0593 §2). + ImportsTool = "build_go_imports" ) // StepTool is one step tool of the module: fetch, test, digest or build. Long is a step that answers at once and @@ -25,7 +27,7 @@ type StepTool struct { // StepTools are the module's step tools. Each step's issue adds its own. func (m *Module) StepTools() []StepTool { - return []StepTool{m.fetchTool(), m.testTool(), m.digestTool(), m.buildTool()} + return []StepTool{m.fetchTool(), m.testTool(), m.digestTool(), m.buildTool(), m.importsTool()} } // Tools is every tool the module serves: its step tools behind the guard, its status and its health. diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index 88fbba9b..2c6289c6 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -48,6 +48,7 @@ "build_go_test", "build_go_digest", "build_go_build", + "build_go_imports", "build_go_release", "build_go_step", "build_go_steps",