diff --git a/modules/mesh-build-go/buildstep/askrun.go b/modules/mesh-build-go/buildstep/askrun.go index 2a0836b2..60bc385e 100644 --- a/modules/mesh-build-go/buildstep/askrun.go +++ b/modules/mesh-build-go/buildstep/askrun.go @@ -25,6 +25,10 @@ type StepFunc func(ctx context.Context, args map[string]any) (any, error) // AskSteps are the step tools a build module runs an ask with. Copy is the image module's alone; Digest the Go module's. type AskSteps struct { Fetch, Build, Copy, Digest, Test StepFunc + // Imports reads the mesh packages the built programs import (the Go module's alone, ADR 0593 §2): it answers + // {"imports": [...]}, sorted, from the workspace the build ran in. A build that built an artifact asks it once, at + // its end; nil reports none. + Imports StepFunc // Toolchain is the toolchain the module builds with now; a build ask naming another is declined (Declining). Nil checks nothing // (an image or an archive names none). Toolchain func(ctx context.Context) (delivery.ToolchainUsed, error) @@ -190,6 +194,26 @@ func (s AskSteps) build(ctx context.Context, job delivery.BuildJob) (any, error) } answer.Artifacts = append(answer.Artifacts, delivery.AskBuilt{Artifact: a.Artifact, Digest: built.Digest, Reference: built.Reference}) } + if s.Imports != nil && f.Workspace != "" { + out, err := s.Imports(ctx, map[string]any{"workspace": f.Workspace, "module": job.Path}) + if err != nil { + return nil, fmt.Errorf("the imports of %s: %w", job.Module, err) + } + r, err := raw(out) + if err != nil { + return nil, err + } + var read struct { + Imports []string `json:"imports"` + } + if err := json.Unmarshal(r, &read); err != nil { + return nil, fmt.Errorf("the imports of %s answered what is not a list: %s", job.Module, r) + } + if err := delivery.CheckImports(read.Imports); err != nil { + return nil, err + } + answer.Imports = read.Imports + } return answer, nil } diff --git a/modules/mesh-build-go/buildstep/askrun_test.go b/modules/mesh-build-go/buildstep/askrun_test.go index 7fe442c3..b0fce397 100644 --- a/modules/mesh-build-go/buildstep/askrun_test.go +++ b/modules/mesh-build-go/buildstep/askrun_test.go @@ -76,6 +76,12 @@ func TestRunAskBuilds(t *testing.T) { steps := r.steps() var released []string steps.Release = func(ws string) error { released = append(released, ws); return nil } + // ADR 0593 §2: a Go module's build ends by saying which mesh packages its programs import. + steps.Imports = func(ctx context.Context, args map[string]any) (any, error) { + b, _ := json.Marshal(args) + r.calls = append(r.calls, "imports "+string(b)) + return map[string]any{"imports": []string{"git.novox.be/novox/mesh-sdk/go/announce", "git.novox.be/novox/mesh-sdk/go/delivery"}}, nil + } // Issue 1585: the ask's package versions reach the fetch, and the ask's own fetch arguments stay as they were. a := ask(delivery.BuildJob{Module: "m", Path: "modules/m", Publish: true, Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}, @@ -92,7 +98,8 @@ func TestRunAskBuilds(t *testing.T) { want := `copy {"artifact":"base","from":"registry/b@sha256:00","module":"m"} fetch {"commit":"` + runCommit + `","packages":{"git.novox.be/novox/mesh-sdk/go":"` + runCommit + `"},"repository":"novox/x"} build {"artifact":"bundle","module":"modules/m","publish":true,"workspace":"/w/1"} -build {"artifact":"lib","module":"modules/m","publish":true,"workspace":"/w/1"}` +build {"artifact":"lib","module":"modules/m","publish":true,"workspace":"/w/1"} +imports {"module":"modules/m","workspace":"/w/1"}` if got != want { t.Fatalf("calls:\n%s\nwant:\n%s", got, want) } @@ -103,6 +110,9 @@ build {"artifact":"lib","module":"modules/m","publish":true,"workspace":"/w/1"}` if len(answer.Artifacts) != 3 || answer.Artifacts[1].Reference != "artifact-store://bundle" { t.Fatalf("answer %+v", answer) } + if !slices.Equal(answer.Imports, []string{"git.novox.be/novox/mesh-sdk/go/announce", "git.novox.be/novox/mesh-sdk/go/delivery"}) { + t.Errorf("the answer's imports are %v", answer.Imports) + } } // A build ask for another toolchain is declined (ADR 0584 clause 2), so the node sits out and the asker puts it again. diff --git a/modules/mesh-build-go/buildstep/buildcpu/buildcpu_agree_test.go b/modules/mesh-build-go/buildstep/buildcpu/buildcpu_agree_test.go new file mode 100644 index 00000000..bf61988d --- /dev/null +++ b/modules/mesh-build-go/buildstep/buildcpu/buildcpu_agree_test.go @@ -0,0 +1,14 @@ +package buildcpu + +import ( + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// The share's names the composer gives are the build modules' (catalogue writes them too, so it does not import this). +func TestTheShareNamesAgreeWithTheCatalogues(t *testing.T) { + if ShareSetting != catalogue.BuildCPUShareSetting || ShareWord != catalogue.BuildCPUShareWord || DefaultPercent != catalogue.BuildCPUDefaultPercent { + t.Fatal("buildcpu's share names differ from catalogue's") + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/asks.go b/modules/mesh-build-go/cmd/mesh-build-go/asks.go index 0465c45d..5579cc6c 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/asks.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/asks.go @@ -37,6 +37,7 @@ func (m *Module) askSteps(steps []StepTool) buildstep.AskSteps { } return buildstep.AskSteps{ Fetch: byName["build_go_fetch"], Build: byName["build_go_build"], Digest: byName["build_go_digest"], Test: byName["build_go_test"], + Imports: byName[ImportsTool], Toolchain: func(ctx context.Context) (delivery.ToolchainUsed, error) { installed, err := m.Probe(ctx) if err != nil { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/imports.go b/modules/mesh-build-go/cmd/mesh-build-go/imports.go new file mode 100644 index 00000000..a0995d10 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/imports.go @@ -0,0 +1,115 @@ +package main + +// The imports step (novox/hq ADR 0593 §2): at a build's end the module says which mesh packages the module's Go programs +// import, so the indexer can take the fingerprint over those packages' content and no other. It is `go list -deps` of +// each Go bundle artifact's program, the tests left out, kept to the packages of the mesh's forge that lie outside the +// module's own directory (a sibling package of the module's own repository counts, a package under the module itself +// comes with its tree). + +import ( + "context" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// ImportsResult is what the imports step answers; a BuildAnswer carries Imports as it is. +type ImportsResult struct { + Imports []string `json:"imports"` +} + +func (m *Module) importsTool() StepTool { + return StepTool{ + Name: ImportsTool, + Description: "Read the mesh packages the Go programs of a module of a fetched workspace import: `go list -deps` of each Go " + + "bundle artifact, without tests, kept to the forge's packages outside the module's own directory, sorted and each once. " + + "It is what a build says at its end for the fingerprint (ADR 0593 §2). Read-only.", + Input: map[string]any{ + "type": "object", + "properties": map[string]any{ + "workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"}, + "module": map[string]any{"type": "string", "description": "the module's directory in the workspace, . for the repository's root"}, + }, + "required": []string{"workspace", "module"}, + }, + Run: m.imports, + } +} + +func (m *Module) imports(ctx context.Context, args map[string]any) (any, error) { + workspace, _ := args["workspace"].(string) + module, _ := args["module"].(string) + if !workspaceOK.MatchString(workspace) { + return nil, fmt.Errorf("%q is not a workspace id", workspace) + } + tree := filepath.Join(m.Work, workspace, "tree") + dir := filepath.Join(tree, filepath.FromSlash(module)) + if rel, err := filepath.Rel(tree, dir); err != nil || strings.HasPrefix(rel, "..") { + return nil, fmt.Errorf("the module directory %q lies outside the workspace", module) + } + raw, err := os.ReadFile(filepath.Join(dir, "module.json")) + if err != nil { + return nil, fmt.Errorf("no module in %q of workspace %s: %w", module, workspace, err) + } + manifest, err := catalogue.Decode(raw) + if err != nil { + return nil, fmt.Errorf("the manifest in %q: %w", module, err) + } + run, err := m.stepRunner(workspace, false) + if err != nil { + return nil, err + } + return ModuleImports(ctx, run, m.buildEnv(dir), dir, manifest) +} + +// ModuleImports are the mesh packages the Go bundle artifacts of manifest, in the module directory dir, import. +func ModuleImports(ctx context.Context, run forgekey.Runner, env []string, dir string, manifest catalogue.Manifest) (ImportsResult, error) { + seen := map[string]bool{} + if manifest.Build != nil { + for _, a := range manifest.Build.Artifacts { + if a.Kind != "bundle" || a.Language != "go" || strings.TrimSpace(a.From) == "" { + continue + } + r := run(ctx, env, "go", "-C", dir, "list", "-deps", "-f", "{{.ImportPath}} {{.Dir}}", "./"+strings.Trim(a.From, "./")) + if r.Status != 0 || r.Err != "" { + return ImportsResult{}, fmt.Errorf("go list of %s: %s", a.Name, causeLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout)) + } + for _, l := range strings.Split(r.Stdout, "\n") { + pkg, where, ok := strings.Cut(strings.TrimSpace(l), " ") + if !ok || !meshPackage(pkg) || ownedBy(dir, where) { + continue + } + seen[pkg] = true + } + } + } + out := []string{} + for p := range seen { + out = append(out, p) + } + slices.Sort(out) + if err := delivery.CheckImports(out); err != nil { + return ImportsResult{}, err + } + return ImportsResult{Imports: out}, nil +} + +func meshPackage(pkg string) bool { + _, _, ok := delivery.PackageRepository(pkg) + return ok +} + +// ownedBy says the package directory where lies under the module directory dir and is not a vendored copy. +func ownedBy(dir, where string) bool { + rel, err := filepath.Rel(dir, where) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return false + } + return rel != "vendor" && !strings.HasPrefix(rel, "vendor"+string(filepath.Separator)) +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/imports_test.go b/modules/mesh-build-go/cmd/mesh-build-go/imports_test.go new file mode 100644 index 00000000..249f0587 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/imports_test.go @@ -0,0 +1,63 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// ADR 0593 §2 (C3): a fixture whose program imports announce, a sibling package of its own repository and a package of its +// own directory reports the first two; a test's import never appears (go list is run without -test), and the standard +// library, a package of another host and the module's own packages are left out. +func TestModuleImportsAreTheMeshPackagesOutsideTheModule(t *testing.T) { + dir := "/w/ws1/tree/modules/demo" + var line string + run := func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + line = name + " " + strings.Join(args, " ") + return forgekey.Ran{Stdout: strings.Join([]string{ + "fmt /usr/lib/go/src/fmt", + "git.novox.be/novox/mesh-sdk/go/announce /cache/mesh-sdk@v1/go/announce", + "git.novox.be/novox/mesh-sdk/go/announce /cache/mesh-sdk@v1/go/announce", + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep /w/ws1/tree/modules/mesh-build-go/buildstep", + "git.novox.be/novox/mesh-catalog/modules/demo/internal/own " + dir + "/internal/own", + "git.novox.be/novox/mesh-sdk/go/vendored " + dir + "/vendor/git.novox.be/novox/mesh-sdk/go/vendored", + "github.com/nats-io/nats.go /cache/nats.go", + }, "\n")} + } + m, err := catalogue.Decode([]byte(`{"module":"demo","version":"1","build":{"artifacts":[` + goBundle + `,{"name":"files","kind":"archive","from":"files"}]}}`)) + if err != nil { + t.Fatal(err) + } + got, err := ModuleImports(context.Background(), run, nil, dir, m) + if err != nil { + t.Fatal(err) + } + want := []string{"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep", "git.novox.be/novox/mesh-sdk/go/announce", "git.novox.be/novox/mesh-sdk/go/vendored"} + if !slices.Equal(got.Imports, want) { + t.Errorf("imports %v, want %v", got.Imports, want) + } + if strings.Contains(line, "-test") || !strings.Contains(line, "list -deps") || !strings.HasSuffix(line, "./cmd/demo-tools") { + t.Errorf("go ran as %q: it lists the program's dependencies, without tests", line) + } +} + +// The step tool reads the manifest of the fetched workspace and answers the same list. +func TestImportsStepReadsTheWorkspace(t *testing.T) { + m, _, _ := buildModule(t, manifestWith(goBundle), "x") + if err := os.WriteFile(filepath.Join(m.Work, "ws1", "tree", "modules", "demo", "go.mod"), []byte("module demo\n"), 0o644); err != nil { + t.Fatal(err) + } + out, err := m.imports(context.Background(), map[string]any{"workspace": "ws1", "module": "modules/demo"}) + if err != nil { + t.Fatal(err) + } + if got := out.(ImportsResult).Imports; len(got) != 0 { + t.Errorf("a fake compiler's go list names no package, and the answer was %v", got) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/tools.go b/modules/mesh-build-go/cmd/mesh-build-go/tools.go index 9a3552de..dbc20eab 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/tools.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/tools.go @@ -11,6 +11,8 @@ import ( const ( StatusTool = "build_go_status" HealthTool = "build_go_health" + // ImportsTool reads the mesh packages a module's programs import (ADR 0593 §2). + ImportsTool = "build_go_imports" ) // StepTool is one step tool of the module: fetch, test, digest or build. Long is a step that answers at once and @@ -25,7 +27,7 @@ type StepTool struct { // StepTools are the module's step tools. Each step's issue adds its own. func (m *Module) StepTools() []StepTool { - return []StepTool{m.fetchTool(), m.testTool(), m.digestTool(), m.buildTool()} + return []StepTool{m.fetchTool(), m.testTool(), m.digestTool(), m.buildTool(), m.importsTool()} } // Tools is every tool the module serves: its step tools behind the guard, its status and its health. diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index 88fbba9b..2c6289c6 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -48,6 +48,7 @@ "build_go_test", "build_go_digest", "build_go_build", + "build_go_imports", "build_go_release", "build_go_step", "build_go_steps",