diff --git a/internal/builder/builder.go b/internal/builder/builder.go index fe6c754..c8a16ae 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -461,6 +461,12 @@ func compile(ctx context.Context, run Runner, tree string, chain Toolchain, // rather than configurable: a module that could move this would be describing its own build. const within = "/app/modules/module" + // **Its own output directory, because a module may be several languages at once.** One module + // is one piece of software and can still carry a daemon in one language, tools in another and + // a package in a third (ADR 0040). Compiling them all into one place would have them overwrite + // each other and then be packed together, so each bundle compiles and packs alone. + out := Out(a.Name) + invocation := []string{ "run", "--rm", "--volume", tree + ":" + within, @@ -468,15 +474,18 @@ func compile(ctx context.Context, run Runner, tree string, chain Toolchain, base, } invocation = append(invocation, chain.Compile...) + if chain.OutputFlag != "" { + invocation = append(invocation, chain.OutputFlag, out) + } // What to compile. Named by the module rather than discovered, so adding a file does not // silently change what a build produces. if len(a.Entrypoints) > 0 { - invocation = append(invocation, sourcesFor(a.Entrypoints)...) + invocation = append(invocation, sourcesFor(a.Entrypoints, out)...) } if _, err := run(ctx, tree, "docker", invocation...); err != nil { return "", err } - return filepath.Join(tree, chain.Output), nil + return filepath.Join(tree, out), nil } // sourcesFor turns compiled entrypoints back into what to compile. @@ -484,10 +493,14 @@ func compile(ctx context.Context, run Runner, tree string, chain Toolchain, // A module names what a tool host should LOAD — compiled paths under the bundle's root — because // that is the thing anything else needs to know. What to compile is the same list with the // language's own extension, which is the toolchain's business rather than the module's. -func sourcesFor(entrypoints []string) []string { - out := make([]string, 0, len(entrypoints)) +func sourcesFor(entrypoints []string, out string) []string { + sources := make([]string, 0, len(entrypoints)) for _, e := range entrypoints { - out = append(out, strings.TrimSuffix(e, filepath.Ext(e))+".ts") + // An entrypoint is named as it will be FOUND — a path inside the unpacked bundle — so the + // source is the same path with the output directory taken off the front and the language's + // own extension on the end. + at := strings.TrimPrefix(strings.TrimPrefix(e, out), "/") + sources = append(sources, strings.TrimSuffix(at, filepath.Ext(at))+".ts") } - return out + return sources } diff --git a/internal/builder/bundle_test.go b/internal/builder/bundle_test.go index 2f40ba9..0b8115a 100644 --- a/internal/builder/bundle_test.go +++ b/internal/builder/bundle_test.go @@ -10,7 +10,7 @@ import ( const aBundle = `{"module":"greeter","version":"1", "build":{"artifacts":[ - {"name":"code","kind":"bundle","language":"typescript","entrypoints":["dist/index.js"]}]}, + {"name":"code","kind":"bundle","language":"typescript","entrypoints":["index.js"]}]}, "resources":[ {"id":"files","type":"archive","path":"/opt/greeter","artifact":"code"}]}` @@ -21,15 +21,28 @@ type compiling struct{ *recorded } func (c compiling) run(ctx context.Context, dir, name string, args ...string) (string, error) { out, err := c.recorded.run(ctx, dir, name, args...) - if name == "docker" && len(args) > 0 && args[0] == "run" { - made := filepath.Join(dir, "dist") - if err := os.MkdirAll(made, 0o755); err != nil { - return "", err - } - if err := os.WriteFile(filepath.Join(made, "index.js"), []byte("console.log(1)"), 0o644); err != nil { - return "", err + if name != "docker" || len(args) == 0 || args[0] != "run" { + return out, err + } + // **Writes where it was TOLD to**, rather than to a fixed directory. A fake that always wrote + // to one place would pass whether or not the builder gave each artifact its own — which is the + // thing being tested. + where := "" + for i, a := range args { + if a == "--outDir" && i+1 < len(args) { + where = args[i+1] } } + if where == "" { + return out, err + } + made := filepath.Join(dir, where) + if err := os.MkdirAll(made, 0o755); err != nil { + return "", err + } + if err := os.WriteFile(filepath.Join(made, "index.js"), []byte("console.log(1)"), 0o644); err != nil { + return "", err + } return out, err } @@ -107,3 +120,51 @@ func TestABundleInAnUnknownLanguageIsRefused(t *testing.T) { t.Fatalf("the refusal does not say what would have worked: %v", err) } } + +// **One module, two bundles, and neither packs the other.** +// +// The case that matters for real modules: a module is one piece of software and may still carry a +// daemon in one language and tools in another (ADR 0040). An earlier version of this compiled +// every bundle into the toolchain's single output directory, so two of them would overwrite each +// other and then be packed together — one artifact containing both, twice. +func TestTwoBundlesInOneModuleArePackedSeparately(t *testing.T) { + const two = `{"module":"greeter","version":"1", + "build":{"artifacts":[ + {"name":"daemon","kind":"bundle","language":"typescript","entrypoints":["index.js"]}, + {"name":"tools","kind":"bundle","language":"typescript","entrypoints":["index.js"]}]}, + "resources":[ + {"id":"a","type":"archive","path":"/opt/greeter/daemon","artifact":"daemon"}, + {"id":"b","type":"archive","path":"/opt/greeter/tools","artifact":"tools"}]}` + + r, workspace := aRepository(t, two, map[string]string{"index.ts": "console.log(1)"}) + held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} + + got, err := Build(context.Background(), compiling{r}.run, r, + "https://forge.invalid/greeter.git", "", "", workspace, held) + if err != nil { + t.Fatalf("a module with two bundles did not build: %v", err) + } + + // Compiled into two different places. + var outputs []string + for _, line := range r.ran { + for _, part := range strings.Fields(line) { + if strings.HasPrefix(part, ".mesh-build/") { + outputs = append(outputs, part) + } + } + } + if len(outputs) != 2 || outputs[0] == outputs[1] { + t.Fatalf("two bundles did not get their own output directories: %v", outputs) + } + + // And published as two artifacts, each with its own digest. + if len(r.archives) != 2 { + t.Fatalf("expected two archives published, got %v", r.archives) + } + first, _ := got.Manifest.Resources[0]["digest"].(string) + second, _ := got.Manifest.Resources[1]["digest"].(string) + if first == "" || second == "" { + t.Fatalf("a bundle was not pinned: %v", got.Manifest.Resources) + } +} diff --git a/internal/builder/toolchain.go b/internal/builder/toolchain.go index 2d23d19..9d50213 100644 --- a/internal/builder/toolchain.go +++ b/internal/builder/toolchain.go @@ -26,13 +26,28 @@ type Toolchain struct { Base string // Artifact is which of that module's artifacts is the compiling one. Artifact string - // Compile is what runs inside it, relative to the module's own directory. The output goes to - // Output, which is what gets packed. + // Compile is what runs inside it, relative to the module's own directory. + // + // The output directory is appended by the builder, per artifact, because one module may + // declare several bundles — a daemon in one language, tools in another, a package in a third — + // and a toolchain with one fixed output would have them overwrite each other and then be + // packed together. Compile []string - // Output is the directory the compiled result lands in, relative to the module's directory. - Output string + // OutputFlag is how this compiler is told where to put its output. + OutputFlag string } +// Out is where one artifact's compiled output lands, inside the module's own directory. +// +// **Per artifact, never per toolchain.** A module is one piece of software and may still be +// written in several languages — a daemon in one, a tool in another, a package in a third (ADR +// 0040). Each bundle is compiled and packed alone, so what a machine unpacks is that artifact and +// nothing else. +// +// Under a directory named for the build rather than beside the source, so a pack never sweeps up +// the module's own working files, and two builds of one commit see the same tree. +func Out(artifact string) string { return ".mesh-build/" + artifact } + // toolchains is every language the mesh can build. // // **A closed list, and adding to it is a decision rather than a configuration.** Every language is @@ -53,9 +68,20 @@ var toolchains = []Toolchain{ Compile: []string{ "node", "/app/node_modules/typescript/bin/tsc", "--module", "NodeNext", "--moduleResolution", "NodeNext", - "--target", "ES2022", "--outDir", "dist", + "--target", "ES2022", }, - Output: "dist", + OutputFlag: "--outDir", + }, + { + Language: "python", + Base: "mesh-tools-python", + Artifact: "build", + // Nothing to compile: what a bundle needs is the module's own code and its dependencies + // resolved, so the "compile" is an install into the output directory. Named here rather + // than left implicit because a reader comparing two toolchains should be able to see what + // each actually does. + Compile: []string{"python", "-m", "pip", "install", "--no-compile", "--target"}, + OutputFlag: "", }, } diff --git a/internal/builder/toolchain_test.go b/internal/builder/toolchain_test.go index e387de3..ffb90ec 100644 --- a/internal/builder/toolchain_test.go +++ b/internal/builder/toolchain_test.go @@ -14,8 +14,8 @@ func TestADeclaredLanguageSelectsItsToolchain(t *testing.T) { if chain.Base == "" || chain.Artifact == "" { t.Fatalf("a toolchain names no base to compile in: %+v", chain) } - if len(chain.Compile) == 0 || chain.Output == "" { - t.Fatalf("a toolchain says nothing about how to compile or where output lands: %+v", chain) + if len(chain.Compile) == 0 { + t.Fatalf("a toolchain says nothing about how to compile: %+v", chain) } } @@ -54,3 +54,36 @@ func TestABundleMustSayWhatItIsWrittenIn(t *testing.T) { t.Fatalf("the refusal does not say a language is required: %v", err) } } + +// **One module, several languages, and each bundle packed alone.** +// +// A module is one piece of software (ADR 0040) and may still carry a daemon in one language, tools +// in another and a package in a third. Compiling them into one output directory would have them +// overwrite each other and then be packed together, so output is a property of the artifact rather +// than of the toolchain. +func TestTwoBundlesInOneModuleDoNotShareAnOutputDirectory(t *testing.T) { + first, second := Out("daemon"), Out("tools") + if first == second { + t.Fatalf("two artifacts compile into the same place (%q), so one would overwrite the "+ + "other and both would be packed together", first) + } + for _, out := range []string{first, second} { + if strings.HasPrefix(out, "/") || strings.Contains(out, "..") { + t.Fatalf("%q leaves the module's own directory", out) + } + } +} + +// And the mesh can say what it builds, which is what a refusal quotes. +func TestTheMeshSaysWhichLanguagesItBuilds(t *testing.T) { + spoken := Languages() + if len(spoken) < 2 { + t.Fatalf("only %v — this test exists to keep the multi-language path real rather than "+ + "theoretical", spoken) + } + for _, language := range spoken { + if _, err := ToolchainFor(language); err != nil { + t.Fatalf("%q is listed as buildable and has no toolchain: %v", language, err) + } + } +}