One module, several languages, each bundle packed alone

A module is one piece of software and may still carry a daemon in one language,
tools in another and a package in a third. The first cut compiled every bundle
into the toolchain's single output directory, so two of them would have
overwritten each other and then been packed together — one artifact containing
both, published twice.

So output is a property of the artifact, not of the toolchain, and the toolchain
says how it is told where to write rather than where it writes. Under a directory
named for the build rather than beside the source, so a pack never sweeps up the
module's own working files.

A second toolchain is declared so the multi-language path is exercised rather
than asserted — a list with one entry cannot fail the way a list with four will.

And the fake compiler in the tests now writes where it was TOLD to. One that
always wrote to a fixed place would have passed whether or not each artifact got
its own directory, which is the whole of what these tests are for.

Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx

internal/builder/builder.go

@@ -461,6 +461,12 @@ func compile(ctx context.Context, run Runner, tree string, chain Toolchain,
 	// rather than configurable: a module that could move this would be describing its own build.
 	const within = "/app/modules/module"
 
+	// **Its own output directory, because a module may be several languages at once.** One module
+	// is one piece of software and can still carry a daemon in one language, tools in another and
+	// a package in a third (ADR 0040). Compiling them all into one place would have them overwrite
+	// each other and then be packed together, so each bundle compiles and packs alone.
+	out := Out(a.Name)
+
 	invocation := []string{
 		"run", "--rm",
 		"--volume", tree + ":" + within,
@@ -468,15 +474,18 @@ func compile(ctx context.Context, run Runner, tree string, chain Toolchain,
 		base,
 	}
 	invocation = append(invocation, chain.Compile...)
+	if chain.OutputFlag != "" {
+		invocation = append(invocation, chain.OutputFlag, out)
+	}
 	// What to compile. Named by the module rather than discovered, so adding a file does not
 	// silently change what a build produces.
 	if len(a.Entrypoints) > 0 {
-		invocation = append(invocation, sourcesFor(a.Entrypoints)...)
+		invocation = append(invocation, sourcesFor(a.Entrypoints, out)...)
 	}
 	if _, err := run(ctx, tree, "docker", invocation...); err != nil {
 		return "", err
 	}
-	return filepath.Join(tree, chain.Output), nil
+	return filepath.Join(tree, out), nil
 }
 
 // sourcesFor turns compiled entrypoints back into what to compile.
@@ -484,10 +493,14 @@ func compile(ctx context.Context, run Runner, tree string, chain Toolchain,
 // A module names what a tool host should LOAD — compiled paths under the bundle's root — because
 // that is the thing anything else needs to know. What to compile is the same list with the
 // language's own extension, which is the toolchain's business rather than the module's.
-func sourcesFor(entrypoints []string) []string {
+func sourcesFor(entrypoints []string, out string) []string {
-	out := make([]string, 0, len(entrypoints))
+	sources := make([]string, 0, len(entrypoints))
 	for _, e := range entrypoints {
-		out = append(out, strings.TrimSuffix(e, filepath.Ext(e))+".ts")
+		// An entrypoint is named as it will be FOUND — a path inside the unpacked bundle — so the
+		// source is the same path with the output directory taken off the front and the language's
+		// own extension on the end.
+		at := strings.TrimPrefix(strings.TrimPrefix(e, out), "/")
+		sources = append(sources, strings.TrimSuffix(at, filepath.Ext(at))+".ts")
 	}
-	return out
+	return sources
 }

internal/builder/bundle_test.go

@@ -10,7 +10,7 @@ import (
 
 const aBundle = `{"module":"greeter","version":"1",
   "build":{"artifacts":[
-    {"name":"code","kind":"bundle","language":"typescript","entrypoints":["dist/index.js"]}]},
+    {"name":"code","kind":"bundle","language":"typescript","entrypoints":["index.js"]}]},
   "resources":[
     {"id":"files","type":"archive","path":"/opt/greeter","artifact":"code"}]}`
 
@@ -21,15 +21,28 @@ type compiling struct{ *recorded }
 
 func (c compiling) run(ctx context.Context, dir, name string, args ...string) (string, error) {
 	out, err := c.recorded.run(ctx, dir, name, args...)
-	if name == "docker" && len(args) > 0 && args[0] == "run" {
+	if name != "docker" || len(args) == 0 || args[0] != "run" {
-		made := filepath.Join(dir, "dist")
+		return out, err
-		if err := os.MkdirAll(made, 0o755); err != nil {
+	}
-			return "", err
+	// **Writes where it was TOLD to**, rather than to a fixed directory. A fake that always wrote
-		}
+	// to one place would pass whether or not the builder gave each artifact its own — which is the
-		if err := os.WriteFile(filepath.Join(made, "index.js"), []byte("console.log(1)"), 0o644); err != nil {
+	// thing being tested.
-			return "", err
+	where := ""
+	for i, a := range args {
+		if a == "--outDir" && i+1 < len(args) {
+			where = args[i+1]
 		}
 	}
+	if where == "" {
+		return out, err
+	}
+	made := filepath.Join(dir, where)
+	if err := os.MkdirAll(made, 0o755); err != nil {
+		return "", err
+	}
+	if err := os.WriteFile(filepath.Join(made, "index.js"), []byte("console.log(1)"), 0o644); err != nil {
+		return "", err
+	}
 	return out, err
 }
 
@@ -107,3 +120,51 @@ func TestABundleInAnUnknownLanguageIsRefused(t *testing.T) {
 		t.Fatalf("the refusal does not say what would have worked: %v", err)
 	}
 }
+
+// **One module, two bundles, and neither packs the other.**
+//
+// The case that matters for real modules: a module is one piece of software and may still carry a
+// daemon in one language and tools in another (ADR 0040). An earlier version of this compiled
+// every bundle into the toolchain's single output directory, so two of them would overwrite each
+// other and then be packed together — one artifact containing both, twice.
+func TestTwoBundlesInOneModuleArePackedSeparately(t *testing.T) {
+	const two = `{"module":"greeter","version":"1",
+	  "build":{"artifacts":[
+	    {"name":"daemon","kind":"bundle","language":"typescript","entrypoints":["index.js"]},
+	    {"name":"tools","kind":"bundle","language":"typescript","entrypoints":["index.js"]}]},
+	  "resources":[
+	    {"id":"a","type":"archive","path":"/opt/greeter/daemon","artifact":"daemon"},
+	    {"id":"b","type":"archive","path":"/opt/greeter/tools","artifact":"tools"}]}`
+
+	r, workspace := aRepository(t, two, map[string]string{"index.ts": "console.log(1)"})
+	held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)}
+
+	got, err := Build(context.Background(), compiling{r}.run, r,
+		"https://forge.invalid/greeter.git", "", "", workspace, held)
+	if err != nil {
+		t.Fatalf("a module with two bundles did not build: %v", err)
+	}
+
+	// Compiled into two different places.
+	var outputs []string
+	for _, line := range r.ran {
+		for _, part := range strings.Fields(line) {
+			if strings.HasPrefix(part, ".mesh-build/") {
+				outputs = append(outputs, part)
+			}
+		}
+	}
+	if len(outputs) != 2 || outputs[0] == outputs[1] {
+		t.Fatalf("two bundles did not get their own output directories: %v", outputs)
+	}
+
+	// And published as two artifacts, each with its own digest.
+	if len(r.archives) != 2 {
+		t.Fatalf("expected two archives published, got %v", r.archives)
+	}
+	first, _ := got.Manifest.Resources[0]["digest"].(string)
+	second, _ := got.Manifest.Resources[1]["digest"].(string)
+	if first == "" || second == "" {
+		t.Fatalf("a bundle was not pinned: %v", got.Manifest.Resources)
+	}
+}

internal/builder/toolchain.go

@@ -26,13 +26,28 @@ type Toolchain struct {
 	Base string
 	// Artifact is which of that module's artifacts is the compiling one.
 	Artifact string
-	// Compile is what runs inside it, relative to the module's own directory. The output goes to
+	// Compile is what runs inside it, relative to the module's own directory.
-	// Output, which is what gets packed.
+	//
+	// The output directory is appended by the builder, per artifact, because one module may
+	// declare several bundles — a daemon in one language, tools in another, a package in a third —
+	// and a toolchain with one fixed output would have them overwrite each other and then be
+	// packed together.
 	Compile []string
-	// Output is the directory the compiled result lands in, relative to the module's directory.
+	// OutputFlag is how this compiler is told where to put its output.
-	Output string
+	OutputFlag string
 }
 
+// Out is where one artifact's compiled output lands, inside the module's own directory.
+//
+// **Per artifact, never per toolchain.** A module is one piece of software and may still be
+// written in several languages — a daemon in one, a tool in another, a package in a third (ADR
+// 0040). Each bundle is compiled and packed alone, so what a machine unpacks is that artifact and
+// nothing else.
+//
+// Under a directory named for the build rather than beside the source, so a pack never sweeps up
+// the module's own working files, and two builds of one commit see the same tree.
+func Out(artifact string) string { return ".mesh-build/" + artifact }
+
 // toolchains is every language the mesh can build.
 //
 // **A closed list, and adding to it is a decision rather than a configuration.** Every language is
@@ -53,9 +68,20 @@ var toolchains = []Toolchain{
 		Compile: []string{
 			"node", "/app/node_modules/typescript/bin/tsc",
 			"--module", "NodeNext", "--moduleResolution", "NodeNext",
-			"--target", "ES2022", "--outDir", "dist",
+			"--target", "ES2022",
 		},
-		Output: "dist",
+		OutputFlag: "--outDir",
+	},
+	{
+		Language: "python",
+		Base:     "mesh-tools-python",
+		Artifact: "build",
+		// Nothing to compile: what a bundle needs is the module's own code and its dependencies
+		// resolved, so the "compile" is an install into the output directory. Named here rather
+		// than left implicit because a reader comparing two toolchains should be able to see what
+		// each actually does.
+		Compile:    []string{"python", "-m", "pip", "install", "--no-compile", "--target"},
+		OutputFlag: "",
 	},
 }
 

internal/builder/toolchain_test.go

@@ -14,8 +14,8 @@ func TestADeclaredLanguageSelectsItsToolchain(t *testing.T) {
 	if chain.Base == "" || chain.Artifact == "" {
 		t.Fatalf("a toolchain names no base to compile in: %+v", chain)
 	}
-	if len(chain.Compile) == 0 || chain.Output == "" {
+	if len(chain.Compile) == 0 {
-		t.Fatalf("a toolchain says nothing about how to compile or where output lands: %+v", chain)
+		t.Fatalf("a toolchain says nothing about how to compile: %+v", chain)
 	}
 }
 
@@ -54,3 +54,36 @@ func TestABundleMustSayWhatItIsWrittenIn(t *testing.T) {
 		t.Fatalf("the refusal does not say a language is required: %v", err)
 	}
 }
+
+// **One module, several languages, and each bundle packed alone.**
+//
+// A module is one piece of software (ADR 0040) and may still carry a daemon in one language, tools
+// in another and a package in a third. Compiling them into one output directory would have them
+// overwrite each other and then be packed together, so output is a property of the artifact rather
+// than of the toolchain.
+func TestTwoBundlesInOneModuleDoNotShareAnOutputDirectory(t *testing.T) {
+	first, second := Out("daemon"), Out("tools")
+	if first == second {
+		t.Fatalf("two artifacts compile into the same place (%q), so one would overwrite the "+
+			"other and both would be packed together", first)
+	}
+	for _, out := range []string{first, second} {
+		if strings.HasPrefix(out, "/") || strings.Contains(out, "..") {
+			t.Fatalf("%q leaves the module's own directory", out)
+		}
+	}
+}
+
+// And the mesh can say what it builds, which is what a refusal quotes.
+func TestTheMeshSaysWhichLanguagesItBuilds(t *testing.T) {
+	spoken := Languages()
+	if len(spoken) < 2 {
+		t.Fatalf("only %v — this test exists to keep the multi-language path real rather than "+
+			"theoretical", spoken)
+	}
+	for _, language := range spoken {
+		if _, err := ToolchainFor(language); err != nil {
+			t.Fatalf("%q is listed as buildable and has no toolchain: %v", language, err)
+		}
+	}
+}