Merge pull request 'The mesh can compile Go, which is why nothing delivered the host' (#154) from feat/142-the-mesh-compiles-its-own-go into main

internal/builder/builder.go

@@ -861,6 +861,15 @@ func compile(ctx context.Context, run Runner, tree string, chain Toolchain,
 	// each other and then be packed together, so each bundle compiles and packs alone.
 	out := Out(a.Name)
 
+	// **The output directory exists before the compiler is told about it.** `tsc --outDir` makes
+	// one; `go build -o` writes a file into a directory and does not create it, failing with a
+	// message about a path rather than about a build. Made here for every toolchain, because which
+	// compilers happen to be forgiving is not a thing a reader should have to know
+	// (novox/hq 04-ISSUES/142).
+	if err := os.MkdirAll(filepath.Join(tree, out), 0o755); err != nil {
+		return "", fmt.Errorf("making the output directory for %s: %w", a.Name, err)
+	}
+
 	invocation := []string{
 		"run", "--rm",
 		"--volume", tree + ":" + within,
@@ -873,8 +882,14 @@ func compile(ctx context.Context, run Runner, tree string, chain Toolchain,
 	}
 	// What to compile. Named by the module rather than discovered, so adding a file does not
 	// silently change what a build produces.
-	if len(a.Entrypoints) > 0 {
+	switch {
-		invocation = append(invocation, sourcesFor(a.Entrypoints, out)...)
+	case chain.Unit == UnitPackage:
+		// One directory, compiled whole: the thing the artifact is built `from`. Relative, because
+		// the compiler runs with the module's own root as its working directory and a package path
+		// that looked absolute would name one inside the toolchain image.
+		invocation = append(invocation, "./"+strings.Trim(a.From, "./"))
+	case len(a.Entrypoints) > 0:
+		invocation = append(invocation, sourcesFor(a.Entrypoints, out, chain.SourceExt)...)
 	}
 	if _, err := run(ctx, tree, "docker", invocation...); err != nil {
 		return "", err
@@ -887,14 +902,16 @@ func compile(ctx context.Context, run Runner, tree string, chain Toolchain,
 // A module names what a tool host should LOAD — compiled paths under the bundle's root — because
 // that is the thing anything else needs to know. What to compile is the same list with the
 // language's own extension, which is the toolchain's business rather than the module's.
-func sourcesFor(entrypoints []string, out string) []string {
+func sourcesFor(entrypoints []string, out, ext string) []string {
 	sources := make([]string, 0, len(entrypoints))
 	for _, e := range entrypoints {
 		// An entrypoint is named as it will be FOUND — a path inside the unpacked bundle — so the
 		// source is the same path with the output directory taken off the front and the language's
-		// own extension on the end.
+		// own extension on the end. **The extension is the toolchain's**, where it used to be the
+		// literal `.ts`: one language's file extension written into the code that serves every
+		// language is a wall the next one hits (novox/hq 04-ISSUES/142).
 		at := strings.TrimPrefix(strings.TrimPrefix(e, out), "/")
-		sources = append(sources, strings.TrimSuffix(at, filepath.Ext(at))+".ts")
+		sources = append(sources, strings.TrimSuffix(at, filepath.Ext(at))+ext)
 	}
 	return sources
 }

internal/builder/go_toolchain_test.go

@@ -0,0 +1,71 @@
+package builder
+
+import (
+	"strings"
+	"testing"
+)
+
+// Nothing could compile the mesh's own components, which is why nothing delivers the host
+// (novox/hq 04-ISSUES/142, and ADR 0141's own insight naming it). The toolchain list was a closed
+// set of typescript and python, and two things in the path beyond it assumed TypeScript.
+
+func TestTheMeshCanCompileGo(t *testing.T) {
+	chain, err := ToolchainFor("go")
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Named, not pinned: the mesh answers with the copy it holds, so moving compiler is a build
+	// rather than an edit to this source (ADR 0044, 0142).
+	if chain.Base != "mesh-tools-go" || chain.Artifact != "build" {
+		t.Fatalf("the go toolchain is based on %s/%s", chain.Base, chain.Artifact)
+	}
+	joined := strings.Join(chain.Compile, " ")
+	// Static, because what a machine holds is a file and not a container: a binary needing a libc
+	// it did not bring is a delivery that works until a machine differs.
+	if !strings.Contains(joined, "CGO_ENABLED=0") {
+		t.Fatalf("the go toolchain does not build statically: %q", joined)
+	}
+	// Reproducible: a version comes from where a component sits, not from its linker (ADR 0142),
+	// so two builds of one commit should produce the same bytes.
+	if !strings.Contains(joined, "-trimpath") {
+		t.Fatalf("the go toolchain leaves build paths in the binary: %q", joined)
+	}
+	if chain.Unit != UnitPackage {
+		t.Fatalf("the go toolchain compiles %q, wanted a package", chain.Unit)
+	}
+}
+
+func TestEveryToolchainSaysWhatItIsPointedAt(t *testing.T) {
+	// The field exists because the compile path used to assume one language. A toolchain that says
+	// nothing would fall through to the entrypoint branch and compile a file list, which for a
+	// compiled language builds a program out of exactly those files and ignores the rest of the
+	// package — a missing symbol rather than a legible refusal.
+	for _, chain := range toolchains {
+		switch chain.Unit {
+		case UnitPackage:
+		case UnitSources:
+			if chain.SourceExt == "" {
+				t.Fatalf("%s compiles a file list and names no source extension", chain.Language)
+			}
+			if !strings.HasPrefix(chain.SourceExt, ".") {
+				t.Fatalf("%s's source extension %q is not an extension", chain.Language, chain.SourceExt)
+			}
+		default:
+			t.Fatalf("%s says it is pointed at %q, which is neither sources nor a package",
+				chain.Language, chain.Unit)
+		}
+	}
+}
+
+func TestAnEntrypointBecomesASourceInItsOwnLanguage(t *testing.T) {
+	// It used to become a `.ts` whatever the language was.
+	out := Out("build")
+	got := sourcesFor([]string{out + "/tools/index.js"}, out, ".ts")
+	if len(got) != 1 || got[0] != "tools/index.ts" {
+		t.Fatalf("a typescript entrypoint became %v", got)
+	}
+	got = sourcesFor([]string{out + "/tools/index.js"}, out, ".py")
+	if len(got) != 1 || got[0] != "tools/index.py" {
+		t.Fatalf("a python entrypoint became %v", got)
+	}
+}

internal/builder/toolchain.go

@@ -35,8 +35,30 @@ type Toolchain struct {
 	Compile []string
 	// OutputFlag is how this compiler is told where to put its output.
 	OutputFlag string
+	// Unit is what this compiler is pointed at: UnitSources, the entrypoint files the module named,
+	// or UnitPackage, the one directory the artifact is built `from`.
+	//
+	// **The difference is the language and not the module.** A TypeScript bundle is a set of files
+	// compiled into a set of files, so what to compile is the module's entrypoints with their source
+	// extension. A Go bundle is a package compiled into one binary, and there is no per-file
+	// compilation to name — pointing `go build` at a file list builds a program out of exactly those
+	// files and ignores the rest of the package, which fails as a missing symbol rather than as a
+	// wrong instruction.
+	Unit string
+	// SourceExt is the extension an entrypoint has in the repository, for UnitSources. An entrypoint
+	// is named as it will be FOUND, inside the unpacked bundle, so the source is the same path with
+	// the output directory taken off the front and this on the end.
+	SourceExt string
 }
 
+// What a toolchain is pointed at.
+const (
+	// UnitSources is a list of files, derived from the module's entrypoints.
+	UnitSources = "sources"
+	// UnitPackage is the single directory the artifact is built `from`, compiled whole.
+	UnitPackage = "package"
+)
+
 // Out is where one artifact's compiled output lands, inside the module's own directory.
 //
 // **Per artifact, never per toolchain.** A module is one piece of software and may still be
@@ -71,6 +93,35 @@ var toolchains = []Toolchain{
 			"--target", "ES2022",
 		},
 		OutputFlag: "--outDir",
+		Unit:       UnitSources,
+		SourceExt:  ".ts",
+	},
+	{
+		Language: "go",
+		Base:     "mesh-tools-go",
+		Artifact: "build",
+		// **The mesh's own components, and not modules.** The warning above this list — that every
+		// language is another implementation of the contracts modules share, so adding one commits
+		// to keeping N implementations in step — does not attach here. Go is how the host, the
+		// control plane and the builder are written, and none of them is a module in that sense:
+		// the host is what APPLIES modules. So there is no SDK obligation, and the reason this
+		// entry did not exist was that nothing needed to compile the mesh itself
+		// (novox/hq ADR 0142, and 04-ISSUES/142 where that is why nothing delivers the host).
+		//
+		// Static, because what a machine ends up holding is a file rather than a container, and a
+		// binary that needs a libc it did not bring is a delivery that works until a machine
+		// differs. Trimmed of its own paths for the same reason a version comes from where it sits
+		// rather than from the linker: two builds of one commit produce the same bytes.
+		Compile: []string{
+			"env", "CGO_ENABLED=0", "GOFLAGS=-trimpath",
+			"go", "build", "-ldflags", "-s -w",
+		},
+		OutputFlag: "-o",
+		// Pointed at the package the artifact is built `from`, compiled whole. Go writes the binary
+		// into the output directory, named after the package — so the bundle a machine unpacks is a
+		// directory holding one executable, which is what the delivery mechanism expects
+		// (novox/hq ADR 0141).
+		Unit: UnitPackage,
 	},
 	{
 		Language: "python",
@@ -82,6 +133,8 @@ var toolchains = []Toolchain{
 		// each actually does.
 		Compile:    []string{"python", "-m", "pip", "install", "--no-compile", "--target"},
 		OutputFlag: "",
+		Unit:       UnitSources,
+		SourceExt:  ".py",
 	},
 }