diff --git a/internal/builder/builder.go b/internal/builder/builder.go index 8658b65..fe6c754 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -129,7 +129,7 @@ func Build(ctx context.Context, run Runner, publish Publisher, // logs can be compared. sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name }) for _, a := range artifacts { - made, err := one(ctx, run, publish, manifest.Module, within, commit, a, args) + made, err := one(ctx, run, publish, manifest.Module, within, commit, a, args, held) if err != nil { return Result{}, err } @@ -215,7 +215,8 @@ func against(within string, manifest catalogue.Manifest) []string { const ManifestName = "module.json" func one(ctx context.Context, run Runner, publish Publisher, - module, tree, commit string, a catalogue.Artifact, args []string) (catalogue.Built, error) { + module, tree, commit string, a catalogue.Artifact, args []string, + held map[string]string) (catalogue.Built, error) { switch a.Kind { case catalogue.ArtifactUpstream: @@ -252,6 +253,46 @@ func one(ctx context.Context, run Runner, publish Publisher, } return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil + case catalogue.ArtifactBundle: + // **The one recipe that both builds and packs.** Everything else either produces an image + // or packs what is already there; this compiles the module's own code first, in a + // toolchain the mesh chose from what the module said it was written in, and packs the + // result. + // + // The compiler runs in a container rather than on the build machine, for the reason every + // other build does: what a build needs installed is the toolchain's business, and a build + // machine that accumulated one toolchain per language would be a machine nobody could + // reproduce. + chain, err := ToolchainFor(a.Language) + if err != nil { + return catalogue.Built{}, fmt.Errorf("%s: %s: %w", module, a.Name, err) + } + base, ok := held[chain.Base+"/"+chain.Artifact] + if !ok { + // Named, not pinned: the mesh answers with the copy it holds. Refused before anything + // is built, saying which module has to exist first, rather than failing inside a + // compile with a message about an image (novox/hq 04-ISSUES/044). + return catalogue.Built{}, fmt.Errorf( + "%s: %s is written in %s, which is compiled by %s's %q artifact, and this mesh "+ + "holds no copy of it. Build %s first", + module, a.Name, chain.Language, chain.Base, chain.Artifact, chain.Base) + } + compiled, err := compile(ctx, run, tree, chain, base, a) + if err != nil { + return catalogue.Built{}, fmt.Errorf("%s: compiling %s failed: %w", module, a.Name, err) + } + body, err := pack(compiled) + if err != nil { + return catalogue.Built{}, fmt.Errorf("%s: packing %s failed: %w", module, a.Name, err) + } + sum := sha256.Sum256(body) + digest := "sha256:" + hex.EncodeToString(sum[:]) + where, err := publish.PublishArchive(ctx, module+"/"+a.Name, body, digest) + if err != nil { + return catalogue.Built{}, err + } + return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: where, Digest: digest}, nil + case catalogue.ArtifactArchive: body, err := pack(filepath.Join(tree, a.From)) if err != nil { @@ -406,3 +447,47 @@ func standingOn(manifest catalogue.Manifest, held map[string]string) ([]string, } return args, nil } + +// compile runs a module's own code through its toolchain, and says where the result is. +// +// **In the module's own directory, under the path the toolchain expects.** A module is compiled +// where its dependencies resolve upward into the base's own library directory, so what it is +// compiled against is exactly what it will run against — the reason every hand-written Dockerfile +// had to choose a working directory carefully, and the reason none of them has to now. +func compile(ctx context.Context, run Runner, tree string, chain Toolchain, + base string, a catalogue.Artifact) (string, error) { + + // Where inside the toolchain the module's source is mounted, and where its output lands. Fixed + // rather than configurable: a module that could move this would be describing its own build. + const within = "/app/modules/module" + + invocation := []string{ + "run", "--rm", + "--volume", tree + ":" + within, + "--workdir", within, + base, + } + invocation = append(invocation, chain.Compile...) + // What to compile. Named by the module rather than discovered, so adding a file does not + // silently change what a build produces. + if len(a.Entrypoints) > 0 { + invocation = append(invocation, sourcesFor(a.Entrypoints)...) + } + if _, err := run(ctx, tree, "docker", invocation...); err != nil { + return "", err + } + return filepath.Join(tree, chain.Output), nil +} + +// sourcesFor turns compiled entrypoints back into what to compile. +// +// A module names what a tool host should LOAD — compiled paths under the bundle's root — because +// that is the thing anything else needs to know. What to compile is the same list with the +// language's own extension, which is the toolchain's business rather than the module's. +func sourcesFor(entrypoints []string) []string { + out := make([]string, 0, len(entrypoints)) + for _, e := range entrypoints { + out = append(out, strings.TrimSuffix(e, filepath.Ext(e))+".ts") + } + return out +} diff --git a/internal/builder/bundle_test.go b/internal/builder/bundle_test.go new file mode 100644 index 0000000..2f40ba9 --- /dev/null +++ b/internal/builder/bundle_test.go @@ -0,0 +1,109 @@ +package builder + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +const aBundle = `{"module":"greeter","version":"1", + "build":{"artifacts":[ + {"name":"code","kind":"bundle","language":"typescript","entrypoints":["dist/index.js"]}]}, + "resources":[ + {"id":"files","type":"archive","path":"/opt/greeter","artifact":"code"}]}` + +// compiling is a runner that behaves like a toolchain: when asked to compile, it leaves output +// where the toolchain says output lands. Without this the pack step has nothing to pack, and the +// test would be asserting on a failure rather than on a build. +type compiling struct{ *recorded } + +func (c compiling) run(ctx context.Context, dir, name string, args ...string) (string, error) { + out, err := c.recorded.run(ctx, dir, name, args...) + if name == "docker" && len(args) > 0 && args[0] == "run" { + made := filepath.Join(dir, "dist") + if err := os.MkdirAll(made, 0o755); err != nil { + return "", err + } + if err := os.WriteFile(filepath.Join(made, "index.js"), []byte("console.log(1)"), 0o644); err != nil { + return "", err + } + } + return out, err +} + +// **A module says what it is written in, and needs no Dockerfile.** This is the whole point of the +// bundle recipe: the same module previously needed a hand-written recipe repeating an incantation +// that is easy to get wrong in ways that fail somewhere else. +func TestABundleIsCompiledAndPackedWithNoDockerfile(t *testing.T) { + r, workspace := aRepository(t, aBundle, map[string]string{"index.ts": "console.log(1)"}) + held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} + + got, err := Build(context.Background(), compiling{r}.run, r, + "https://forge.invalid/greeter.git", "", "", workspace, held) + if err != nil { + t.Fatalf("a module with a language and no Dockerfile did not build: %v", err) + } + + // Compiled in the toolchain the mesh chose, not in one the module named. + var compiled string + for _, line := range r.ran { + if strings.HasPrefix(line, "docker run") { + compiled = line + } + } + if compiled == "" { + t.Fatalf("nothing was compiled:\n%s", strings.Join(r.ran, "\n")) + } + if !strings.Contains(compiled, "mesh-tools/build@sha256:") { + t.Fatalf("the compile did not run in the mesh's own toolchain: %s", compiled) + } + if strings.Contains(strings.Join(r.ran, "\n"), "docker build") { + t.Fatalf("a bundle invoked a Dockerfile build, which is the thing it exists to avoid:\n%s", + strings.Join(r.ran, "\n")) + } + + // And pinned by a digest of what came out, like any other artifact. + digest, _ := got.Manifest.Resources[0]["digest"].(string) + if !strings.HasPrefix(digest, "sha256:") { + t.Fatalf("the bundle was not pinned: %v", got.Manifest.Resources[0]) + } +} + +// **Refused before anything is built, naming what to build first.** A base the mesh has not built +// is not a compile that fails on its first line — it is a question somebody can answer, and saying +// it early is the difference between a fixable message and one about a missing image. +func TestABundleWhoseToolchainIsNotHeldIsRefusedFirst(t *testing.T) { + r, workspace := aRepository(t, aBundle, map[string]string{"index.ts": "console.log(1)"}) + + _, err := Build(context.Background(), compiling{r}.run, r, + "https://forge.invalid/greeter.git", "", "", workspace, nil) + if err == nil { + t.Fatal("a bundle was built with no toolchain to compile it in") + } + if !strings.Contains(err.Error(), "mesh-tools") { + t.Fatalf("the refusal does not name what has to be built first: %v", err) + } + for _, line := range r.ran { + if strings.HasPrefix(line, "docker run") { + t.Fatalf("a compile was attempted before the refusal: %s", line) + } + } +} + +// A language the mesh does not build is refused the same way, and names what it can build. +func TestABundleInAnUnknownLanguageIsRefused(t *testing.T) { + manifest := strings.Replace(aBundle, `"language":"typescript"`, `"language":"cobol"`, 1) + r, workspace := aRepository(t, manifest, map[string]string{"index.ts": "x"}) + + _, err := Build(context.Background(), compiling{r}.run, r, + "https://forge.invalid/greeter.git", "", "", workspace, + map[string]string{"mesh-tools/build": "registry.invalid/x@sha256:" + strings.Repeat("c", 64)}) + if err == nil { + t.Fatal("a language nothing can compile was accepted") + } + if !strings.Contains(err.Error(), "typescript") { + t.Fatalf("the refusal does not say what would have worked: %v", err) + } +} diff --git a/internal/builder/toolchain.go b/internal/builder/toolchain.go new file mode 100644 index 0000000..2d23d19 --- /dev/null +++ b/internal/builder/toolchain.go @@ -0,0 +1,99 @@ +package builder + +import ( + "fmt" + "sort" + "strings" +) + +// What a language implies, so a module does not have to say it. +// +// **A module says what it is written in; this says what that means.** The alternative is what the +// mesh had: every module carrying a Dockerfile that repeated the same incantation, and most of the +// catalogue never converted because the incantation is easy to get wrong in ways that fail +// somewhere else (novox/hq 03-DESIGN/01-to-be/18-building-a-module.md). +// +// A toolchain is deliberately not configurable by the module. Anything a module could override +// here it would be writing a Dockerfile to override, and then this bought nothing. + +// Toolchain is how one language is compiled into a bundle. +type Toolchain struct { + // Language is what a module declares to select this. + Language string + // Base is the module whose artifact provides the compiler, named rather than pinned: the mesh + // answers with the copy it holds, so a recipe never names one particular build of it + // (novox/hq 04-ISSUES/044). + Base string + // Artifact is which of that module's artifacts is the compiling one. + Artifact string + // Compile is what runs inside it, relative to the module's own directory. The output goes to + // Output, which is what gets packed. + Compile []string + // Output is the directory the compiled result lands in, relative to the module's directory. + Output string +} + +// toolchains is every language the mesh can build. +// +// **A closed list, and adding to it is a decision rather than a configuration.** Every language is +// permanent: it needs an SDK carrying the broker client, sealed-credential reading, the event +// envelope and tool serving, and the contracts every module shares change rarely and cascade when +// they do (novox/hq ADR 0039). A mesh whose languages disagree about the envelope fails by ignoring +// messages rather than by failing to compile, so a new entry here is a commitment to keeping N +// implementations of one contract in step. +var toolchains = []Toolchain{ + { + Language: "typescript", + Base: "mesh-tools", + Artifact: "build", + // Invoked by its real path rather than through node_modules/.bin, whose entries are + // symlinks to a launcher that requires its library relatively — and the base image's own + // assembly resolves them away, leaving a launcher whose relative require points nowhere. + // Every module's hand-written Dockerfile had to know this. Now none of them does. + Compile: []string{ + "node", "/app/node_modules/typescript/bin/tsc", + "--module", "NodeNext", "--moduleResolution", "NodeNext", + "--target", "ES2022", "--outDir", "dist", + }, + Output: "dist", + }, +} + +// ToolchainFor is what builds this language, or says what it can build. +func ToolchainFor(language string) (Toolchain, error) { + want := strings.ToLower(strings.TrimSpace(language)) + if want == "" { + return Toolchain{}, fmt.Errorf( + "a bundle must say what language it is written in: the mesh chooses the compiler, and "+ + "it cannot choose one for a module that has not said. It can build %s", spoken()) + } + for _, t := range toolchains { + if t.Language == want { + return t, nil + } + } + return Toolchain{}, fmt.Errorf( + "%q is not a language this mesh builds. It can build %s — and adding one is a decision "+ + "rather than a setting, because every language is another implementation of the "+ + "contracts every module shares", language, spoken()) +} + +// spoken lists the languages, so a refusal says what would have worked. +func spoken() string { + names := make([]string, 0, len(toolchains)) + for _, t := range toolchains { + names = append(names, t.Language) + } + sort.Strings(names) + return strings.Join(names, ", ") +} + +// Languages is every language the mesh can build, for anything that wants to say so. +func Languages() []string { + names := make([]string, 0, len(toolchains)) + for _, t := range toolchains { + names = append(names, t.Language) + } + sort.Strings(names) + return names +} diff --git a/internal/builder/toolchain_test.go b/internal/builder/toolchain_test.go new file mode 100644 index 0000000..e387de3 --- /dev/null +++ b/internal/builder/toolchain_test.go @@ -0,0 +1,56 @@ +package builder + +import ( + "strings" + "testing" +) + +// A language the mesh builds resolves to the toolchain that builds it. +func TestADeclaredLanguageSelectsItsToolchain(t *testing.T) { + chain, err := ToolchainFor("typescript") + if err != nil { + t.Fatalf("typescript is not buildable: %v", err) + } + if chain.Base == "" || chain.Artifact == "" { + t.Fatalf("a toolchain names no base to compile in: %+v", chain) + } + if len(chain.Compile) == 0 || chain.Output == "" { + t.Fatalf("a toolchain says nothing about how to compile or where output lands: %+v", chain) + } +} + +// Case and stray whitespace are a module author's slip, not a different language. +func TestALanguageIsMatchedLoosely(t *testing.T) { + for _, said := range []string{"TypeScript", " typescript ", "TYPESCRIPT"} { + if _, err := ToolchainFor(said); err != nil { + t.Fatalf("%q was refused: %v", said, err) + } + } +} + +// **A refusal says what would have worked.** A module author who names a language the mesh does +// not build is one word away from a language it does, and a bare "unsupported" makes them go +// looking for a list that exists in one place in the source. +func TestAnUnknownLanguageSaysWhatIsBuildable(t *testing.T) { + _, err := ToolchainFor("cobol") + if err == nil { + t.Fatal("a language nothing can build was accepted") + } + for _, want := range Languages() { + if !strings.Contains(err.Error(), want) { + t.Fatalf("the refusal does not mention %q, which would have worked: %v", want, err) + } + } +} + +// And saying nothing is its own message: the mesh chooses the compiler, so a bundle that names no +// language has not asked for anything in particular — which is a mistake rather than a default. +func TestABundleMustSayWhatItIsWrittenIn(t *testing.T) { + _, err := ToolchainFor("") + if err == nil { + t.Fatal("a bundle with no language was accepted, so the mesh guessed a compiler") + } + if !strings.Contains(err.Error(), "must say") { + t.Fatalf("the refusal does not say a language is required: %v", err) + } +} diff --git a/internal/catalogue/build.go b/internal/catalogue/build.go index 361c65e..d7df29a 100644 --- a/internal/catalogue/build.go +++ b/internal/catalogue/build.go @@ -88,12 +88,16 @@ func (m Manifest) Resolve(built []Built) (Manifest, error) { switch artifact.Kind { case ArtifactImage, ArtifactUpstream: filled["image"] = artifact.Reference - case ArtifactArchive: + case ArtifactArchive, ArtifactBundle: + // The same on the wire: both are bytes fetched by digest and unpacked. They differ in + // how they were made — one packed as it stood, the other compiled first — and a + // machine has no reason to care which. filled["source"] = artifact.Reference filled["digest"] = artifact.Digest default: - return Manifest{}, fmt.Errorf("%s: %q is a %q, and an artifact is %q, %q or %q", - m.Module, named, artifact.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream) + return Manifest{}, fmt.Errorf("%s: %q is a %q, and an artifact is %q, %q, %q or %q", + m.Module, named, artifact.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream, + ArtifactBundle) } out.Resources = append(out.Resources, filled) } @@ -119,15 +123,40 @@ func (b *Build) problems(module string) []string { } seen[a.Name] = true switch a.Kind { - case ArtifactImage, ArtifactArchive, ArtifactUpstream: + case ArtifactImage, ArtifactArchive, ArtifactUpstream, ArtifactBundle: default: problems = append(problems, fmt.Sprintf( - "%s: %q is a %q, and an artifact is %q, %q or %q", - module, a.Name, a.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream)) + "%s: %q is a %q, and an artifact is %q, %q, %q or %q", + module, a.Name, a.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream, + ArtifactBundle)) } - if a.From == "" { - problems = append(problems, fmt.Sprintf( - "%s: %q says nothing about what it is built from", module, a.Name)) + // **A bundle is built from the module itself, so it says a language instead.** Everything + // else names what it is built from: a Dockerfile, a directory, somebody else's reference. + // A bundle's source is the module's own directory by definition, and what it needs to say + // is which compiler — because the mesh chooses that, and cannot choose for a module that + // has not said. + if a.Kind == ArtifactBundle { + if a.From != "" { + problems = append(problems, fmt.Sprintf( + "%s: %q is a bundle and names what it is built from (%q). A bundle is built "+ + "from the module's own directory; what it says is the language", + module, a.Name, a.From)) + } + if strings.TrimSpace(a.Language) == "" { + problems = append(problems, fmt.Sprintf( + "%s: %q is a bundle and says no language, so nothing can choose a compiler "+ + "for it", module, a.Name)) + } + } else { + if a.From == "" { + problems = append(problems, fmt.Sprintf( + "%s: %q says nothing about what it is built from", module, a.Name)) + } + if a.Language != "" { + problems = append(problems, fmt.Sprintf( + "%s: %q is a %q and names a language. Only a bundle is compiled by the mesh; "+ + "everything else brings its own recipe", module, a.Name, a.Kind)) + } } // An upstream image is named, not read from the repository, so the path rule does not // apply to it — and applying it anyway would refuse every reference with a registry host diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index 2085d2d..7f6eb9c 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -389,6 +389,26 @@ type Artifact struct { // version and an operating system — while letting each be built and published separately. // Empty means the whole recipe, which is what a module with one image says by saying nothing. Target string `json:"target,omitempty"` + + // Language is what this module's code is written in, for a bundle. + // + // **Declared, never guessed.** Inferring it from what files happen to be present makes a + // module's build depend on a directory listing, and a module that adds a stray file builds + // differently for a reason nobody can see. It is also the only thing a bundle needs to say: + // everything else about the toolchain — which compiler, which flags, which base — is the + // mesh's, and a module that could override it would be writing a Dockerfile again. + // + // Empty for every other kind, which do not compile. + Language string `json:"language,omitempty"` + + // Entrypoints are the compiled files a tool host should load from this module, relative to the + // bundle's root. + // + // **Named rather than derived from which files exist**, for the same reason as the language: + // the module knows what it serves, and a build that guesses would change meaning when + // somebody adds a helper. An empty list is a bundle that is run rather than loaded — a + // provisioner or a step, named by whatever runs it. + Entrypoints []string `json:"entrypoints,omitempty"` } // Kinds an artifact may be. @@ -397,6 +417,21 @@ const ( ArtifactImage = "image" // ArtifactArchive is a directory in this repository, packed. ArtifactArchive = "archive" + // ArtifactBundle is this module's own code, COMPILED by a toolchain and then packed. + // + // **The one recipe that both builds and packs**, and the reason it exists is the authoring + // burden. An `archive` packs a directory as it stands, so shipping compiled output means + // compiling somewhere first — which means a Dockerfile, repeating the same incantation in + // every module: two base arguments, a working directory chosen so the SDK resolves upward, the + // compiler invoked by absolute path because the usual symlink is resolved away when the base is + // assembled, a second stage, an environment variable naming the entrypoints. Most of the + // catalogue is unconverted and that is why. + // + // A bundle says what the module is written in and nothing about how. The mesh knows what a + // language implies, which is the whole of the difference: a Dockerfile is right for software + // that needs a particular base, and wrong for "compile my module's code", which is the same + // operation every time. + ArtifactBundle = "bundle" // ArtifactUpstream is an image somebody else built, mirrored into the mesh's own registry and // pinned by the digest it lands with. //