package builder import ( "archive/tar" "compress/gzip" "context" "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "io" "os" "os/exec" "path/filepath" "regexp" "sort" "strconv" "strings" "time" "github.com/novox/mesh-controller/internal/catalogue" ) // Turning a repository into artifacts the mesh can pin. // // **This runs on a node, not in the control plane.** Building needs a container runtime and a // working tree, and the control plane deliberately cannot run commands on a machine — what it may // send is bounded by the declaration language (novox/hq ADR 0005), and "run this build" is not in // it. So the builder is something a node runs *as a module*, given work over the broker like // anything else, and this package is what it does when it gets some. // // The alternative — the control plane holding a docker socket — would make it the one component // that can do anything on a machine, which is the property the whole design is arranged to avoid. // Runner runs a command in a directory and returns what it said. Injected so the tests do not // need docker and git, and so the failure of either is reported rather than assumed. type Runner func(ctx context.Context, dir string, name string, args ...string) (string, error) // Publisher puts an artifact somewhere a machine can fetch it, and says how to refer to it. type Publisher interface { // PublishImage pushes a locally built image and returns a reference pinned by digest. PublishImage(ctx context.Context, localTag, repository string) (string, error) // PublishArchive stores bytes and returns where to fetch them from. PublishArchive(ctx context.Context, repository string, body []byte, digest string) (string, error) } // Result is everything one build produced. type Result struct { // Against is every pinned image this build was built on top of, read out of its own inputs. // // **Derived, not declared** (novox/hq ADR 0009): a declared list of dependencies drifts from // what the code actually uses, and an artifact is out of date when anything it was built // against moved. These are artifact references rather than module-versions, because that is // what a build input names; resolving them to modules is the catalogue's work, since it is // what knows which module-version published which artifact. Against []string // Manifest is the module as the mesh should hold it: artifacts resolved to digests. Manifest catalogue.Manifest // Commit is what was built, so "is this current?" is answerable without building again. Commit string // Built is each artifact, for reporting. Built []catalogue.Built } // Build clones a repository at a ref, reads its manifest, produces what it declares, publishes // each, and returns the manifest the mesh should hold. // // **Nothing is published until everything is built.** A module whose image succeeded and whose // archive failed would otherwise leave half of itself in the store under a digest the mesh never // records — reachable, unreferenced, and indistinguishable from something in use. func Build(ctx context.Context, run Runner, publish Publisher, repository, path, ref, workspace string, held map[string]string, npmrc Npmrc, log Log) (Result, error) { say := logging(log) say("clone", "%s%s at %s", repository, describePath(path), refOrHead(ref)) // Made rather than required. A builder that fails because the directory it was told to work // in does not exist is a builder that needs a setup step nobody documented. if err := os.MkdirAll(workspace, 0o755); err != nil { return Result{}, err } tree := filepath.Join(workspace, "source") if err := os.RemoveAll(tree); err != nil { return Result{}, err } // A fresh clone every time rather than a fetch into a tree that is already there. A build // that reuses a working tree can succeed because of something a previous build left behind, // and that is a build nobody can reproduce. if _, err := run(ctx, workspace, "git", "clone", "--quiet", repository, tree); err != nil { say("clone", "FAILED: %v", err) return Result{}, fmt.Errorf("cannot clone %s: %w", repository, err) } say("clone", "done") if ref != "" { if _, err := run(ctx, tree, "git", "checkout", "--quiet", ref); err != nil { return Result{}, fmt.Errorf("%s has no %s: %w", repository, ref, err) } } commit, err := run(ctx, tree, "git", "rev-parse", "HEAD") if err != nil { return Result{}, err } commit = strings.TrimSpace(commit) say("commit", "%s", short(commit)) // A module is a repository and a path within it (novox/hq ADR 0069). The ordinary case is an // empty path, meaning the repository's root; a repository holding several modules names each // by its own directory, which is what the catalogue is and what the system this replaces has // always done. within, err := inside(tree, path) if err != nil { return Result{}, err } raw, err := os.ReadFile(filepath.Join(within, ManifestName)) if err != nil { return Result{}, fmt.Errorf( "%s has no %s at %s, so there is nothing saying what it is: %w", repository, ManifestName, describe(path), err) } manifest, err := catalogue.ParseManifest(raw) if err != nil { say("manifest", "INVALID: %v", err) return Result{}, err } say("manifest", "%s v%s — %d artifact(s)", manifest.Module, manifest.Version, artifactCount(manifest)) // A build-time credential, written into the build context as .npmrc, but ONLY for a module that // asks for it: a `package` artifact (which publishes), or an image whose Dockerfile COPYs .npmrc. // Writing it into every context would put a per-run credential in `COPY . .` of modules that // never resolve a mesh package — making their image non-deterministic (a needless rollout every // build) and leaking the credential into a build stage. Absent entirely with no registry, which // is the bootstrap case (novox/hq ADR 0076). var npmrcPath string if npmrc.Enabled() && manifest.Build != nil && wantsPackages(manifest, within) { content, err := npmrc.File() if err != nil { return Result{}, err } npmrcPath = filepath.Join(within, ".npmrc") if err := os.WriteFile(npmrcPath, []byte(content), 0o600); err != nil { return Result{}, fmt.Errorf("cannot write the package-registry credential for the build: %w", err) } say("packages", "resolving %s from the mesh's package registry", npmrc.Scope) } var built []catalogue.Built if manifest.Build != nil { // What this module said it stands on, answered with what this mesh actually holds. Done // before anything is built, so a missing base is refused in front of the person who can // fix it rather than inside a build that stops on its own first line. // An image published elsewhere that the build stands on is copied into the mesh's own // registry first, like an upstream artifact (ADR 0096), and the recipe is handed the copy. // Genesis has nowhere to copy to and pulls it into this machine's store instead. mirror := func(ctx context.Context, from, repository string) (string, error) { if m, can := publish.(Mirrorer); can { say("bases", "copying %s into the mesh's registry", from) return m.MirrorImage(ctx, from, repository) } if _, err := run(ctx, tree, "docker", "pull", from); err != nil { return "", fmt.Errorf("cannot fetch %s: %w", from, err) } return from, nil } args, err := standingOn(ctx, manifest, held, mirror) if err != nil { say("bases", "UNMET: %v", err) return Result{}, err } if len(args) > 0 { say("bases", "%d resolved from what the mesh holds", len(args)/2) } artifacts := append([]catalogue.Artifact{}, manifest.Build.Artifacts...) // Ordered, so two builds of one commit do the same work in the same sequence and their // logs can be compared. sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name }) for _, a := range artifacts { say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a)) made, err := one(ctx, run, publish, manifest.Module, within, commit, a, args, held, npmrcPath, say) if err != nil { say("artifact", "%s FAILED: %v", a.Name, err) return Result{}, err } say("artifact", "%s done — %s", a.Name, describeMade(made)) built = append(built, made) } } resolved, err := manifest.Resolve(built) if err != nil { say("resolve", "FAILED: %v", err) return Result{}, err } say("done", "%s at %s — %d artifact(s) pinned", manifest.Module, short(commit), len(built)) return Result{Manifest: resolved, Commit: commit, Built: built, Against: against(within, manifest)}, nil } // Log is where a build says what it is doing, step by step. Nil is silent — the tests pass none, // and a build with nowhere to speak must still build. type Log func(step, message string) func logging(log Log) func(step, format string, args ...any) { if log == nil { return func(string, string, ...any) {} } return func(step, format string, args ...any) { log(step, fmt.Sprintf(format, args...)) } } func describePath(path string) string { if path == "" { return "" } return " at " + path } func refOrHead(ref string) string { if ref == "" { return "HEAD" } return ref } func artifactCount(m catalogue.Manifest) int { if m.Build == nil { return 0 } return len(m.Build.Artifacts) } func langSuffix(a catalogue.Artifact) string { if a.Language != "" { return ", " + a.Language } return "" } func describeMade(made catalogue.Built) string { if made.Digest != "" { return made.Kind + " " + short(strings.TrimPrefix(made.Digest, "sha256:")) } return made.Kind + " " + made.Reference } // inside resolves a module's path within a clone, and refuses one that leaves it. // // **A build reads only its own tree.** A path of `../../etc` would otherwise make a build read — // and an archive artifact publish — whatever the build machine happens to hold, which is the one // thing a machine that builds other people's repositories must not do. func inside(tree, path string) (string, error) { if path == "" { return tree, nil } if filepath.IsAbs(path) { return "", fmt.Errorf( "a module's path is inside its repository, and %q is an absolute path", path) } within := filepath.Join(tree, path) rel, err := filepath.Rel(tree, within) if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { return "", fmt.Errorf( "%q leaves the repository, and a build reads only its own tree", path) } return within, nil } // describe says where a manifest was looked for, in words a person can act on. func describe(path string) string { if path == "" { return "its root" } return path } // pinnedImage matches an image reference pinned by digest, which is the only kind a build input is // allowed to name — a tag is something somebody else can move under you. var pinnedImage = regexp.MustCompile(`[A-Za-z0-9][A-Za-z0-9._/:-]*@sha256:[0-9a-f]{64}`) // against reads what this module's image artifacts are built on top of, out of the files that // build them. Nothing is guessed: a reference that is not written down is not reported. func against(within string, manifest catalogue.Manifest) []string { if manifest.Build == nil { return nil } seen := map[string]bool{} var out []string for _, a := range manifest.Build.Artifacts { if a.Kind != catalogue.ArtifactImage || a.From == "" { continue } body, err := os.ReadFile(filepath.Join(within, a.From)) if err != nil { // Not fatal: the build itself already failed if this file was needed and missing, and // reporting no edges is honest where inventing them would not be. continue } for _, found := range pinnedImage.FindAllString(string(body), -1) { if !seen[found] { seen[found] = true out = append(out, found) } } } sort.Strings(out) return out } // ManifestName is the one file a module repository must have. // // At the root, and named the same in every repository. A convention somebody can look for beats a // setting somebody has to find. const ManifestName = "module.json" // wantsPackages reports whether this module's build resolves anything from the mesh's package // registry, so the credential is written into its context only then. A package artifact always // does; an image does when its Dockerfile names .npmrc — the file it would COPY to authenticate. func wantsPackages(manifest catalogue.Manifest, within string) bool { for _, a := range manifest.Build.Artifacts { switch a.Kind { case catalogue.ArtifactPackage: return true case catalogue.ArtifactImage: raw, err := os.ReadFile(filepath.Join(within, a.From)) if err == nil && strings.Contains(string(raw), ".npmrc") { return true } } } return false } func one(ctx context.Context, run Runner, publish Publisher, module, tree, commit string, a catalogue.Artifact, args []string, held map[string]string, npmrc string, say func(step, format string, args ...any)) (catalogue.Built, error) { switch a.Kind { case catalogue.ArtifactUpstream: // Mirrored, not built: copied under a name of the mesh's own, so what a machine fetches is // pinned by a digest this registry assigned rather than by a tag somebody else can move. // // **Between registries, never through this machine's image store** (novox/hq // 04-ISSUES/046, ADR 0096). A published image is an index over several architectures; // pulled, the store keeps the index and refuses to push one platform out of it, and // every variant of pull-then-push failed the same way. A copy moves what is there. if mirror, can := publish.(Mirrorer); can { say("mirror", "copying %s into the mesh's registry", a.From) reference, err := mirror.MirrorImage(ctx, a.From, module+"/"+a.Name) if err != nil { return catalogue.Built{}, fmt.Errorf("%s: %w", module, err) } return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil } // Genesis has no registry to copy into: the image stays in this machine's store, named by // its own id, as every artifact does before there is anywhere to publish. say("mirror", "pulling %s", a.From) if _, err := run(ctx, tree, "docker", "pull", a.From); err != nil { return catalogue.Built{}, fmt.Errorf("%s: cannot fetch %s: %w", module, a.From, err) } reference, err := publish.PublishImage(ctx, a.From, module+"/"+a.Name) if err != nil { return catalogue.Built{}, err } return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil case catalogue.ArtifactImage: // Tagged by commit rather than by version, because a version is what a person calls a // release and a commit is what was actually built. The mesh pins the digest anyway; this // is only so a person looking at the build node can tell what is there. local := fmt.Sprintf("%s-%s:%s", module, a.Name, short(commit)) // The bases this module named, resolved to what this mesh holds. A recipe reads them as // build arguments, so a module says which module it stands on and never which copy. // **A recipe fetches nothing the manifest did not declare** (novox/hq 04-ISSUES/064). A FROM // or a COPY --from naming a registry image that is not a declared base is a build that // reaches a public registry on its own — and works when that registry answers, which is // sometimes. Refused here, in front of the person who can declare it, not inside a build // that fails with "pull access denied" for a reason that is not the mesh's. recipe, err := os.ReadFile(filepath.Join(tree, a.From)) if err != nil { return catalogue.Built{}, fmt.Errorf("%s: cannot read the recipe %s: %w", module, a.From, err) } declared := map[string]bool{} for i := 0; i+1 < len(args); i += 2 { if args[i] == "--build-arg" { declared[strings.SplitN(args[i+1], "=", 2)[0]] = true } } bases, copies := undeclaredFetches(string(recipe), declared) if len(copies) > 0 { return catalogue.Built{}, fmt.Errorf( "%s: the recipe %s copies out of %s, which the manifest does not declare. A build "+ "reaching a public registry on its own works only when that registry answers; "+ "declare it under build.on as {\"arg\": \"\", \"image\": \"@sha256:…\"} "+ "and read it from that argument (novox/hq ADR 0097)", module, a.From, strings.Join(copies, ", ")) } if len(bases) > 0 { // Said, not yet refused: the mesh's own images start FROM a public base — the control // plane's, the builder's, the tool runtime's — and refusing those refuses genesis. // They declare their bases next; until then a base fetched on its own is named here, // with the remedy, every build. say("recipe", "UNDECLARED base(s) %s in %s — declare each under build.on as "+ "{arg, image@sha256:…} and read it from that argument (novox/hq ADR 0097)", strings.Join(bases, ", "), a.From) } invocation := append([]string{"build", "-f", a.From, "-t", local}, args...) if a.Target != "" { invocation = append(invocation, "--target", a.Target) } if npmrc != "" { // Host network for the build, so a RUN reaching the package registry finds it where the // binding says it is — the machine's own loopback, where the registry answers. The // credential itself is in the context as .npmrc, COPY'd by a stage that is not published; // buildkit is not required, because this machine's docker may not carry buildx. invocation = append(invocation, "--network", "host") } invocation = append(invocation, ".") say("image", "docker build -f %s", a.From) if _, err := run(ctx, tree, "docker", invocation...); err != nil { return catalogue.Built{}, fmt.Errorf("%s: building %s failed: %w", module, a.Name, err) } say("image", "built, publishing") reference, err := publish.PublishImage(ctx, local, module+"/"+a.Name) if err != nil { return catalogue.Built{}, err } return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil case catalogue.ArtifactBundle: // **The one recipe that both builds and packs.** Everything else either produces an image // or packs what is already there; this compiles the module's own code first, in a // toolchain the mesh chose from what the module said it was written in, and packs the // result. // // The compiler runs in a container rather than on the build machine, for the reason every // other build does: what a build needs installed is the toolchain's business, and a build // machine that accumulated one toolchain per language would be a machine nobody could // reproduce. chain, err := ToolchainFor(a.Language) if err != nil { return catalogue.Built{}, fmt.Errorf("%s: %s: %w", module, a.Name, err) } base, ok := held[chain.Base+"/"+chain.Artifact] if !ok { // Named, not pinned: the mesh answers with the copy it holds. Refused before anything // is built, saying which module has to exist first, rather than failing inside a // compile with a message about an image (novox/hq 04-ISSUES/044). return catalogue.Built{}, fmt.Errorf( "%s: %s is written in %s, which is compiled by %s's %q artifact, and this mesh "+ "holds no copy of it. Build %s first", module, a.Name, chain.Language, chain.Base, chain.Artifact, chain.Base) } say("bundle", "compiling %s in %s's toolchain", a.Language, chain.Base) compiled, err := compile(ctx, run, tree, chain, base, a) if err != nil { return catalogue.Built{}, fmt.Errorf("%s: compiling %s failed: %w", module, a.Name, err) } say("bundle", "compiled, packing") body, err := pack(compiled) if err != nil { return catalogue.Built{}, fmt.Errorf("%s: packing %s failed: %w", module, a.Name, err) } sum := sha256.Sum256(body) digest := "sha256:" + hex.EncodeToString(sum[:]) where, err := publish.PublishArchive(ctx, module+"/"+a.Name, body, digest) if err != nil { return catalogue.Built{}, err } return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: where, Digest: digest}, nil case catalogue.ArtifactPackage: // Built and published on a public base, to the mesh's package registry, by version // (novox/hq ADR 0076). Not an image, not an archive: nothing on a machine references it, so // there is no Publisher call — the container itself publishes, with the credential the // build was handed. say("package", "building and publishing %s (%s)", a.Name, a.Language) reference, err := publishPackage(ctx, run, module, tree, a, npmrc, say) if err != nil { return catalogue.Built{}, fmt.Errorf("%s: publishing %s failed: %w", module, a.Name, err) } say("package", "published %s", reference) return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil case catalogue.ArtifactArchive: body, err := pack(filepath.Join(tree, a.From)) if err != nil { return catalogue.Built{}, fmt.Errorf("%s: packing %s failed: %w", module, a.Name, err) } sum := sha256.Sum256(body) digest := "sha256:" + hex.EncodeToString(sum[:]) where, err := publish.PublishArchive(ctx, module+"/"+a.Name, body, digest) if err != nil { return catalogue.Built{}, err } return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: where, Digest: digest}, nil } return catalogue.Built{}, fmt.Errorf("%s: %q is a %q, which is not something this builds", module, a.Name, a.Kind) } // pack tars and gzips a directory. // // **Deterministically**: entries sorted, and no timestamps, uid, gid or original names carried // through. Two builds of one commit must produce one digest, or nothing downstream can tell "this // changed" from "this was built again" — and every rebuild would look like a change to every // machine holding it. func pack(root string) ([]byte, error) { info, err := os.Stat(root) if err != nil { return nil, err } if !info.IsDir() { return nil, fmt.Errorf("%s is not a directory", root) } var paths []string err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { if err != nil { return err } if info.IsDir() || !info.Mode().IsRegular() { // Only files. A symlink or a device in an archive is refused by the host that unpacks // it, so putting one in would build something that cannot be applied. if !info.IsDir() && !info.Mode().IsRegular() { return fmt.Errorf("%s is neither a file nor a directory, and an archive carries "+ "only those", path) } return nil } paths = append(paths, path) return nil }) if err != nil { return nil, err } // filepath.Walk is documented to walk in lexical order, so this is belt and braces rather // than load-bearing — and no test distinguishes it, which is worth saying rather than // implying otherwise. It stays because the cost is nothing and the failure it guards against // is silent: an archive whose digest changes because the traversal did. sort.Strings(paths) var out strings.Builder zipped := gzip.NewWriter(&stringWriter{&out}) writer := tar.NewWriter(zipped) for _, path := range paths { body, err := os.ReadFile(path) if err != nil { return nil, err } relative, err := filepath.Rel(root, path) if err != nil { return nil, err } info, err := os.Stat(path) if err != nil { return nil, err } mode := int64(info.Mode().Perm()) if err := writer.WriteHeader(&tar.Header{ Name: filepath.ToSlash(relative), Mode: mode, Size: int64(len(body)), Typeflag: tar.TypeReg, // Everything else left at its zero value on purpose — see the note above. }); err != nil { return nil, err } if _, err := writer.Write(body); err != nil { return nil, err } } if err := writer.Close(); err != nil { return nil, err } if err := zipped.Close(); err != nil { return nil, err } return []byte(out.String()), nil } type stringWriter struct{ to *strings.Builder } func (w *stringWriter) Write(p []byte) (int, error) { return w.to.Write(p) } func short(commit string) string { if len(commit) > 8 { return commit[:8] } return commit } // Command is a Runner that actually runs things. func Command(ctx context.Context, dir, name string, args ...string) (string, error) { // **Every command is echoed before it runs**, with where. On a build that hangs, the last line // is exactly the command it is inside — which is the difference between "the builder did // nothing" and "git clone is waiting on a network that will not answer". Silent on success is // what made an empty workspace unreadable. started := timeNow() fmt.Fprintf(os.Stderr, " $ (%s) %s %s\n", short(filepath.Base(dir)), name, strings.Join(args, " ")) cmd := exec.CommandContext(ctx, name, args...) cmd.Dir = dir out, err := cmd.CombinedOutput() if err != nil { fmt.Fprintf(os.Stderr, " ! %s %s failed after %s\n", name, args[0], since(started)) return string(out), fmt.Errorf("%s %s: %w\n%s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out))) } fmt.Fprintf(os.Stderr, " ✓ %s %s (%s)\n", name, firstArg(args), since(started)) return string(out), nil } func firstArg(args []string) string { if len(args) == 0 { return "" } return args[0] } var _ io.Writer = (*stringWriter)(nil) // standingOn turns the bases a module named into build arguments for what this mesh holds. // // **Refused rather than defaulted** (novox/hq issue 044). A module naming a base the mesh has not // built cannot be built here yet, and the useful sentence names which module is missing — not the // one a container runtime produces when a recipe's first line refers to an image nobody has. // // The order is fixed so two builds of one commit invoke the same command. func standingOn(ctx context.Context, manifest catalogue.Manifest, held map[string]string, mirror func(ctx context.Context, from, repository string) (string, error)) ([]string, error) { if manifest.Build == nil || len(manifest.Build.On) == 0 { return nil, nil } on := append([]catalogue.BuildsOn{}, manifest.Build.On...) sort.Slice(on, func(i, j int) bool { return on[i].Arg < on[j].Arg }) var args []string for _, base := range on { if base.Image != "" { // A vendor's image, declared (novox/hq 04-ISSUES/064, ADR 0097). Pinned, because a tag // is what somebody else can move; copied into the mesh's registry, because a build // that reaches a public registry on its own is a build that works sometimes. if base.Arg == "" || base.Module != "" || base.Artifact != "" { return nil, fmt.Errorf( "%s stands on the image %s, and a base is either a module's artifact or an "+ "image — never both — read from one build argument", manifest.Module, base.Image) } if !strings.Contains(base.Image, "@sha256:") { return nil, fmt.Errorf( "%s stands on the image %q, which is not pinned by digest. A tag is what "+ "somebody else can move; name it as @sha256:…", manifest.Module, base.Image) } reference, err := mirror(ctx, base.Image, manifest.Module+"/on-"+strings.ToLower(base.Arg)) if err != nil { return nil, fmt.Errorf("%s stands on %s: %w", manifest.Module, base.Image, err) } args = append(args, "--build-arg", base.Arg+"="+reference) continue } if base.Arg == "" || base.Module == "" || base.Artifact == "" { return nil, fmt.Errorf( "%s says its build stands on something, and does not say all of what: a base "+ "needs the module, the artifact, and the build argument the recipe reads it "+ "from", manifest.Module) } key := base.Module + "/" + base.Artifact reference, has := held[key] if !has { return nil, fmt.Errorf( "%s builds on %s, and this mesh has not built it. Build %s first — every module "+ "in this toolchain stands on it, so it is the thing to have before anything "+ "else", manifest.Module, key, base.Module) } args = append(args, "--build-arg", base.Arg+"="+reference) } return args, nil } // compile runs a module's own code through its toolchain, and says where the result is. // // **In the module's own directory, under the path the toolchain expects.** A module is compiled // where its dependencies resolve upward into the base's own library directory, so what it is // compiled against is exactly what it will run against — the reason every hand-written Dockerfile // had to choose a working directory carefully, and the reason none of them has to now. // publishPackage builds a `package` artifact on a public base image and publishes it to the mesh's // package registry by version. The credential arrives as an .npmrc file the build was handed // (novox/hq ADR 0076); it is mounted read-only into the container rather than baked, because a // package build produces no image to leak it into. The reference returned is name@version, read from // the module's own package.json — the same two fields npm publishes under. func publishPackage(ctx context.Context, run Runner, module, dir string, a catalogue.Artifact, npmrc string, say func(step, format string, args ...any)) (string, error) { recipe, ok := packageRecipes[a.Language] if !ok { return "", fmt.Errorf( "a package written in %q cannot be built: no public toolchain is known for it", a.Language) } if npmrc == "" { // A package with nowhere to be published is not built. Said here rather than failing inside // npm publish with a message about a registry that is simply absent. return "", fmt.Errorf( "%s is a package and this build was given no package registry to publish it to", a.Name) } raw, err := os.ReadFile(filepath.Join(dir, "package.json")) if err != nil { return "", fmt.Errorf("a package is published by name and version, and %s has no package.json: %w", module, err) } var pkg struct { Name string `json:"name"` Version string `json:"version"` } if err := json.Unmarshal(raw, &pkg); err != nil { return "", fmt.Errorf("%s's package.json is not readable: %w", module, err) } if pkg.Name == "" || pkg.Version == "" { return "", fmt.Errorf("%s's package.json names no %s to publish under", module, either(pkg.Name == "", "name", "version")) } const within = "/app/module" invocation := []string{ "run", "--rm", // Host network, so the publish reaches the registry at the address the binding names. "--network", "host", "--volume", dir + ":" + within, // Read-only, so a build cannot alter the credential, and at /root where npm reads it. "--volume", npmrc + ":/root/.npmrc:ro", "--workdir", within, recipe.Base, "sh", "-c", recipe.Script, } if _, err := run(ctx, dir, "docker", invocation...); err != nil { return "", err } return pkg.Name + "@" + pkg.Version, nil } // either names whichever of two fields is the missing one, for a message that says which. func either(first bool, a, b string) string { if first { return a } return b } func compile(ctx context.Context, run Runner, tree string, chain Toolchain, base string, a catalogue.Artifact) (string, error) { // Where inside the toolchain the module's source is mounted, and where its output lands. Fixed // rather than configurable: a module that could move this would be describing its own build. const within = "/app/modules/module" // **Its own output directory, because a module may be several languages at once.** One module // is one piece of software and can still carry a daemon in one language, tools in another and // a package in a third (ADR 0040). Compiling them all into one place would have them overwrite // each other and then be packed together, so each bundle compiles and packs alone. out := Out(a.Name) invocation := []string{ "run", "--rm", "--volume", tree + ":" + within, "--workdir", within, base, } invocation = append(invocation, chain.Compile...) if chain.OutputFlag != "" { invocation = append(invocation, chain.OutputFlag, out) } // What to compile. Named by the module rather than discovered, so adding a file does not // silently change what a build produces. if len(a.Entrypoints) > 0 { invocation = append(invocation, sourcesFor(a.Entrypoints, out)...) } if _, err := run(ctx, tree, "docker", invocation...); err != nil { return "", err } return filepath.Join(tree, out), nil } // sourcesFor turns compiled entrypoints back into what to compile. // // A module names what a tool host should LOAD — compiled paths under the bundle's root — because // that is the thing anything else needs to know. What to compile is the same list with the // language's own extension, which is the toolchain's business rather than the module's. func sourcesFor(entrypoints []string, out string) []string { sources := make([]string, 0, len(entrypoints)) for _, e := range entrypoints { // An entrypoint is named as it will be FOUND — a path inside the unpacked bundle — so the // source is the same path with the output directory taken off the front and the language's // own extension on the end. at := strings.TrimPrefix(strings.TrimPrefix(e, out), "/") sources = append(sources, strings.TrimSuffix(at, filepath.Ext(at))+".ts") } return sources } func timeNow() time.Time { return time.Now() } func since(t time.Time) string { return time.Since(t).Round(time.Millisecond).String() } // undeclaredFetches is every image a recipe reaches for that is neither a declared build argument // nor one of its own stages nor `scratch`, in two lists: the bases it starts `FROM`, and the images // it `COPY --from`s out of — a vendor's tool, the case novox/hq 04-ISSUES/064 is about. func undeclaredFetches(recipe string, declared map[string]bool) (bases, copies []string) { stages := map[string]bool{} seen := map[string]bool{} var out *[]string note := func(ref string) { ref = strings.TrimSpace(ref) switch { case ref == "" || ref == "scratch" || stages[strings.ToLower(ref)]: return case strings.HasPrefix(ref, "$"): name := strings.Trim(strings.TrimPrefix(ref, "$"), "{}") if cut := strings.IndexAny(name, ":-"); cut >= 0 { name = name[:cut] } if !declared[name] { if !seen[ref] { seen[ref] = true *out = append(*out, ref+" (a build argument the manifest does not declare)") } } return } // A stage referenced by number (COPY --from=0) is its own recipe's. if _, err := strconv.Atoi(ref); err == nil { return } if !seen[ref] { seen[ref] = true *out = append(*out, ref) } } for _, raw := range strings.Split(recipe, "\n") { line := strings.TrimSpace(raw) if line == "" || strings.HasPrefix(line, "#") { continue } fields := strings.Fields(line) switch strings.ToUpper(fields[0]) { case "FROM": // FROM [--platform=…] [AS ] out = &bases var ref string for i := 1; i < len(fields); i++ { if strings.HasPrefix(fields[i], "--") { continue } ref = fields[i] if i+2 < len(fields) && strings.EqualFold(fields[i+1], "AS") { stages[strings.ToLower(fields[i+2])] = true } break } note(ref) case "COPY", "ADD": out = &copies for _, f := range fields[1:] { if strings.HasPrefix(f, "--from=") { note(strings.TrimPrefix(f, "--from=")) } } } } return bases, copies }