mesh-controller/internal/builder/builder.go

package builder

import (
	"archive/tar"
	"compress/gzip"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/novox/mesh-controller/internal/catalogue"
)

// Turning a repository into artifacts the mesh can pin.
//
// **This runs on a node, not in the control plane.** Building needs a container runtime and a
// working tree, and the control plane deliberately cannot run commands on a machine — what it may
// send is bounded by the declaration language (novox/hq ADR 0005), and "run this build" is not in
// it. So the builder is something a node runs *as a module*, given work over the broker like
// anything else, and this package is what it does when it gets some.
//
// The alternative — the control plane holding a docker socket — would make it the one component
// that can do anything on a machine, which is the property the whole design is arranged to avoid.

// Runner runs a command in a directory and returns what it said. Injected so the tests do not
// need docker and git, and so the failure of either is reported rather than assumed.
type Runner func(ctx context.Context, dir string, name string, args ...string) (string, error)

// Publisher puts an artifact somewhere a machine can fetch it, and says how to refer to it.
type Publisher interface {
	// PublishImage pushes a locally built image and returns a reference pinned by digest.
	PublishImage(ctx context.Context, localTag, repository string) (string, error)
	// PublishArchive stores bytes and returns where to fetch them from.
	PublishArchive(ctx context.Context, repository string, body []byte, digest string) (string, error)
}

// Result is everything one build produced.
type Result struct {
	// Against is every pinned image this build was built on top of, read out of its own inputs.
	//
	// **Derived, not declared** (novox/hq ADR 0009): a declared list of dependencies drifts from
	// what the code actually uses, and an artifact is out of date when anything it was built
	// against moved. These are artifact references rather than module-versions, because that is
	// what a build input names; resolving them to modules is the catalogue's work, since it is
	// what knows which module-version published which artifact.
	Against []string

	// Manifest is the module as the mesh should hold it: artifacts resolved to digests.
	Manifest catalogue.Manifest
	// Commit is what was built, so "is this current?" is answerable without building again.
	Commit string
	// Built is each artifact, for reporting.
	Built []catalogue.Built

	// Read is every repository this build read source from besides the module's own — the second
	// repository an artifact's recipe names (ArtifactContext). Reported because the manifest the
	// mesh keeps carries no build section, so nothing else could say that a merge there is a
	// change to this module (novox/hq 04-ISSUES/131).
	Read []catalogue.ArtifactContext
}

// GitCredential is the forge credential a clone may present when the server asks for one.
//
// **Offered, never pushed.** It is written as a git credential-store file and named to git with
// `-c credential.helper=store`, so git itself decides when it applies: only on an authentication
// challenge, and only for the URL it was written for — scheme, host and port included. A public
// repository clones exactly as before, and a repository on any other host is never shown it.
type GitCredential struct {
	// URL is the credential-store line — scheme://user:password@host[:port] — naming the one
	// server this credential belongs to. Empty means the builder holds none and every clone is
	// anonymous, as it always was.
	URL string
}

// Build clones a repository at a ref, reads its manifest, produces what it declares, publishes
// each, and returns the manifest the mesh should hold.
//
// **Nothing is published until everything is built.** A module whose image succeeded and whose
// archive failed would otherwise leave half of itself in the store under a digest the mesh never
// records — reachable, unreferenced, and indistinguishable from something in use.
func Build(ctx context.Context, run Runner, publish Publisher,
	repository, path, ref, workspace string, held map[string]string, npmrc Npmrc,
	forge GitCredential, log Log) (Result, error) {

	say := logging(log)
	say("clone", "%s%s at %s", repository, describePath(path), refOrHead(ref))

	// Made rather than required. A builder that fails because the directory it was told to work
	// in does not exist is a builder that needs a setup step nobody documented.
	if err := os.MkdirAll(workspace, 0o755); err != nil {
		return Result{}, err
	}
	// The credential is a file git reads, never an argument: a URL carrying a password in argv
	// would be readable by anything that can list processes for as long as a clone runs.
	credentials := ""
	if forge.URL != "" {
		credentials = filepath.Join(workspace, "git-credentials")
		if err := os.WriteFile(credentials, []byte(forge.URL+"\n"), 0o600); err != nil {
			return Result{}, err
		}
	}
	tree := filepath.Join(workspace, "source")
	if err := os.RemoveAll(tree); err != nil {
		return Result{}, err
	}
	// A fresh clone every time rather than a fetch into a tree that is already there. A build
	// that reuses a working tree can succeed because of something a previous build left behind,
	// and that is a build nobody can reproduce.
	if _, err := run(ctx, workspace, "git", cloneWith(credentials, "clone", "--quiet", repository, tree)...); err != nil {
		say("clone", "FAILED: %v", err)
		return Result{}, fmt.Errorf("cannot clone %s: %w", repository, err)
	}
	say("clone", "done")
	if ref != "" {
		if _, err := run(ctx, tree, "git", "checkout", "--quiet", ref); err != nil {
			return Result{}, fmt.Errorf("%s has no %s: %w", repository, ref, err)
		}
	}
	commit, err := run(ctx, tree, "git", "rev-parse", "HEAD")
	if err != nil {
		return Result{}, err
	}
	commit = strings.TrimSpace(commit)
	say("commit", "%s", short(commit))

	// A module is a repository and a path within it (novox/hq ADR 0069). The ordinary case is an
	// empty path, meaning the repository's root; a repository holding several modules names each
	// by its own directory, which is what the catalogue is and what the system this replaces has
	// always done.
	within, err := inside(tree, path)
	if err != nil {
		return Result{}, err
	}

	raw, err := os.ReadFile(filepath.Join(within, ManifestName))
	if err != nil {
		return Result{}, fmt.Errorf(
			"%s has no %s at %s, so there is nothing saying what it is: %w",
			repository, ManifestName, describe(path), err)
	}
	manifest, err := catalogue.ParseManifest(raw)
	if err != nil {
		say("manifest", "INVALID: %v", err)
		return Result{}, err
	}
	say("manifest", "%s v%s — %d artifact(s)", manifest.Module, manifest.Version, artifactCount(manifest))

	// A build-time credential, written into the build context as .npmrc, but ONLY for a module that
	// asks for it: a `package` artifact (which publishes), or an image whose Dockerfile COPYs .npmrc.
	// Writing it into every context would put a per-run credential in `COPY . .` of modules that
	// never resolve a mesh package — making their image non-deterministic (a needless rollout every
	// build) and leaking the credential into a build stage. Absent entirely with no registry, which
	// is the bootstrap case (novox/hq ADR 0076).
	var npmrcPath string
	if npmrc.Enabled() && manifest.Build != nil && wantsPackages(manifest, within) {
		content, err := npmrc.File()
		if err != nil {
			return Result{}, err
		}
		npmrcPath = filepath.Join(within, ".npmrc")
		if err := os.WriteFile(npmrcPath, []byte(content), 0o600); err != nil {
			return Result{}, fmt.Errorf("cannot write the package-registry credential for the build: %w", err)
		}
		say("packages", "resolving %s from the mesh's package registry", npmrc.Scope)
	}

	var built []catalogue.Built
	// stoodOn is every base the build was handed, as resolved — the edges the catalogue derives.
	var stoodOn []string
	if manifest.Build != nil {
		// What this module said it stands on, answered with what this mesh actually holds. Done
		// before anything is built, so a missing base is refused in front of the person who can
		// fix it rather than inside a build that stops on its own first line.
		// An image published elsewhere that the build stands on is copied into the mesh's own
		// registry first, like an upstream artifact (ADR 0096), and the recipe is handed the copy.
		// Genesis has nowhere to copy to and pulls it into this machine's store instead.
		mirror := func(ctx context.Context, from, repository string) (string, error) {
			if m, can := publish.(Mirrorer); can {
				say("bases", "copying %s into the mesh's registry", from)
				return m.MirrorImage(ctx, from, repository)
			}
			if _, err := run(ctx, tree, "docker", "pull", from); err != nil {
				return "", fmt.Errorf("cannot fetch %s: %w", from, err)
			}
			return from, nil
		}
		args, bases, err := standingOn(ctx, manifest, held, mirror)
		if err != nil {
			say("bases", "UNMET: %v", err)
			return Result{}, err
		}
		stoodOn = bases
		if len(args) > 0 {
			say("bases", "%d resolved from what the mesh holds", len(args)/2)
		}
		artifacts := append([]catalogue.Artifact{}, manifest.Build.Artifacts...)
		// Ordered, so two builds of one commit do the same work in the same sequence and their
		// logs can be compared.
		sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name })
		for _, a := range artifacts {
			say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a))
			made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, say)
			if err != nil {
				say("artifact", "%s FAILED: %v", a.Name, err)
				return Result{}, err
			}
			say("artifact", "%s done — %s", a.Name, describeMade(made))
			built = append(built, made)
		}
	}

	resolved, err := manifest.Resolve(built)
	if err != nil {
		say("resolve", "FAILED: %v", err)
		return Result{}, err
	}
	say("done", "%s at %s — %d artifact(s) pinned", manifest.Module, short(commit), len(built))
	return Result{Manifest: resolved, Commit: commit, Built: built,
		Against: against(within, manifest, stoodOn), Read: readBy(manifest)}, nil
}

// Log is where a build says what it is doing, step by step. Nil is silent — the tests pass none,
// and a build with nowhere to speak must still build.
type Log func(step, message string)

func logging(log Log) func(step, format string, args ...any) {
	if log == nil {
		return func(string, string, ...any) {}
	}
	return func(step, format string, args ...any) {
		log(step, fmt.Sprintf(format, args...))
	}
}

// contextFrom clones an image artifact's own build context, when it names one apart from this
// module's own repository — a fresh tree, the same way the module's own is, keyed by artifact
// name so two artifacts of one module naming different contexts do not collide.
func contextFrom(ctx context.Context, run Runner, workspace, artifact, credentials string,
	from catalogue.ArtifactContext, say func(step, format string, args ...any)) (string, error) {
	say("context", "cloning %s at %s for %s", from.Repository, refOrHead(from.Ref), artifact)
	dir := filepath.Join(workspace, "context-"+artifact)
	if err := os.RemoveAll(dir); err != nil {
		return "", err
	}
	if _, err := run(ctx, workspace, "git", cloneWith(credentials, "clone", "--quiet", from.Repository, dir)...); err != nil {
		return "", fmt.Errorf("cannot clone %s: %w", from.Repository, err)
	}
	if from.Ref != "" {
		if _, err := run(ctx, dir, "git", "checkout", "--quiet", from.Ref); err != nil {
			return "", fmt.Errorf("%s has no %s: %w", from.Repository, from.Ref, err)
		}
	}
	say("context", "done")
	return dir, nil
}

// cloneWith is a git invocation that may offer a stored credential.
//
// The first `-c credential.helper=` clears every helper the environment might carry, so exactly
// one place answers an authentication challenge: the file the builder wrote. Without a file, the
// invocation is exactly what it always was.
func cloneWith(credentials string, rest ...string) []string {
	if credentials == "" {
		return rest
	}
	return append([]string{
		"-c", "credential.helper=",
		"-c", "credential.helper=store --file=" + credentials,
	}, rest...)
}

func describePath(path string) string {
	if path == "" {
		return ""
	}
	return " at " + path
}

func refOrHead(ref string) string {
	if ref == "" {
		return "HEAD"
	}
	return ref
}

func artifactCount(m catalogue.Manifest) int {
	if m.Build == nil {
		return 0
	}
	return len(m.Build.Artifacts)
}

func langSuffix(a catalogue.Artifact) string {
	if a.Language != "" {
		return ", " + a.Language
	}
	return ""
}

func describeMade(made catalogue.Built) string {
	if made.Digest != "" {
		return made.Kind + " " + short(strings.TrimPrefix(made.Digest, "sha256:"))
	}
	return made.Kind + " " + made.Reference
}

// inside resolves a module's path within a clone, and refuses one that leaves it.
//
// **A build reads only its own tree.** A path of `../../etc` would otherwise make a build read —
// and an archive artifact publish — whatever the build machine happens to hold, which is the one
// thing a machine that builds other people's repositories must not do.
func inside(tree, path string) (string, error) {
	if path == "" {
		return tree, nil
	}
	if filepath.IsAbs(path) {
		return "", fmt.Errorf(
			"a module's path is inside its repository, and %q is an absolute path", path)
	}
	within := filepath.Join(tree, path)
	rel, err := filepath.Rel(tree, within)
	if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
		return "", fmt.Errorf(
			"%q leaves the repository, and a build reads only its own tree", path)
	}
	return within, nil
}

// describe says where a manifest was looked for, in words a person can act on.
func describe(path string) string {
	if path == "" {
		return "its root"
	}
	return path
}

// pinnedImage matches an image reference pinned by digest, which is the only kind a build input is
// allowed to name — a tag is something somebody else can move under you.
var pinnedImage = regexp.MustCompile(`[A-Za-z0-9][A-Za-z0-9._/:-]*@sha256:[0-9a-f]{64}`)

// against is what this module's image artifacts are built on top of: every base the mesh resolved
// and handed the recipe as a build argument (`build.on`), and any image a recipe pins by digest
// itself. Nothing is guessed: a reference that was neither resolved nor written down is not
// reported.
//
// **The resolved bases are the edges.** A recipe reads its base from an argument (`FROM
// ${RUNTIME_BASE}`), so the digest is never in the file, and a derivation that read files alone
// recorded no edge for any module on the mesh — which is why nothing knew what a changed base
// meant to rebuild (novox/hq 04-ISSUES/131).
func against(within string, manifest catalogue.Manifest, resolved []string) []string {
	if manifest.Build == nil {
		return nil
	}
	seen := map[string]bool{}
	var out []string
	for _, r := range resolved {
		if r != "" && !seen[r] {
			seen[r] = true
			out = append(out, r)
		}
	}
	for _, a := range manifest.Build.Artifacts {
		if a.Kind != catalogue.ArtifactImage || a.From == "" {
			continue
		}
		body, err := os.ReadFile(filepath.Join(within, a.From))
		if err != nil {
			// Not fatal: the build itself already failed if this file was needed and missing, and
			// reporting no edges is honest where inventing them would not be.
			continue
		}
		for _, found := range pinnedImage.FindAllString(string(body), -1) {
			if !seen[found] {
				seen[found] = true
				out = append(out, found)
			}
		}
	}
	sort.Strings(out)
	return out
}

// ManifestName is the one file a module repository must have.
//
// At the root, and named the same in every repository. A convention somebody can look for beats a
// setting somebody has to find.
const ManifestName = "module.json"

// wantsPackages reports whether this module's build resolves anything from the mesh's package
// registry, so the credential is written into its context only then. A package artifact always
// does; an image does when its Dockerfile names .npmrc — the file it would COPY to authenticate.
func wantsPackages(manifest catalogue.Manifest, within string) bool {
	for _, a := range manifest.Build.Artifacts {
		switch a.Kind {
		case catalogue.ArtifactPackage:
			return true
		case catalogue.ArtifactImage:
			raw, err := os.ReadFile(filepath.Join(within, a.From))
			if err == nil && strings.Contains(string(raw), ".npmrc") {
				return true
			}
		}
	}
	return false
}

func one(ctx context.Context, run Runner, publish Publisher,
	module, tree, workspace, commit, credentials string, a catalogue.Artifact, args []string,
	held map[string]string, npmrc string, say func(step, format string, args ...any)) (catalogue.Built, error) {

	switch a.Kind {
	case catalogue.ArtifactUpstream:
		// Mirrored, not built: copied under a name of the mesh's own, so what a machine fetches is
		// pinned by a digest this registry assigned rather than by a tag somebody else can move.
		//
		// **Between registries, never through this machine's image store** (novox/hq
		// 04-ISSUES/046, ADR 0096). A published image is an index over several architectures;
		// pulled, the store keeps the index and refuses to push one platform out of it, and
		// every variant of pull-then-push failed the same way. A copy moves what is there.
		if mirror, can := publish.(Mirrorer); can {
			say("mirror", "copying %s into the mesh's registry", a.From)
			reference, err := mirror.MirrorImage(ctx, a.From, module+"/"+a.Name)
			if err != nil {
				return catalogue.Built{}, fmt.Errorf("%s: %w", module, err)
			}
			return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil
		}
		// Genesis has no registry to copy into: the image stays in this machine's store, named by
		// its own id, as every artifact does before there is anywhere to publish.
		say("mirror", "pulling %s", a.From)
		if _, err := run(ctx, tree, "docker", "pull", a.From); err != nil {
			return catalogue.Built{}, fmt.Errorf("%s: cannot fetch %s: %w", module, a.From, err)
		}
		reference, err := publish.PublishImage(ctx, a.From, module+"/"+a.Name)
		if err != nil {
			return catalogue.Built{}, err
		}
		return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil

	case catalogue.ArtifactImage:
		// Tagged by commit rather than by version, because a version is what a person calls a
		// release and a commit is what was actually built. The mesh pins the digest anyway; this
		// is only so a person looking at the build node can tell what is there.
		local := fmt.Sprintf("%s-%s:%s", module, a.Name, short(commit))
		// The bases this module named, resolved to what this mesh holds. A recipe reads them as
		// build arguments, so a module says which module it stands on and never which copy.
		// **A recipe fetches nothing the manifest did not declare** (novox/hq 04-ISSUES/064). A FROM
		// or a COPY --from naming a registry image that is not a declared base is a build that
		// reaches a public registry on its own — and works when that registry answers, which is
		// sometimes. Refused here, in front of the person who can declare it, not inside a build
		// that fails with "pull access denied" for a reason that is not the mesh's.
		recipe, err := os.ReadFile(filepath.Join(tree, a.From))
		if err != nil {
			return catalogue.Built{}, fmt.Errorf("%s: cannot read the recipe %s: %w", module, a.From, err)
		}
		declared := map[string]bool{}
		for i := 0; i+1 < len(args); i += 2 {
			if args[i] == "--build-arg" {
				declared[strings.SplitN(args[i+1], "=", 2)[0]] = true
			}
		}
		bases, copies := undeclaredFetches(string(recipe), declared)
		if len(copies) > 0 {
			return catalogue.Built{}, fmt.Errorf(
				"%s: the recipe %s copies out of %s, which the manifest does not declare. A build "+
					"reaching a public registry on its own works only when that registry answers; "+
					"declare it under build.on as {\"arg\": \"<NAME>\", \"image\": \"<image>@sha256:…\"} "+
					"and read it from that argument (novox/hq ADR 0097)",
				module, a.From, strings.Join(copies, ", "))
		}
		if len(bases) > 0 {
			// Refused, since the mesh's own images declare theirs (ADR 0097): a base fetched on
			// its own is a build that works when a public registry answers, which is sometimes.
			return catalogue.Built{}, fmt.Errorf(
				"%s: the recipe %s starts FROM %s, which the manifest does not declare. Declare "+
					"each under build.on as {\"arg\": \"<NAME>\", \"image\": \"<image>@sha256:…\"} "+
					"and start FROM ${<NAME>} (novox/hq ADR 0097)",
				module, a.From, strings.Join(bases, ", "))
		}
		// The recipe is always read from this module's own tree, at this module's own commit — only
		// the context docker build's final argument names can come from somewhere else, when the
		// artifact says so.
		recipePath := a.From
		buildDir := tree
		if a.Context != nil {
			cloned, err := contextFrom(ctx, run, workspace, a.Name, credentials, *a.Context, say)
			if err != nil {
				return catalogue.Built{}, fmt.Errorf("%s: %s's context: %w", module, a.Name, err)
			}
			// docker build accepts -f outside the context it is given; the recipe stays exactly
			// where it was read from and validated against, absolute so the working directory
			// switching to the cloned context does not change which file that is.
			absRecipe, err := filepath.Abs(filepath.Join(tree, a.From))
			if err != nil {
				return catalogue.Built{}, fmt.Errorf("%s: %s's recipe: %w", module, a.Name, err)
			}
			recipePath = absRecipe
			buildDir = cloned
		}
		invocation := append([]string{"build", "-f", recipePath, "-t", local}, args...)
		if a.Target != "" {
			invocation = append(invocation, "--target", a.Target)
		}
		if npmrc != "" {
			// Host network for the build, so a RUN reaching the package registry finds it where the
			// binding says it is — the machine's own loopback, where the registry answers. The
			// credential itself is in the context as .npmrc, COPY'd by a stage that is not published;
			// buildkit is not required, because this machine's docker may not carry buildx.
			invocation = append(invocation, "--network", "host")
		}
		invocation = append(invocation, ".")
		say("image", "docker build -f %s", recipePath)
		if _, err := run(ctx, buildDir, "docker", invocation...); err != nil {
			return catalogue.Built{}, fmt.Errorf("%s: building %s failed: %w", module, a.Name, err)
		}
		say("image", "built, publishing")
		reference, err := publish.PublishImage(ctx, local, module+"/"+a.Name)
		if err != nil {
			return catalogue.Built{}, err
		}
		return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil

	case catalogue.ArtifactBundle:
		// **The one recipe that both builds and packs.** Everything else either produces an image
		// or packs what is already there; this compiles the module's own code first, in a
		// toolchain the mesh chose from what the module said it was written in, and packs the
		// result.
		//
		// The compiler runs in a container rather than on the build machine, for the reason every
		// other build does: what a build needs installed is the toolchain's business, and a build
		// machine that accumulated one toolchain per language would be a machine nobody could
		// reproduce.
		chain, err := ToolchainFor(a.Language)
		if err != nil {
			return catalogue.Built{}, fmt.Errorf("%s: %s: %w", module, a.Name, err)
		}
		base, ok := held[chain.Base+"/"+chain.Artifact]
		if !ok {
			// Named, not pinned: the mesh answers with the copy it holds. Refused before anything
			// is built, saying which module has to exist first, rather than failing inside a
			// compile with a message about an image (novox/hq 04-ISSUES/044).
			return catalogue.Built{}, fmt.Errorf(
				"%s: %s is written in %s, which is compiled by %s's %q artifact, and this mesh "+
					"holds no copy of it. Build %s first",
				module, a.Name, chain.Language, chain.Base, chain.Artifact, chain.Base)
		}
		say("bundle", "compiling %s in %s's toolchain", a.Language, chain.Base)
		compiled, err := compile(ctx, run, tree, chain, base, a)
		if err != nil {
			return catalogue.Built{}, fmt.Errorf("%s: compiling %s failed: %w", module, a.Name, err)
		}
		say("bundle", "compiled, packing")
		body, err := pack(compiled)
		if err != nil {
			return catalogue.Built{}, fmt.Errorf("%s: packing %s failed: %w", module, a.Name, err)
		}
		sum := sha256.Sum256(body)
		digest := "sha256:" + hex.EncodeToString(sum[:])
		where, err := publish.PublishArchive(ctx, module+"/"+a.Name, body, digest)
		if err != nil {
			return catalogue.Built{}, err
		}
		return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: where, Digest: digest}, nil

	case catalogue.ArtifactPackage:
		// Built and published on a public base, to the mesh's package registry, by version
		// (novox/hq ADR 0076). Not an image, not an archive: nothing on a machine references it, so
		// there is no Publisher call — the container itself publishes, with the credential the
		// build was handed.
		say("package", "building and publishing %s (%s)", a.Name, a.Language)
		reference, err := publishPackage(ctx, run, module, tree, a, npmrc, say)
		if err != nil {
			return catalogue.Built{}, fmt.Errorf("%s: publishing %s failed: %w", module, a.Name, err)
		}
		say("package", "published %s", reference)
		return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil

	case catalogue.ArtifactArchive:
		body, err := pack(filepath.Join(tree, a.From))
		if err != nil {
			return catalogue.Built{}, fmt.Errorf("%s: packing %s failed: %w", module, a.Name, err)
		}
		sum := sha256.Sum256(body)
		digest := "sha256:" + hex.EncodeToString(sum[:])
		where, err := publish.PublishArchive(ctx, module+"/"+a.Name, body, digest)
		if err != nil {
			return catalogue.Built{}, err
		}
		return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: where, Digest: digest}, nil
	}
	return catalogue.Built{}, fmt.Errorf("%s: %q is a %q, which is not something this builds",
		module, a.Name, a.Kind)
}

// pack tars and gzips a directory.
//
// **Deterministically**: entries sorted, and no timestamps, uid, gid or original names carried
// through. Two builds of one commit must produce one digest, or nothing downstream can tell "this
// changed" from "this was built again" — and every rebuild would look like a change to every
// machine holding it.
func pack(root string) ([]byte, error) {
	info, err := os.Stat(root)
	if err != nil {
		return nil, err
	}
	if !info.IsDir() {
		return nil, fmt.Errorf("%s is not a directory", root)
	}

	var paths []string
	err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() || !info.Mode().IsRegular() {
			// Only files. A symlink or a device in an archive is refused by the host that unpacks
			// it, so putting one in would build something that cannot be applied.
			if !info.IsDir() && !info.Mode().IsRegular() {
				return fmt.Errorf("%s is neither a file nor a directory, and an archive carries "+
					"only those", path)
			}
			return nil
		}
		paths = append(paths, path)
		return nil
	})
	if err != nil {
		return nil, err
	}
	// filepath.Walk is documented to walk in lexical order, so this is belt and braces rather
	// than load-bearing — and no test distinguishes it, which is worth saying rather than
	// implying otherwise. It stays because the cost is nothing and the failure it guards against
	// is silent: an archive whose digest changes because the traversal did.
	sort.Strings(paths)

	var out strings.Builder
	zipped := gzip.NewWriter(&stringWriter{&out})
	writer := tar.NewWriter(zipped)
	for _, path := range paths {
		body, err := os.ReadFile(path)
		if err != nil {
			return nil, err
		}
		relative, err := filepath.Rel(root, path)
		if err != nil {
			return nil, err
		}
		info, err := os.Stat(path)
		if err != nil {
			return nil, err
		}
		mode := int64(info.Mode().Perm())
		if err := writer.WriteHeader(&tar.Header{
			Name: filepath.ToSlash(relative), Mode: mode, Size: int64(len(body)),
			Typeflag: tar.TypeReg,
			// Everything else left at its zero value on purpose — see the note above.
		}); err != nil {
			return nil, err
		}
		if _, err := writer.Write(body); err != nil {
			return nil, err
		}
	}
	if err := writer.Close(); err != nil {
		return nil, err
	}
	if err := zipped.Close(); err != nil {
		return nil, err
	}
	return []byte(out.String()), nil
}

type stringWriter struct{ to *strings.Builder }

func (w *stringWriter) Write(p []byte) (int, error) { return w.to.Write(p) }

func short(commit string) string {
	if len(commit) > 8 {
		return commit[:8]
	}
	return commit
}

// Command is a Runner that actually runs things.
func Command(ctx context.Context, dir, name string, args ...string) (string, error) {
	// **Every command is echoed before it runs**, with where. On a build that hangs, the last line
	// is exactly the command it is inside — which is the difference between "the builder did
	// nothing" and "git clone is waiting on a network that will not answer". Silent on success is
	// what made an empty workspace unreadable.
	started := timeNow()
	fmt.Fprintf(os.Stderr, "  $ (%s) %s %s\n", short(filepath.Base(dir)), name, strings.Join(args, " "))
	cmd := exec.CommandContext(ctx, name, args...)
	cmd.Dir = dir
	out, err := cmd.CombinedOutput()
	if err != nil {
		fmt.Fprintf(os.Stderr, "  ! %s %s failed after %s\n", name, args[0], since(started))
		return string(out), fmt.Errorf("%s %s: %w\n%s",
			name, strings.Join(args, " "), err, strings.TrimSpace(string(out)))
	}
	fmt.Fprintf(os.Stderr, "  ✓ %s %s (%s)\n", name, firstArg(args), since(started))
	return string(out), nil
}

func firstArg(args []string) string {
	if len(args) == 0 {
		return ""
	}
	return args[0]
}

var _ io.Writer = (*stringWriter)(nil)

// standingOn turns the bases a module named into build arguments for what this mesh holds.
//
// **Refused rather than defaulted** (novox/hq issue 044). A module naming a base the mesh has not
// built cannot be built here yet, and the useful sentence names which module is missing — not the
// one a container runtime produces when a recipe's first line refers to an image nobody has.
//
// The order is fixed so two builds of one commit invoke the same command. Returned alongside the
// arguments is every reference they resolved to, which is what the build stood on.
func standingOn(ctx context.Context, manifest catalogue.Manifest, held map[string]string,
	mirror func(ctx context.Context, from, repository string) (string, error)) ([]string, []string, error) {
	if manifest.Build == nil || len(manifest.Build.On) == 0 {
		return nil, nil, nil
	}
	on := append([]catalogue.BuildsOn{}, manifest.Build.On...)
	sort.Slice(on, func(i, j int) bool { return on[i].Arg < on[j].Arg })

	var args, resolved []string
	for _, base := range on {
		if base.Image != "" {
			// A vendor's image, declared (novox/hq 04-ISSUES/064, ADR 0097). Pinned, because a tag
			// is what somebody else can move; copied into the mesh's registry, because a build
			// that reaches a public registry on its own is a build that works sometimes.
			if base.Arg == "" || base.Module != "" || base.Artifact != "" {
				return nil, nil, fmt.Errorf(
					"%s stands on the image %s, and a base is either a module's artifact or an "+
						"image — never both — read from one build argument", manifest.Module, base.Image)
			}
			if !strings.Contains(base.Image, "@sha256:") {
				return nil, nil, fmt.Errorf(
					"%s stands on the image %q, which is not pinned by digest. A tag is what "+
						"somebody else can move; name it as <image>@sha256:…", manifest.Module, base.Image)
			}
			reference, err := mirror(ctx, base.Image, manifest.Module+"/on-"+strings.ToLower(base.Arg))
			if err != nil {
				return nil, nil, fmt.Errorf("%s stands on %s: %w", manifest.Module, base.Image, err)
			}
			args = append(args, "--build-arg", base.Arg+"="+reference)
			resolved = append(resolved, reference)
			continue
		}
		if base.Arg == "" || base.Module == "" || base.Artifact == "" {
			return nil, nil, fmt.Errorf(
				"%s says its build stands on something, and does not say all of what: a base "+
					"needs the module, the artifact, and the build argument the recipe reads it "+
					"from", manifest.Module)
		}
		key := base.Module + "/" + base.Artifact
		reference, has := held[key]
		if !has {
			return nil, nil, fmt.Errorf(
				"%s builds on %s, and this mesh has not built it. Build %s first — every module "+
					"in this toolchain stands on it, so it is the thing to have before anything "+
					"else", manifest.Module, key, base.Module)
		}
		args = append(args, "--build-arg", base.Arg+"="+reference)
		resolved = append(resolved, reference)
	}
	return args, resolved, nil
}

// compile runs a module's own code through its toolchain, and says where the result is.
//
// **In the module's own directory, under the path the toolchain expects.** A module is compiled
// where its dependencies resolve upward into the base's own library directory, so what it is
// compiled against is exactly what it will run against — the reason every hand-written Dockerfile
// had to choose a working directory carefully, and the reason none of them has to now.
// publishPackage builds a `package` artifact on a public base image and publishes it to the mesh's
// package registry by version. The credential arrives as an .npmrc file the build was handed
// (novox/hq ADR 0076); it is mounted read-only into the container rather than baked, because a
// package build produces no image to leak it into. The reference returned is name@version, read from
// the module's own package.json — the same two fields npm publishes under.
func publishPackage(ctx context.Context, run Runner, module, dir string, a catalogue.Artifact,
	npmrc string, say func(step, format string, args ...any)) (string, error) {

	recipe, ok := packageRecipes[a.Language]
	if !ok {
		return "", fmt.Errorf(
			"a package written in %q cannot be built: no public toolchain is known for it", a.Language)
	}
	if npmrc == "" {
		// A package with nowhere to be published is not built. Said here rather than failing inside
		// npm publish with a message about a registry that is simply absent.
		return "", fmt.Errorf(
			"%s is a package and this build was given no package registry to publish it to", a.Name)
	}

	raw, err := os.ReadFile(filepath.Join(dir, "package.json"))
	if err != nil {
		return "", fmt.Errorf("a package is published by name and version, and %s has no package.json: %w", module, err)
	}
	var pkg struct {
		Name    string `json:"name"`
		Version string `json:"version"`
	}
	if err := json.Unmarshal(raw, &pkg); err != nil {
		return "", fmt.Errorf("%s's package.json is not readable: %w", module, err)
	}
	if pkg.Name == "" || pkg.Version == "" {
		return "", fmt.Errorf("%s's package.json names no %s to publish under",
			module, either(pkg.Name == "", "name", "version"))
	}

	const within = "/app/module"
	invocation := []string{
		"run", "--rm",
		// Host network, so the publish reaches the registry at the address the binding names.
		"--network", "host",
		"--volume", dir + ":" + within,
		// Read-only, so a build cannot alter the credential, and at /root where npm reads it.
		"--volume", npmrc + ":/root/.npmrc:ro",
		"--workdir", within,
		recipe.Base,
		"sh", "-c", recipe.Script,
	}
	if _, err := run(ctx, dir, "docker", invocation...); err != nil {
		return "", err
	}
	return pkg.Name + "@" + pkg.Version, nil
}

// either names whichever of two fields is the missing one, for a message that says which.
func either(first bool, a, b string) string {
	if first {
		return a
	}
	return b
}

func compile(ctx context.Context, run Runner, tree string, chain Toolchain,
	base string, a catalogue.Artifact) (string, error) {

	// Where inside the toolchain the module's source is mounted, and where its output lands. Fixed
	// rather than configurable: a module that could move this would be describing its own build.
	const within = "/app/modules/module"

	// **Its own output directory, because a module may be several languages at once.** One module
	// is one piece of software and can still carry a daemon in one language, tools in another and
	// a package in a third (ADR 0040). Compiling them all into one place would have them overwrite
	// each other and then be packed together, so each bundle compiles and packs alone.
	out := Out(a.Name)

	invocation := []string{
		"run", "--rm",
		"--volume", tree + ":" + within,
		"--workdir", within,
		base,
	}
	invocation = append(invocation, chain.Compile...)
	if chain.OutputFlag != "" {
		invocation = append(invocation, chain.OutputFlag, out)
	}
	// What to compile. Named by the module rather than discovered, so adding a file does not
	// silently change what a build produces.
	if len(a.Entrypoints) > 0 {
		invocation = append(invocation, sourcesFor(a.Entrypoints, out)...)
	}
	if _, err := run(ctx, tree, "docker", invocation...); err != nil {
		return "", err
	}
	return filepath.Join(tree, out), nil
}

// sourcesFor turns compiled entrypoints back into what to compile.
//
// A module names what a tool host should LOAD — compiled paths under the bundle's root — because
// that is the thing anything else needs to know. What to compile is the same list with the
// language's own extension, which is the toolchain's business rather than the module's.
func sourcesFor(entrypoints []string, out string) []string {
	sources := make([]string, 0, len(entrypoints))
	for _, e := range entrypoints {
		// An entrypoint is named as it will be FOUND — a path inside the unpacked bundle — so the
		// source is the same path with the output directory taken off the front and the language's
		// own extension on the end.
		at := strings.TrimPrefix(strings.TrimPrefix(e, out), "/")
		sources = append(sources, strings.TrimSuffix(at, filepath.Ext(at))+".ts")
	}
	return sources
}

func timeNow() time.Time       { return time.Now() }
func since(t time.Time) string { return time.Since(t).Round(time.Millisecond).String() }

// undeclaredFetches is every image a recipe reaches for that is neither a declared build argument
// nor one of its own stages nor `scratch`, in two lists: the bases it starts `FROM`, and the images
// it `COPY --from`s out of — a vendor's tool, the case novox/hq 04-ISSUES/064 is about.
func undeclaredFetches(recipe string, declared map[string]bool) (bases, copies []string) {
	stages := map[string]bool{}
	seen := map[string]bool{}
	var out *[]string
	note := func(ref string) {
		ref = strings.TrimSpace(ref)
		switch {
		case ref == "" || ref == "scratch" || stages[strings.ToLower(ref)]:
			return
		case strings.HasPrefix(ref, "$"):
			name := strings.Trim(strings.TrimPrefix(ref, "$"), "{}")
			if cut := strings.IndexAny(name, ":-"); cut >= 0 {
				name = name[:cut]
			}
			if !declared[name] {
				if !seen[ref] {
					seen[ref] = true
					*out = append(*out, ref+" (a build argument the manifest does not declare)")
				}
			}
			return
		}
		// A stage referenced by number (COPY --from=0) is its own recipe's.
		if _, err := strconv.Atoi(ref); err == nil {
			return
		}
		if !seen[ref] {
			seen[ref] = true
			*out = append(*out, ref)
		}
	}
	for _, line := range instructions(recipe) {
		fields := strings.Fields(line)
		switch strings.ToUpper(fields[0]) {
		case "FROM":
			// FROM [--platform=…] <ref> [AS <name>]
			out = &bases
			var ref string
			for i := 1; i < len(fields); i++ {
				if strings.HasPrefix(fields[i], "--") {
					continue
				}
				ref = fields[i]
				if i+2 < len(fields) && strings.EqualFold(fields[i+1], "AS") {
					stages[strings.ToLower(fields[i+2])] = true
				}
				break
			}
			note(ref)
		case "COPY", "ADD":
			out = &copies
			for _, f := range fields[1:] {
				if strings.HasPrefix(f, "--from=") {
					note(strings.TrimPrefix(f, "--from="))
				}
			}
		case "RUN":
			// RUN --mount=type=bind,from=<image>,… reaches for an image exactly as COPY --from does.
			out = &copies
			for _, f := range fields[1:] {
				if !strings.HasPrefix(f, "--mount=") {
					continue
				}
				for _, opt := range strings.Split(strings.TrimPrefix(f, "--mount="), ",") {
					if from, found := strings.CutPrefix(opt, "from="); found {
						note(from)
					}
				}
			}
		}
	}
	return bases, copies
}

// instructions is a recipe as its instructions, one per line: continuations joined, comments and
// blank lines dropped, and heredoc bodies (`COPY <<EOF … EOF`) skipped — a Python file written into
// an image is not a list of images to fetch. The review found a `COPY \` continued onto the next
// line slip past the check, and a stage named on a continuation line refused as a fetch.
func instructions(recipe string) []string {
	var out []string
	var current strings.Builder
	var heredoc string
	flush := func() {
		if line := strings.TrimSpace(current.String()); line != "" && !strings.HasPrefix(line, "#") {
			out = append(out, line)
		}
		current.Reset()
	}
	for _, raw := range strings.Split(recipe, "\n") {
		if heredoc != "" {
			if strings.TrimSpace(raw) == heredoc {
				heredoc = ""
			}
			continue
		}
		line := strings.TrimRight(raw, " \t")
		if strings.HasPrefix(strings.TrimSpace(line), "#") && current.Len() == 0 {
			continue
		}
		if strings.HasSuffix(line, "\\") {
			current.WriteString(strings.TrimSuffix(line, "\\"))
			current.WriteString(" ")
			continue
		}
		current.WriteString(line)
		if at := strings.Index(current.String(), "<<"); at >= 0 {
			// `<<EOF`, `<<-EOF`, `<<'EOF'`, `<<"EOF"`: the body runs to a line that is the word.
			word := strings.Fields(current.String()[at+2:])
			if len(word) > 0 {
				heredoc = strings.Trim(strings.TrimPrefix(word[0], "-"), `'"`)
			}
		}
		flush()
	}
	flush()
	return out
}

// readBy is every repository other than the module's own that this build's recipes read source from,
// each once and in a fixed order, so two builds of one commit report the same thing the same way.
func readBy(manifest catalogue.Manifest) []catalogue.ArtifactContext {
	if manifest.Build == nil {
		return nil
	}
	seen := map[string]bool{}
	var out []catalogue.ArtifactContext
	for _, a := range manifest.Build.Artifacts {
		if a.Context == nil || a.Context.Repository == "" {
			continue
		}
		key := a.Context.Repository + "#" + a.Context.Ref
		if seen[key] {
			continue
		}
		seen[key] = true
		out = append(out, *a.Context)
	}
	sort.Slice(out, func(i, j int) bool {
		if out[i].Repository != out[j].Repository {
			return out[i].Repository < out[j].Repository
		}
		return out[i].Ref < out[j].Ref
	})
	return out
}