A new 'package' artifact kind builds a module's own code on a public base image and publishes it to the mesh's package registry by version (hq ADR 0076) — the SDK above all, which the toolchain is built from and so cannot be built in the toolchain. The credential a build needs to resolve or publish packages is rendered as an .npmrc (basic auth, hq ADR 0048) and given to an image build as a buildkit secret, never a layer, so a token is not baked into the toolchain image. Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx
693 lines
27 KiB
Go
693 lines
27 KiB
Go
package builder
|
|
|
|
import (
|
|
"archive/tar"
|
|
"compress/gzip"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-control/internal/catalogue"
|
|
)
|
|
|
|
// Turning a repository into artifacts the mesh can pin.
|
|
//
|
|
// **This runs on a node, not in the control plane.** Building needs a container runtime and a
|
|
// working tree, and the control plane deliberately cannot run commands on a machine — what it may
|
|
// send is bounded by the declaration language (novox/hq ADR 0005), and "run this build" is not in
|
|
// it. So the builder is something a node runs *as a module*, given work over the broker like
|
|
// anything else, and this package is what it does when it gets some.
|
|
//
|
|
// The alternative — the control plane holding a docker socket — would make it the one component
|
|
// that can do anything on a machine, which is the property the whole design is arranged to avoid.
|
|
|
|
// Runner runs a command in a directory and returns what it said. Injected so the tests do not
|
|
// need docker and git, and so the failure of either is reported rather than assumed.
|
|
type Runner func(ctx context.Context, dir string, name string, args ...string) (string, error)
|
|
|
|
// Publisher puts an artifact somewhere a machine can fetch it, and says how to refer to it.
|
|
type Publisher interface {
|
|
// PublishImage pushes a locally built image and returns a reference pinned by digest.
|
|
PublishImage(ctx context.Context, localTag, repository string) (string, error)
|
|
// PublishArchive stores bytes and returns where to fetch them from.
|
|
PublishArchive(ctx context.Context, repository string, body []byte, digest string) (string, error)
|
|
}
|
|
|
|
// Result is everything one build produced.
|
|
type Result struct {
|
|
// Against is every pinned image this build was built on top of, read out of its own inputs.
|
|
//
|
|
// **Derived, not declared** (novox/hq ADR 0009): a declared list of dependencies drifts from
|
|
// what the code actually uses, and an artifact is out of date when anything it was built
|
|
// against moved. These are artifact references rather than module-versions, because that is
|
|
// what a build input names; resolving them to modules is the catalogue's work, since it is
|
|
// what knows which module-version published which artifact.
|
|
Against []string
|
|
|
|
// Manifest is the module as the mesh should hold it: artifacts resolved to digests.
|
|
Manifest catalogue.Manifest
|
|
// Commit is what was built, so "is this current?" is answerable without building again.
|
|
Commit string
|
|
// Built is each artifact, for reporting.
|
|
Built []catalogue.Built
|
|
}
|
|
|
|
// Build clones a repository at a ref, reads its manifest, produces what it declares, publishes
|
|
// each, and returns the manifest the mesh should hold.
|
|
//
|
|
// **Nothing is published until everything is built.** A module whose image succeeded and whose
|
|
// archive failed would otherwise leave half of itself in the store under a digest the mesh never
|
|
// records — reachable, unreferenced, and indistinguishable from something in use.
|
|
func Build(ctx context.Context, run Runner, publish Publisher,
|
|
repository, path, ref, workspace string, held map[string]string, npmrc Npmrc, log Log) (Result, error) {
|
|
|
|
say := logging(log)
|
|
say("clone", "%s%s at %s", repository, describePath(path), refOrHead(ref))
|
|
|
|
// Made rather than required. A builder that fails because the directory it was told to work
|
|
// in does not exist is a builder that needs a setup step nobody documented.
|
|
if err := os.MkdirAll(workspace, 0o755); err != nil {
|
|
return Result{}, err
|
|
}
|
|
tree := filepath.Join(workspace, "source")
|
|
if err := os.RemoveAll(tree); err != nil {
|
|
return Result{}, err
|
|
}
|
|
// A fresh clone every time rather than a fetch into a tree that is already there. A build
|
|
// that reuses a working tree can succeed because of something a previous build left behind,
|
|
// and that is a build nobody can reproduce.
|
|
if _, err := run(ctx, workspace, "git", "clone", "--quiet", repository, tree); err != nil {
|
|
say("clone", "FAILED: %v", err)
|
|
return Result{}, fmt.Errorf("cannot clone %s: %w", repository, err)
|
|
}
|
|
say("clone", "done")
|
|
if ref != "" {
|
|
if _, err := run(ctx, tree, "git", "checkout", "--quiet", ref); err != nil {
|
|
return Result{}, fmt.Errorf("%s has no %s: %w", repository, ref, err)
|
|
}
|
|
}
|
|
commit, err := run(ctx, tree, "git", "rev-parse", "HEAD")
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
commit = strings.TrimSpace(commit)
|
|
say("commit", "%s", short(commit))
|
|
|
|
// A module is a repository and a path within it (novox/hq ADR 0069). The ordinary case is an
|
|
// empty path, meaning the repository's root; a repository holding several modules names each
|
|
// by its own directory, which is what the catalogue is and what the system this replaces has
|
|
// always done.
|
|
within, err := inside(tree, path)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
|
|
raw, err := os.ReadFile(filepath.Join(within, ManifestName))
|
|
if err != nil {
|
|
return Result{}, fmt.Errorf(
|
|
"%s has no %s at %s, so there is nothing saying what it is: %w",
|
|
repository, ManifestName, describe(path), err)
|
|
}
|
|
manifest, err := catalogue.ParseManifest(raw)
|
|
if err != nil {
|
|
say("manifest", "INVALID: %v", err)
|
|
return Result{}, err
|
|
}
|
|
say("manifest", "%s v%s — %d artifact(s)", manifest.Module, manifest.Version, artifactCount(manifest))
|
|
|
|
// A build-time credential, written where a build can mount it but never where it can be copied
|
|
// into an image or committed: under the workspace, beside the clone, not inside it. Absent when
|
|
// this mesh has no package registry yet, which is the bootstrap case (novox/hq ADR 0076).
|
|
var npmrcPath string
|
|
if npmrc.Enabled() {
|
|
content, err := npmrc.File()
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
npmrcPath = filepath.Join(workspace, "npmrc")
|
|
if err := os.WriteFile(npmrcPath, []byte(content), 0o600); err != nil {
|
|
return Result{}, fmt.Errorf("cannot write the package-registry credential for the build: %w", err)
|
|
}
|
|
say("packages", "resolving %s from the mesh's package registry", npmrc.Scope)
|
|
}
|
|
|
|
var built []catalogue.Built
|
|
if manifest.Build != nil {
|
|
// What this module said it stands on, answered with what this mesh actually holds. Done
|
|
// before anything is built, so a missing base is refused in front of the person who can
|
|
// fix it rather than inside a build that stops on its own first line.
|
|
args, err := standingOn(manifest, held)
|
|
if err != nil {
|
|
say("bases", "UNMET: %v", err)
|
|
return Result{}, err
|
|
}
|
|
if len(args) > 0 {
|
|
say("bases", "%d resolved from what the mesh holds", len(args)/2)
|
|
}
|
|
artifacts := append([]catalogue.Artifact{}, manifest.Build.Artifacts...)
|
|
// Ordered, so two builds of one commit do the same work in the same sequence and their
|
|
// logs can be compared.
|
|
sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name })
|
|
for _, a := range artifacts {
|
|
say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a))
|
|
made, err := one(ctx, run, publish, manifest.Module, within, commit, a, args, held, npmrcPath, say)
|
|
if err != nil {
|
|
say("artifact", "%s FAILED: %v", a.Name, err)
|
|
return Result{}, err
|
|
}
|
|
say("artifact", "%s done — %s", a.Name, describeMade(made))
|
|
built = append(built, made)
|
|
}
|
|
}
|
|
|
|
resolved, err := manifest.Resolve(built)
|
|
if err != nil {
|
|
say("resolve", "FAILED: %v", err)
|
|
return Result{}, err
|
|
}
|
|
say("done", "%s at %s — %d artifact(s) pinned", manifest.Module, short(commit), len(built))
|
|
return Result{Manifest: resolved, Commit: commit, Built: built,
|
|
Against: against(within, manifest)}, nil
|
|
}
|
|
|
|
// Log is where a build says what it is doing, step by step. Nil is silent — the tests pass none,
|
|
// and a build with nowhere to speak must still build.
|
|
type Log func(step, message string)
|
|
|
|
func logging(log Log) func(step, format string, args ...any) {
|
|
if log == nil {
|
|
return func(string, string, ...any) {}
|
|
}
|
|
return func(step, format string, args ...any) {
|
|
log(step, fmt.Sprintf(format, args...))
|
|
}
|
|
}
|
|
|
|
func describePath(path string) string {
|
|
if path == "" {
|
|
return ""
|
|
}
|
|
return " at " + path
|
|
}
|
|
|
|
func refOrHead(ref string) string {
|
|
if ref == "" {
|
|
return "HEAD"
|
|
}
|
|
return ref
|
|
}
|
|
|
|
func artifactCount(m catalogue.Manifest) int {
|
|
if m.Build == nil {
|
|
return 0
|
|
}
|
|
return len(m.Build.Artifacts)
|
|
}
|
|
|
|
func langSuffix(a catalogue.Artifact) string {
|
|
if a.Language != "" {
|
|
return ", " + a.Language
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func describeMade(made catalogue.Built) string {
|
|
if made.Digest != "" {
|
|
return made.Kind + " " + short(strings.TrimPrefix(made.Digest, "sha256:"))
|
|
}
|
|
return made.Kind + " " + made.Reference
|
|
}
|
|
|
|
// inside resolves a module's path within a clone, and refuses one that leaves it.
|
|
//
|
|
// **A build reads only its own tree.** A path of `../../etc` would otherwise make a build read —
|
|
// and an archive artifact publish — whatever the build machine happens to hold, which is the one
|
|
// thing a machine that builds other people's repositories must not do.
|
|
func inside(tree, path string) (string, error) {
|
|
if path == "" {
|
|
return tree, nil
|
|
}
|
|
if filepath.IsAbs(path) {
|
|
return "", fmt.Errorf(
|
|
"a module's path is inside its repository, and %q is an absolute path", path)
|
|
}
|
|
within := filepath.Join(tree, path)
|
|
rel, err := filepath.Rel(tree, within)
|
|
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
|
return "", fmt.Errorf(
|
|
"%q leaves the repository, and a build reads only its own tree", path)
|
|
}
|
|
return within, nil
|
|
}
|
|
|
|
// describe says where a manifest was looked for, in words a person can act on.
|
|
func describe(path string) string {
|
|
if path == "" {
|
|
return "its root"
|
|
}
|
|
return path
|
|
}
|
|
|
|
// pinnedImage matches an image reference pinned by digest, which is the only kind a build input is
|
|
// allowed to name — a tag is something somebody else can move under you.
|
|
var pinnedImage = regexp.MustCompile(`[A-Za-z0-9][A-Za-z0-9._/:-]*@sha256:[0-9a-f]{64}`)
|
|
|
|
// against reads what this module's image artifacts are built on top of, out of the files that
|
|
// build them. Nothing is guessed: a reference that is not written down is not reported.
|
|
func against(within string, manifest catalogue.Manifest) []string {
|
|
if manifest.Build == nil {
|
|
return nil
|
|
}
|
|
seen := map[string]bool{}
|
|
var out []string
|
|
for _, a := range manifest.Build.Artifacts {
|
|
if a.Kind != catalogue.ArtifactImage || a.From == "" {
|
|
continue
|
|
}
|
|
body, err := os.ReadFile(filepath.Join(within, a.From))
|
|
if err != nil {
|
|
// Not fatal: the build itself already failed if this file was needed and missing, and
|
|
// reporting no edges is honest where inventing them would not be.
|
|
continue
|
|
}
|
|
for _, found := range pinnedImage.FindAllString(string(body), -1) {
|
|
if !seen[found] {
|
|
seen[found] = true
|
|
out = append(out, found)
|
|
}
|
|
}
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// ManifestName is the one file a module repository must have.
|
|
//
|
|
// At the root, and named the same in every repository. A convention somebody can look for beats a
|
|
// setting somebody has to find.
|
|
const ManifestName = "module.json"
|
|
|
|
func one(ctx context.Context, run Runner, publish Publisher,
|
|
module, tree, commit string, a catalogue.Artifact, args []string,
|
|
held map[string]string, npmrc string, say func(step, format string, args ...any)) (catalogue.Built, error) {
|
|
|
|
switch a.Kind {
|
|
case catalogue.ArtifactUpstream:
|
|
// Mirrored, not built. Pulled by the reference the module names and pushed under a name
|
|
// of the mesh's own, so what a machine fetches is pinned by a digest this registry
|
|
// assigned rather than by a tag somebody else can move.
|
|
say("mirror", "pulling %s", a.From)
|
|
if _, err := run(ctx, tree, "docker", "pull", a.From); err != nil {
|
|
return catalogue.Built{}, fmt.Errorf("%s: cannot fetch %s: %w", module, a.From, err)
|
|
}
|
|
say("mirror", "publishing under the mesh's own name")
|
|
reference, err := publish.PublishImage(ctx, a.From, module+"/"+a.Name)
|
|
if err != nil {
|
|
return catalogue.Built{}, err
|
|
}
|
|
return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil
|
|
|
|
case catalogue.ArtifactImage:
|
|
// Tagged by commit rather than by version, because a version is what a person calls a
|
|
// release and a commit is what was actually built. The mesh pins the digest anyway; this
|
|
// is only so a person looking at the build node can tell what is there.
|
|
local := fmt.Sprintf("%s-%s:%s", module, a.Name, short(commit))
|
|
// The bases this module named, resolved to what this mesh holds. A recipe reads them as
|
|
// build arguments, so a module says which module it stands on and never which copy.
|
|
invocation := append([]string{"build", "-f", a.From, "-t", local}, args...)
|
|
if a.Target != "" {
|
|
invocation = append(invocation, "--target", a.Target)
|
|
}
|
|
if npmrc != "" {
|
|
// Given to the build as a buildkit secret, so a RUN that needs the package registry mounts
|
|
// it at that step and it is in no image layer. A Dockerfile that does not ask for it is
|
|
// unaffected; the secret is simply not read (novox/hq ADR 0076).
|
|
invocation = append(invocation, "--secret", "id=npmrc,src="+npmrc)
|
|
}
|
|
invocation = append(invocation, ".")
|
|
say("image", "docker build -f %s", a.From)
|
|
if _, err := run(ctx, tree, "docker", invocation...); err != nil {
|
|
return catalogue.Built{}, fmt.Errorf("%s: building %s failed: %w", module, a.Name, err)
|
|
}
|
|
say("image", "built, publishing")
|
|
reference, err := publish.PublishImage(ctx, local, module+"/"+a.Name)
|
|
if err != nil {
|
|
return catalogue.Built{}, err
|
|
}
|
|
return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil
|
|
|
|
case catalogue.ArtifactBundle:
|
|
// **The one recipe that both builds and packs.** Everything else either produces an image
|
|
// or packs what is already there; this compiles the module's own code first, in a
|
|
// toolchain the mesh chose from what the module said it was written in, and packs the
|
|
// result.
|
|
//
|
|
// The compiler runs in a container rather than on the build machine, for the reason every
|
|
// other build does: what a build needs installed is the toolchain's business, and a build
|
|
// machine that accumulated one toolchain per language would be a machine nobody could
|
|
// reproduce.
|
|
chain, err := ToolchainFor(a.Language)
|
|
if err != nil {
|
|
return catalogue.Built{}, fmt.Errorf("%s: %s: %w", module, a.Name, err)
|
|
}
|
|
base, ok := held[chain.Base+"/"+chain.Artifact]
|
|
if !ok {
|
|
// Named, not pinned: the mesh answers with the copy it holds. Refused before anything
|
|
// is built, saying which module has to exist first, rather than failing inside a
|
|
// compile with a message about an image (novox/hq 04-ISSUES/044).
|
|
return catalogue.Built{}, fmt.Errorf(
|
|
"%s: %s is written in %s, which is compiled by %s's %q artifact, and this mesh "+
|
|
"holds no copy of it. Build %s first",
|
|
module, a.Name, chain.Language, chain.Base, chain.Artifact, chain.Base)
|
|
}
|
|
say("bundle", "compiling %s in %s's toolchain", a.Language, chain.Base)
|
|
compiled, err := compile(ctx, run, tree, chain, base, a)
|
|
if err != nil {
|
|
return catalogue.Built{}, fmt.Errorf("%s: compiling %s failed: %w", module, a.Name, err)
|
|
}
|
|
say("bundle", "compiled, packing")
|
|
body, err := pack(compiled)
|
|
if err != nil {
|
|
return catalogue.Built{}, fmt.Errorf("%s: packing %s failed: %w", module, a.Name, err)
|
|
}
|
|
sum := sha256.Sum256(body)
|
|
digest := "sha256:" + hex.EncodeToString(sum[:])
|
|
where, err := publish.PublishArchive(ctx, module+"/"+a.Name, body, digest)
|
|
if err != nil {
|
|
return catalogue.Built{}, err
|
|
}
|
|
return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: where, Digest: digest}, nil
|
|
|
|
case catalogue.ArtifactPackage:
|
|
// Built and published on a public base, to the mesh's package registry, by version
|
|
// (novox/hq ADR 0076). Not an image, not an archive: nothing on a machine references it, so
|
|
// there is no Publisher call — the container itself publishes, with the credential the
|
|
// build was handed.
|
|
say("package", "building and publishing %s (%s)", a.Name, a.Language)
|
|
reference, err := publishPackage(ctx, run, module, tree, a, npmrc, say)
|
|
if err != nil {
|
|
return catalogue.Built{}, fmt.Errorf("%s: publishing %s failed: %w", module, a.Name, err)
|
|
}
|
|
say("package", "published %s", reference)
|
|
return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil
|
|
|
|
case catalogue.ArtifactArchive:
|
|
body, err := pack(filepath.Join(tree, a.From))
|
|
if err != nil {
|
|
return catalogue.Built{}, fmt.Errorf("%s: packing %s failed: %w", module, a.Name, err)
|
|
}
|
|
sum := sha256.Sum256(body)
|
|
digest := "sha256:" + hex.EncodeToString(sum[:])
|
|
where, err := publish.PublishArchive(ctx, module+"/"+a.Name, body, digest)
|
|
if err != nil {
|
|
return catalogue.Built{}, err
|
|
}
|
|
return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: where, Digest: digest}, nil
|
|
}
|
|
return catalogue.Built{}, fmt.Errorf("%s: %q is a %q, which is not something this builds",
|
|
module, a.Name, a.Kind)
|
|
}
|
|
|
|
// pack tars and gzips a directory.
|
|
//
|
|
// **Deterministically**: entries sorted, and no timestamps, uid, gid or original names carried
|
|
// through. Two builds of one commit must produce one digest, or nothing downstream can tell "this
|
|
// changed" from "this was built again" — and every rebuild would look like a change to every
|
|
// machine holding it.
|
|
func pack(root string) ([]byte, error) {
|
|
info, err := os.Stat(root)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !info.IsDir() {
|
|
return nil, fmt.Errorf("%s is not a directory", root)
|
|
}
|
|
|
|
var paths []string
|
|
err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if info.IsDir() || !info.Mode().IsRegular() {
|
|
// Only files. A symlink or a device in an archive is refused by the host that unpacks
|
|
// it, so putting one in would build something that cannot be applied.
|
|
if !info.IsDir() && !info.Mode().IsRegular() {
|
|
return fmt.Errorf("%s is neither a file nor a directory, and an archive carries "+
|
|
"only those", path)
|
|
}
|
|
return nil
|
|
}
|
|
paths = append(paths, path)
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// filepath.Walk is documented to walk in lexical order, so this is belt and braces rather
|
|
// than load-bearing — and no test distinguishes it, which is worth saying rather than
|
|
// implying otherwise. It stays because the cost is nothing and the failure it guards against
|
|
// is silent: an archive whose digest changes because the traversal did.
|
|
sort.Strings(paths)
|
|
|
|
var out strings.Builder
|
|
zipped := gzip.NewWriter(&stringWriter{&out})
|
|
writer := tar.NewWriter(zipped)
|
|
for _, path := range paths {
|
|
body, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
relative, err := filepath.Rel(root, path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
mode := int64(info.Mode().Perm())
|
|
if err := writer.WriteHeader(&tar.Header{
|
|
Name: filepath.ToSlash(relative), Mode: mode, Size: int64(len(body)),
|
|
Typeflag: tar.TypeReg,
|
|
// Everything else left at its zero value on purpose — see the note above.
|
|
}); err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := writer.Write(body); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
if err := writer.Close(); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := zipped.Close(); err != nil {
|
|
return nil, err
|
|
}
|
|
return []byte(out.String()), nil
|
|
}
|
|
|
|
type stringWriter struct{ to *strings.Builder }
|
|
|
|
func (w *stringWriter) Write(p []byte) (int, error) { return w.to.Write(p) }
|
|
|
|
func short(commit string) string {
|
|
if len(commit) > 8 {
|
|
return commit[:8]
|
|
}
|
|
return commit
|
|
}
|
|
|
|
// Command is a Runner that actually runs things.
|
|
func Command(ctx context.Context, dir, name string, args ...string) (string, error) {
|
|
// **Every command is echoed before it runs**, with where. On a build that hangs, the last line
|
|
// is exactly the command it is inside — which is the difference between "the builder did
|
|
// nothing" and "git clone is waiting on a network that will not answer". Silent on success is
|
|
// what made an empty workspace unreadable.
|
|
started := timeNow()
|
|
fmt.Fprintf(os.Stderr, " $ (%s) %s %s\n", short(filepath.Base(dir)), name, strings.Join(args, " "))
|
|
cmd := exec.CommandContext(ctx, name, args...)
|
|
cmd.Dir = dir
|
|
out, err := cmd.CombinedOutput()
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, " ! %s %s failed after %s\n", name, args[0], since(started))
|
|
return string(out), fmt.Errorf("%s %s: %w\n%s",
|
|
name, strings.Join(args, " "), err, strings.TrimSpace(string(out)))
|
|
}
|
|
fmt.Fprintf(os.Stderr, " ✓ %s %s (%s)\n", name, firstArg(args), since(started))
|
|
return string(out), nil
|
|
}
|
|
|
|
func firstArg(args []string) string {
|
|
if len(args) == 0 {
|
|
return ""
|
|
}
|
|
return args[0]
|
|
}
|
|
|
|
var _ io.Writer = (*stringWriter)(nil)
|
|
|
|
// standingOn turns the bases a module named into build arguments for what this mesh holds.
|
|
//
|
|
// **Refused rather than defaulted** (novox/hq issue 044). A module naming a base the mesh has not
|
|
// built cannot be built here yet, and the useful sentence names which module is missing — not the
|
|
// one a container runtime produces when a recipe's first line refers to an image nobody has.
|
|
//
|
|
// The order is fixed so two builds of one commit invoke the same command.
|
|
func standingOn(manifest catalogue.Manifest, held map[string]string) ([]string, error) {
|
|
if manifest.Build == nil || len(manifest.Build.On) == 0 {
|
|
return nil, nil
|
|
}
|
|
on := append([]catalogue.BuildsOn{}, manifest.Build.On...)
|
|
sort.Slice(on, func(i, j int) bool { return on[i].Arg < on[j].Arg })
|
|
|
|
var args []string
|
|
for _, base := range on {
|
|
if base.Arg == "" || base.Module == "" || base.Artifact == "" {
|
|
return nil, fmt.Errorf(
|
|
"%s says its build stands on something, and does not say all of what: a base "+
|
|
"needs the module, the artifact, and the build argument the recipe reads it "+
|
|
"from", manifest.Module)
|
|
}
|
|
key := base.Module + "/" + base.Artifact
|
|
reference, has := held[key]
|
|
if !has {
|
|
return nil, fmt.Errorf(
|
|
"%s builds on %s, and this mesh has not built it. Build %s first — every module "+
|
|
"in this toolchain stands on it, so it is the thing to have before anything "+
|
|
"else", manifest.Module, key, base.Module)
|
|
}
|
|
args = append(args, "--build-arg", base.Arg+"="+reference)
|
|
}
|
|
return args, nil
|
|
}
|
|
|
|
// compile runs a module's own code through its toolchain, and says where the result is.
|
|
//
|
|
// **In the module's own directory, under the path the toolchain expects.** A module is compiled
|
|
// where its dependencies resolve upward into the base's own library directory, so what it is
|
|
// compiled against is exactly what it will run against — the reason every hand-written Dockerfile
|
|
// had to choose a working directory carefully, and the reason none of them has to now.
|
|
// publishPackage builds a `package` artifact on a public base image and publishes it to the mesh's
|
|
// package registry by version. The credential arrives as an .npmrc file the build was handed
|
|
// (novox/hq ADR 0076); it is mounted read-only into the container rather than baked, because a
|
|
// package build produces no image to leak it into. The reference returned is name@version, read from
|
|
// the module's own package.json — the same two fields npm publishes under.
|
|
func publishPackage(ctx context.Context, run Runner, module, dir string, a catalogue.Artifact,
|
|
npmrc string, say func(step, format string, args ...any)) (string, error) {
|
|
|
|
recipe, ok := packageRecipes[a.Language]
|
|
if !ok {
|
|
return "", fmt.Errorf(
|
|
"a package written in %q cannot be built: no public toolchain is known for it", a.Language)
|
|
}
|
|
if npmrc == "" {
|
|
// A package with nowhere to be published is not built. Said here rather than failing inside
|
|
// npm publish with a message about a registry that is simply absent.
|
|
return "", fmt.Errorf(
|
|
"%s is a package and this build was given no package registry to publish it to", a.Name)
|
|
}
|
|
|
|
raw, err := os.ReadFile(filepath.Join(dir, "package.json"))
|
|
if err != nil {
|
|
return "", fmt.Errorf("a package is published by name and version, and %s has no package.json: %w", module, err)
|
|
}
|
|
var pkg struct {
|
|
Name string `json:"name"`
|
|
Version string `json:"version"`
|
|
}
|
|
if err := json.Unmarshal(raw, &pkg); err != nil {
|
|
return "", fmt.Errorf("%s's package.json is not readable: %w", module, err)
|
|
}
|
|
if pkg.Name == "" || pkg.Version == "" {
|
|
return "", fmt.Errorf("%s's package.json names no %s to publish under",
|
|
module, either(pkg.Name == "", "name", "version"))
|
|
}
|
|
|
|
const within = "/app/module"
|
|
invocation := []string{
|
|
"run", "--rm",
|
|
"--volume", dir + ":" + within,
|
|
// Read-only, so a build cannot alter the credential, and at /root where npm reads it.
|
|
"--volume", npmrc + ":/root/.npmrc:ro",
|
|
"--workdir", within,
|
|
recipe.Base,
|
|
"sh", "-c", recipe.Script,
|
|
}
|
|
if _, err := run(ctx, dir, "docker", invocation...); err != nil {
|
|
return "", err
|
|
}
|
|
return pkg.Name + "@" + pkg.Version, nil
|
|
}
|
|
|
|
// either names whichever of two fields is the missing one, for a message that says which.
|
|
func either(first bool, a, b string) string {
|
|
if first {
|
|
return a
|
|
}
|
|
return b
|
|
}
|
|
|
|
func compile(ctx context.Context, run Runner, tree string, chain Toolchain,
|
|
base string, a catalogue.Artifact) (string, error) {
|
|
|
|
// Where inside the toolchain the module's source is mounted, and where its output lands. Fixed
|
|
// rather than configurable: a module that could move this would be describing its own build.
|
|
const within = "/app/modules/module"
|
|
|
|
// **Its own output directory, because a module may be several languages at once.** One module
|
|
// is one piece of software and can still carry a daemon in one language, tools in another and
|
|
// a package in a third (ADR 0040). Compiling them all into one place would have them overwrite
|
|
// each other and then be packed together, so each bundle compiles and packs alone.
|
|
out := Out(a.Name)
|
|
|
|
invocation := []string{
|
|
"run", "--rm",
|
|
"--volume", tree + ":" + within,
|
|
"--workdir", within,
|
|
base,
|
|
}
|
|
invocation = append(invocation, chain.Compile...)
|
|
if chain.OutputFlag != "" {
|
|
invocation = append(invocation, chain.OutputFlag, out)
|
|
}
|
|
// What to compile. Named by the module rather than discovered, so adding a file does not
|
|
// silently change what a build produces.
|
|
if len(a.Entrypoints) > 0 {
|
|
invocation = append(invocation, sourcesFor(a.Entrypoints, out)...)
|
|
}
|
|
if _, err := run(ctx, tree, "docker", invocation...); err != nil {
|
|
return "", err
|
|
}
|
|
return filepath.Join(tree, out), nil
|
|
}
|
|
|
|
// sourcesFor turns compiled entrypoints back into what to compile.
|
|
//
|
|
// A module names what a tool host should LOAD — compiled paths under the bundle's root — because
|
|
// that is the thing anything else needs to know. What to compile is the same list with the
|
|
// language's own extension, which is the toolchain's business rather than the module's.
|
|
func sourcesFor(entrypoints []string, out string) []string {
|
|
sources := make([]string, 0, len(entrypoints))
|
|
for _, e := range entrypoints {
|
|
// An entrypoint is named as it will be FOUND — a path inside the unpacked bundle — so the
|
|
// source is the same path with the output directory taken off the front and the language's
|
|
// own extension on the end.
|
|
at := strings.TrimPrefix(strings.TrimPrefix(e, out), "/")
|
|
sources = append(sources, strings.TrimSuffix(at, filepath.Ext(at))+".ts")
|
|
}
|
|
return sources
|
|
}
|
|
|
|
func timeNow() time.Time { return time.Now() }
|
|
func since(t time.Time) string { return time.Since(t).Round(time.Millisecond).String() }
|