From 4b9bc50aad2a342ea3c90231291f2ca5d115890a Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 16 Sep 2026 10:27:26 +0200 Subject: [PATCH] The builder resolves the SDK from the mesh registry, and can publish packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A new 'package' artifact kind builds a module's own code on a public base image and publishes it to the mesh's package registry by version (hq ADR 0076) — the SDK above all, which the toolchain is built from and so cannot be built in the toolchain. The credential a build needs to resolve or publish packages is rendered as an .npmrc (basic auth, hq ADR 0048) and given to an image build as a buildkit secret, never a layer, so a token is not baked into the toolchain image. Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- cmd/mesh-builder/main.go | 98 ++++++++++++++- cmd/mesh-builder/once.go | 6 +- internal/builder/builder.go | 102 +++++++++++++++- internal/builder/builder_test.go | 20 +-- internal/builder/bundle_test.go | 8 +- internal/builder/packages.go | 127 +++++++++++++++++++ internal/builder/packages_test.go | 196 ++++++++++++++++++++++++++++++ internal/catalogue/build.go | 12 +- internal/catalogue/build_test.go | 38 ++++++ internal/catalogue/manifest.go | 7 ++ 10 files changed, 588 insertions(+), 26 deletions(-) create mode 100644 internal/builder/packages.go create mode 100644 internal/builder/packages_test.go diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index 2ca09b0..9a5c9e2 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -55,6 +55,11 @@ is dialled except the broker. MESH_BROKER_FILE a file the mesh sealed to this machine holding the same MESH_REGISTRY host:port to publish artifacts to, when the mesh has not said MESH_BINDING a file the mesh wrote saying where the artifact store is + MESH_PACKAGE_BINDING a file the mesh wrote saying where the package registry is + MESH_NPM_TOKEN_FILE a file the mesh sealed holding the token for it + MESH_NPM_REGISTRY a package registry URL, when the mesh has not said (a person, the bootstrap) + MESH_NPM_TOKEN the token for it, likewise + MESH_NPM_SCOPE the scope it answers for (default: @novox) MESH_WORKSPACE where to clone and build (default: a temporary directory) It also builds one module and stops, which is how a mesh is raised — before there is a @@ -189,11 +194,18 @@ func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publis } fmt.Fprintln(os.Stderr) - built, err := builder.Build(ctx, builder.Command, publisher, - request.Repository, request.Path, request.Ref, workspace, request.Held, - func(step, message string) { - fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message) - }) + npmrc, err := packagesFrom() + var built builder.Result + if err == nil { + // The package-registry credential is a build input, so it is resolved before the clone: a + // build that could not have resolved its dependencies is refused in front of the reason, + // not after a clone that then fails at npm ci. + built, err = builder.Build(ctx, builder.Command, publisher, + request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc, + func(step, message string) { + fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message) + }) + } if err != nil { // A failure is a result. A build that fails and says nothing is indistinguishable from a // builder that is not running, and those want completely different responses. @@ -279,6 +291,82 @@ func moduleOf(manifest json.RawMessage) string { return named.Module } +// packagesFrom is where a build resolves the mesh's own published packages — the SDK above all +// (novox/hq ADR 0076, issue 053). +// +// Preferably from the mesh: a package-registry binding names the endpoint the way the artifact +// store's binding does, and a sealed token file the credential the way the broker's does. The +// environment variables remain for a builder run by a person, and for the bootstrap, where there is +// no registry yet — there the result is disabled and a build that needs no mesh-published dependency +// builds anyway. +func packagesFrom() (builder.Npmrc, error) { + scope := strings.TrimSpace(os.Getenv("MESH_NPM_SCOPE")) + if scope == "" { + scope = "@novox" + } + + registry := strings.TrimSpace(os.Getenv("MESH_NPM_REGISTRY")) + var username string + if path := strings.TrimSpace(os.Getenv("MESH_PACKAGE_BINDING")); path != "" { + raw, err := os.ReadFile(path) + if err != nil { + return builder.Npmrc{}, fmt.Errorf("cannot read what the mesh said about the package registry: %w", err) + } + var told struct { + From string `json:"from"` + At string `json:"at"` + As string `json:"as"` + Serves map[string]any `json:"serves"` + } + if err := json.Unmarshal(raw, &told); err != nil { + return builder.Npmrc{}, fmt.Errorf("%s is not a binding: %w", path, err) + } + if told.At == "" { + return builder.Npmrc{}, fmt.Errorf( + "%s says the package registry is on %q and gives no address for it", path, told.From) + } + // Composed from what the provider serves, so nothing here knows gitea's URL shape from + // another registry's: it states its port, the path its registry answers on, and the scheme. + scheme := "https" + if s, ok := told.Serves["scheme"]; ok { + scheme = fmt.Sprintf("%v", s) + } + port, ok := told.Serves["port"] + if !ok { + return builder.Npmrc{}, fmt.Errorf("%s says nothing about which port the package registry answers on", path) + } + npmPath, ok := told.Serves["npm-path"] + if !ok { + return builder.Npmrc{}, fmt.Errorf("%s says nothing about the path the package registry answers on", path) + } + registry = fmt.Sprintf("%s://%s:%v%v", scheme, told.At, port, npmPath) + username = told.As + } + + // The credential the mesh sealed to this machine. The mesh authenticates the ordinary way — a + // generated password the provider only applies (novox/hq ADR 0048) — so with a username this is + // a password (basic auth); without one it is a bearer token a provider minted. + secret := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN")) + if path := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN_FILE")); path != "" { + raw, err := os.ReadFile(path) + if err != nil { + return builder.Npmrc{}, fmt.Errorf("cannot read this builder's package-registry credential: %w", err) + } + secret = strings.TrimSpace(string(raw)) + } + if u := strings.TrimSpace(os.Getenv("MESH_NPM_USER")); u != "" { + username = u + } + + if registry == "" && secret == "" { + return builder.Npmrc{}, nil + } + if username != "" { + return builder.Npmrc{Scope: scope, Registry: registry, Username: username, Password: secret}, nil + } + return builder.Npmrc{Scope: scope, Registry: registry, Token: secret}, nil +} + func short(commit string) string { if len(commit) > 8 { return commit[:8] diff --git a/cmd/mesh-builder/once.go b/cmd/mesh-builder/once.go index e6f7196..8423c67 100644 --- a/cmd/mesh-builder/once.go +++ b/cmd/mesh-builder/once.go @@ -84,7 +84,11 @@ func buildOnce(ctx context.Context, args []string) error { } fmt.Fprintln(os.Stderr) - built, buildErr := builder.Build(ctx, builder.Command, publisher, repository, *path, *ref, where, bases, + npmrc, err := packagesFrom() + if err != nil { + return err + } + built, buildErr := builder.Build(ctx, builder.Command, publisher, repository, *path, *ref, where, bases, npmrc, func(step, message string) { fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message) }) if buildErr != nil { return buildErr diff --git a/internal/builder/builder.go b/internal/builder/builder.go index 54342ed..a7cddcc 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -6,6 +6,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" "io" "os" @@ -68,7 +69,7 @@ type Result struct { // archive failed would otherwise leave half of itself in the store under a digest the mesh never // records — reachable, unreferenced, and indistinguishable from something in use. func Build(ctx context.Context, run Runner, publish Publisher, - repository, path, ref, workspace string, held map[string]string, log Log) (Result, error) { + repository, path, ref, workspace string, held map[string]string, npmrc Npmrc, log Log) (Result, error) { say := logging(log) say("clone", "%s%s at %s", repository, describePath(path), refOrHead(ref)) @@ -124,6 +125,22 @@ func Build(ctx context.Context, run Runner, publish Publisher, } say("manifest", "%s v%s — %d artifact(s)", manifest.Module, manifest.Version, artifactCount(manifest)) + // A build-time credential, written where a build can mount it but never where it can be copied + // into an image or committed: under the workspace, beside the clone, not inside it. Absent when + // this mesh has no package registry yet, which is the bootstrap case (novox/hq ADR 0076). + var npmrcPath string + if npmrc.Enabled() { + content, err := npmrc.File() + if err != nil { + return Result{}, err + } + npmrcPath = filepath.Join(workspace, "npmrc") + if err := os.WriteFile(npmrcPath, []byte(content), 0o600); err != nil { + return Result{}, fmt.Errorf("cannot write the package-registry credential for the build: %w", err) + } + say("packages", "resolving %s from the mesh's package registry", npmrc.Scope) + } + var built []catalogue.Built if manifest.Build != nil { // What this module said it stands on, answered with what this mesh actually holds. Done @@ -143,7 +160,7 @@ func Build(ctx context.Context, run Runner, publish Publisher, sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name }) for _, a := range artifacts { say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a)) - made, err := one(ctx, run, publish, manifest.Module, within, commit, a, args, held, say) + made, err := one(ctx, run, publish, manifest.Module, within, commit, a, args, held, npmrcPath, say) if err != nil { say("artifact", "%s FAILED: %v", a.Name, err) return Result{}, err @@ -282,7 +299,7 @@ const ManifestName = "module.json" func one(ctx context.Context, run Runner, publish Publisher, module, tree, commit string, a catalogue.Artifact, args []string, - held map[string]string, say func(step, format string, args ...any)) (catalogue.Built, error) { + held map[string]string, npmrc string, say func(step, format string, args ...any)) (catalogue.Built, error) { switch a.Kind { case catalogue.ArtifactUpstream: @@ -311,6 +328,12 @@ func one(ctx context.Context, run Runner, publish Publisher, if a.Target != "" { invocation = append(invocation, "--target", a.Target) } + if npmrc != "" { + // Given to the build as a buildkit secret, so a RUN that needs the package registry mounts + // it at that step and it is in no image layer. A Dockerfile that does not ask for it is + // unaffected; the secret is simply not read (novox/hq ADR 0076). + invocation = append(invocation, "--secret", "id=npmrc,src="+npmrc) + } invocation = append(invocation, ".") say("image", "docker build -f %s", a.From) if _, err := run(ctx, tree, "docker", invocation...); err != nil { @@ -365,6 +388,19 @@ func one(ctx context.Context, run Runner, publish Publisher, } return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: where, Digest: digest}, nil + case catalogue.ArtifactPackage: + // Built and published on a public base, to the mesh's package registry, by version + // (novox/hq ADR 0076). Not an image, not an archive: nothing on a machine references it, so + // there is no Publisher call — the container itself publishes, with the credential the + // build was handed. + say("package", "building and publishing %s (%s)", a.Name, a.Language) + reference, err := publishPackage(ctx, run, module, tree, a, npmrc, say) + if err != nil { + return catalogue.Built{}, fmt.Errorf("%s: publishing %s failed: %w", module, a.Name, err) + } + say("package", "published %s", reference) + return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil + case catalogue.ArtifactArchive: body, err := pack(filepath.Join(tree, a.From)) if err != nil { @@ -541,6 +577,66 @@ func standingOn(manifest catalogue.Manifest, held map[string]string) ([]string, // where its dependencies resolve upward into the base's own library directory, so what it is // compiled against is exactly what it will run against — the reason every hand-written Dockerfile // had to choose a working directory carefully, and the reason none of them has to now. +// publishPackage builds a `package` artifact on a public base image and publishes it to the mesh's +// package registry by version. The credential arrives as an .npmrc file the build was handed +// (novox/hq ADR 0076); it is mounted read-only into the container rather than baked, because a +// package build produces no image to leak it into. The reference returned is name@version, read from +// the module's own package.json — the same two fields npm publishes under. +func publishPackage(ctx context.Context, run Runner, module, dir string, a catalogue.Artifact, + npmrc string, say func(step, format string, args ...any)) (string, error) { + + recipe, ok := packageRecipes[a.Language] + if !ok { + return "", fmt.Errorf( + "a package written in %q cannot be built: no public toolchain is known for it", a.Language) + } + if npmrc == "" { + // A package with nowhere to be published is not built. Said here rather than failing inside + // npm publish with a message about a registry that is simply absent. + return "", fmt.Errorf( + "%s is a package and this build was given no package registry to publish it to", a.Name) + } + + raw, err := os.ReadFile(filepath.Join(dir, "package.json")) + if err != nil { + return "", fmt.Errorf("a package is published by name and version, and %s has no package.json: %w", module, err) + } + var pkg struct { + Name string `json:"name"` + Version string `json:"version"` + } + if err := json.Unmarshal(raw, &pkg); err != nil { + return "", fmt.Errorf("%s's package.json is not readable: %w", module, err) + } + if pkg.Name == "" || pkg.Version == "" { + return "", fmt.Errorf("%s's package.json names no %s to publish under", + module, either(pkg.Name == "", "name", "version")) + } + + const within = "/app/module" + invocation := []string{ + "run", "--rm", + "--volume", dir + ":" + within, + // Read-only, so a build cannot alter the credential, and at /root where npm reads it. + "--volume", npmrc + ":/root/.npmrc:ro", + "--workdir", within, + recipe.Base, + "sh", "-c", recipe.Script, + } + if _, err := run(ctx, dir, "docker", invocation...); err != nil { + return "", err + } + return pkg.Name + "@" + pkg.Version, nil +} + +// either names whichever of two fields is the missing one, for a message that says which. +func either(first bool, a, b string) string { + if first { + return a + } + return b +} + func compile(ctx context.Context, run Runner, tree string, chain Toolchain, base string, a catalogue.Artifact) (string, error) { diff --git a/internal/builder/builder_test.go b/internal/builder/builder_test.go index a97a93f..6938d71 100644 --- a/internal/builder/builder_test.go +++ b/internal/builder/builder_test.go @@ -108,7 +108,7 @@ func TestABuildProducesAManifestThePinsAreIn(t *testing.T) { r, workspace := aRepository(t, withBoth, map[string]string{ "Dockerfile": "FROM scratch", "files/theme.conf": "dark", }) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, nil) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, Npmrc{}, nil) if err != nil { t.Fatal(err) } @@ -134,7 +134,7 @@ func TestTwoBuildsOfOneCommitProduceOneDigest(t *testing.T) { }) // A year apart, so a packer carrying timestamps cannot accidentally agree. r.stamped = time.Date(2020+i, time.March, 3, 4, 5, 6, 0, time.UTC) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, nil) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, nil) if err != nil { t.Fatal(err) } @@ -154,7 +154,7 @@ func TestNothingIsPublishedUntilEverythingIsBuilt(t *testing.T) { // unreferenced, and indistinguishable from something in use. r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch"}) // `files` is missing, so packing the archive fails — after the image would have been pushed. - _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, nil) + _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, nil) if err == nil { t.Fatal("a build with a missing input succeeded") } @@ -166,7 +166,7 @@ func TestNothingIsPublishedUntilEverythingIsBuilt(t *testing.T) { func TestARepositoryWithNoManifestSaysSo(t *testing.T) { workspace := t.TempDir() r := &recorded{contents: map[string]string{"README.md": "nothing to see"}} - _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, nil) + _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, nil) if err == nil { t.Fatal("a repository with nothing saying what it is was built") } @@ -179,7 +179,7 @@ func TestAModuleThatBuildsNothingStillProducesAManifest(t *testing.T) { // Most of what a person installs is configuration. r, workspace := aRepository(t, `{"module":"shell","version":"1","resources":[ {"id":"rc","type":"file","path":"/etc/zsh/zshrc","content":"setopt"}]}`, nil) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/shell.git", "", "", workspace, nil, nil) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/shell.git", "", "", workspace, nil, Npmrc{}, nil) if err != nil { t.Fatal(err) } @@ -209,7 +209,7 @@ func TestTheTreeIsFreshEveryTime(t *testing.T) { if err := os.WriteFile(leftover, []byte("stale"), 0o644); err != nil { t.Fatal(err) } - if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, nil); err != nil { + if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, nil); err != nil { t.Fatal(err) } if _, err := os.Stat(leftover); err == nil { @@ -222,7 +222,7 @@ func TestABuildThatCannotPushFails(t *testing.T) { "Dockerfile": "FROM scratch", "files/a": "b", }) r.failPush = true - if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, nil); err == nil { + if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, nil); err == nil { t.Fatal("a build that could publish nothing reported success") } } @@ -236,7 +236,7 @@ func TestAnUpstreamImageIsMirroredRatherThanBuilt(t *testing.T) { "resources":[{"id":"db","type":"container","name":"mesh-postgres","artifact":"store"}]}` r, workspace := aRepository(t, mirrors, nil) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/postgres.git", "", "", workspace, nil, nil) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/postgres.git", "", "", workspace, nil, Npmrc{}, nil) if err != nil { t.Fatal(err) } @@ -302,7 +302,7 @@ func TestAModuleIsBuiltFromItsPathWithinTheRepository(t *testing.T) { "modules/other/" + ManifestName: `{"module":"other","version":"1"}`, }} got, err := Build(context.Background(), r.run, r, - "https://forge.invalid/catalogue.git", "modules/shell", "", t.TempDir(), nil, nil) + "https://forge.invalid/catalogue.git", "modules/shell", "", t.TempDir(), nil, Npmrc{}, nil) if err != nil { t.Fatal(err) } @@ -321,7 +321,7 @@ func TestAPathThatLeavesTheRepositoryIsRefused(t *testing.T) { for _, escaping := range []string{"../../etc", "/etc"} { r := &recorded{contents: map[string]string{ManifestName: withBoth}} _, err := Build(context.Background(), r.run, r, - "https://forge.invalid/x.git", escaping, "", t.TempDir(), nil, nil) + "https://forge.invalid/x.git", escaping, "", t.TempDir(), nil, Npmrc{}, nil) if err == nil { t.Fatalf("%q was accepted as a module's path", escaping) } diff --git a/internal/builder/bundle_test.go b/internal/builder/bundle_test.go index ffbce41..5cf8846 100644 --- a/internal/builder/bundle_test.go +++ b/internal/builder/bundle_test.go @@ -54,7 +54,7 @@ func TestABundleIsCompiledAndPackedWithNoDockerfile(t *testing.T) { held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} got, err := Build(context.Background(), compiling{r}.run, r, - "https://forge.invalid/greeter.git", "", "", workspace, held, nil) + "https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, nil) if err != nil { t.Fatalf("a module with a language and no Dockerfile did not build: %v", err) } @@ -91,7 +91,7 @@ func TestABundleWhoseToolchainIsNotHeldIsRefusedFirst(t *testing.T) { r, workspace := aRepository(t, aBundle, map[string]string{"index.ts": "console.log(1)"}) _, err := Build(context.Background(), compiling{r}.run, r, - "https://forge.invalid/greeter.git", "", "", workspace, nil, nil) + "https://forge.invalid/greeter.git", "", "", workspace, nil, Npmrc{}, nil) if err == nil { t.Fatal("a bundle was built with no toolchain to compile it in") } @@ -112,7 +112,7 @@ func TestABundleInAnUnknownLanguageIsRefused(t *testing.T) { _, err := Build(context.Background(), compiling{r}.run, r, "https://forge.invalid/greeter.git", "", "", workspace, - map[string]string{"mesh-tools/build": "registry.invalid/x@sha256:" + strings.Repeat("c", 64)}, nil) + map[string]string{"mesh-tools/build": "registry.invalid/x@sha256:" + strings.Repeat("c", 64)}, Npmrc{}, nil) if err == nil { t.Fatal("a language nothing can compile was accepted") } @@ -140,7 +140,7 @@ func TestTwoBundlesInOneModuleArePackedSeparately(t *testing.T) { held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} got, err := Build(context.Background(), compiling{r}.run, r, - "https://forge.invalid/greeter.git", "", "", workspace, held, nil) + "https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, nil) if err != nil { t.Fatalf("a module with two bundles did not build: %v", err) } diff --git a/internal/builder/packages.go b/internal/builder/packages.go new file mode 100644 index 0000000..d30cacd --- /dev/null +++ b/internal/builder/packages.go @@ -0,0 +1,127 @@ +package builder + +import ( + "encoding/base64" + "fmt" + "net/url" + "strings" +) + +// Npmrc is what a build needs to resolve the mesh's own shared library — and any other package the +// mesh publishes — from the mesh's package registry rather than from a git URL (novox/hq ADR 0076, +// issue 053). +// +// It is a build-time credential, not a runtime one. A module compiled inside the toolchain image +// resolves the SDK there, once, when that image is built; the running container never speaks to the +// package registry. So this is given to the *builder*, the way the artifact store is +// (`whereToPublish`), and reaches a build as a secret rather than a layer — see Secret. +// +// The registry's exact URL shape is the provider's business, not the builder's: it arrives whole, +// either from the binding the mesh writes (a package-registry provider's `serves` facts) or from the +// environment when a person runs a build by hand. Nothing here knows gitea from verdaccio. +type Npmrc struct { + // Scope is the npm scope the registry answers for, e.g. "@novox". A build resolves only this + // scope from the mesh; everything else resolves the ordinary way, so a mesh with no internet + // still cannot pull the public registry's version of a name the mesh also publishes. + Scope string + // Registry is the full base URL a client uses for this scope, e.g. + // "https:///api/packages//npm/". Trailing slash tolerated either way. + Registry string + // Token authenticates to the registry as a bearer token, when a provider mints one. Left empty + // when the mesh authenticates the ordinary way it authenticates everything — a generated + // password it applies and seals — for which see Username and Password. + Token string + // Username and Password authenticate by basic auth, which is what gitea and verdaccio both + // accept and what lets the credential be a mesh-generated password the provider's provisioner + // applies and the mesh seals to the consumer — the same shape a database password takes. The + // username is the consumer's mesh identity. Ignored when Token is set. + Username string + Password string +} + +// Enabled reports whether there is a registry to resolve against at all. A bootstrap build that +// runs before any package registry exists has none, and must still build whatever needs no +// mesh-published dependency. +func (n Npmrc) Enabled() bool { + return strings.TrimSpace(n.Scope) != "" && strings.TrimSpace(n.Registry) != "" +} + +// File renders the .npmrc a build mounts. Two lines: which registry answers for the scope, and the +// token to present to it. The auth line is keyed by the registry URL with its scheme removed, which +// is how npm matches a stored credential to a request. +// +// It returns an error rather than a malformed file, because an .npmrc that npm parses but points +// nowhere fails much later, inside a build, as a package that cannot be found. +func (n Npmrc) File() (string, error) { + scope := strings.TrimSpace(n.Scope) + if !strings.HasPrefix(scope, "@") { + return "", fmt.Errorf("a package-registry scope is written with its leading @, not %q", scope) + } + reg := strings.TrimSpace(n.Registry) + if !strings.HasPrefix(reg, "http://") && !strings.HasPrefix(reg, "https://") { + return "", fmt.Errorf("a package registry is reached over http(s), and %q is neither", reg) + } + if !strings.HasSuffix(reg, "/") { + // npm's per-scope registry key is matched by prefix, and the auth key below is derived from + // it; a missing trailing slash makes the two disagree and the token is never sent. + reg += "/" + } + parsed, err := url.Parse(reg) + if err != nil { + return "", fmt.Errorf("%q is not a usable registry URL: %w", reg, err) + } + // The auth key is the URL without its scheme, e.g. "//host/api/packages/owner/npm/". + authKey := "//" + parsed.Host + parsed.EscapedPath() + + var auth string + switch { + case strings.TrimSpace(n.Token) != "": + auth = fmt.Sprintf("%s:_authToken=%s\n", authKey, strings.TrimSpace(n.Token)) + case strings.TrimSpace(n.Username) != "" && n.Password != "": + // npm reads the password base64-encoded, and always-auth so it presents the credential to + // reads as well as writes — a private registry answers neither without it. + enc := base64.StdEncoding.EncodeToString([]byte(n.Password)) + auth = fmt.Sprintf("%s:username=%s\n%s:_password=%s\n%s:always-auth=true\n", + authKey, strings.TrimSpace(n.Username), authKey, enc, authKey) + default: + return "", fmt.Errorf( + "the package registry at %s was given neither a token nor a username and password", reg) + } + return fmt.Sprintf("%s:registry=%s\n%s", scope, reg, auth), nil +} + +// packageRecipe is how a `package` artifact is built and published: on a PUBLIC base image, never +// the mesh toolchain, because the toolchain is built from the package this produces (the SDK). The +// script builds the module, then publishes it to the mesh's package registry unless that exact +// version is already there — so a re-run of genesis, which must be safe, does not fail on a version +// it published a moment ago. +type packageRecipe struct { + Base string + Script string +} + +var packageRecipes = map[string]packageRecipe{ + "typescript": { + Base: "node:22-bookworm-slim", + Script: `set -e +npm install --no-audit --no-fund +npm run build +name="$(node -p "require('./package.json').name")" +ver="$(node -p "require('./package.json').version")" +if npm view "$name@$ver" version >/dev/null 2>&1; then + echo "mesh-builder: $name@$ver is already published, leaving it" +else + npm publish +fi`, + }, +} + +// PackageLanguages is the languages a package artifact can be written in, for a manifest check that +// wants to refuse one it cannot build before a build starts. +func PackageLanguages() []string { + out := make([]string, 0, len(packageRecipes)) + for l := range packageRecipes { + out = append(out, l) + } + return out +} diff --git a/internal/builder/packages_test.go b/internal/builder/packages_test.go new file mode 100644 index 0000000..a26a287 --- /dev/null +++ b/internal/builder/packages_test.go @@ -0,0 +1,196 @@ +package builder + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestNpmrcRendersRegistryAndTokenForTheScope(t *testing.T) { + n := Npmrc{ + Scope: "@novox", + Registry: "https://forge.invalid/api/packages/novox/npm/", + Token: "a-token", + } + got, err := n.File() + if err != nil { + t.Fatalf("a complete credential did not render: %v", err) + } + if !strings.Contains(got, "@novox:registry=https://forge.invalid/api/packages/novox/npm/") { + t.Fatalf("the scope's registry line is missing:\n%s", got) + } + // The auth line is keyed by the URL without its scheme, or npm never sends the token. + if !strings.Contains(got, "//forge.invalid/api/packages/novox/npm/:_authToken=a-token") { + t.Fatalf("the auth line does not match the registry key:\n%s", got) + } +} + +func TestNpmrcAddsATrailingSlashSoTheAuthKeyMatches(t *testing.T) { + n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm", Token: "t"} + got, err := n.File() + if err != nil { + t.Fatal(err) + } + if !strings.Contains(got, "registry=https://forge.invalid/api/packages/novox/npm/\n") { + t.Fatalf("a missing trailing slash was not normalised:\n%s", got) + } +} + +func TestNpmrcRefusesTheHalfConfigured(t *testing.T) { + cases := map[string]Npmrc{ + "scope without @": {Scope: "novox", Registry: "https://x.invalid/", Token: "t"}, + "registry not http": {Scope: "@novox", Registry: "ftp://x.invalid/", Token: "t"}, + "no token": {Scope: "@novox", Registry: "https://x.invalid/", Token: ""}, + } + for name, n := range cases { + if _, err := n.File(); err == nil { + t.Fatalf("%s rendered an .npmrc rather than refusing", name) + } + } +} + +func TestNpmrcDisabledUntilThereIsARegistry(t *testing.T) { + if (Npmrc{}).Enabled() { + t.Fatal("an empty credential reported itself usable") + } + if (Npmrc{Scope: "@novox"}).Enabled() { + t.Fatal("a scope with no registry reported itself usable") + } + if !(Npmrc{Scope: "@novox", Registry: "https://x.invalid/"}).Enabled() { + t.Fatal("a scope and a registry did not count as usable") + } +} + +// The credential reaches an image build as a buildkit secret and never as a file inside the build +// context, because a token copied into a layer is a token published (novox/hq ADR 0076). +func TestAnImageBuildGetsTheCredentialAsASecretNotALayer(t *testing.T) { + r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch", "files/x": "y"}) + n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"} + if _, err := Build(context.Background(), r.run, r, + "https://forge.invalid/meshboard.git", "", "", workspace, nil, n, nil); err != nil { + t.Fatalf("the build failed: %v", err) + } + + var build string + for _, line := range r.ran { + if strings.HasPrefix(line, "docker build") { + build = line + } + } + if build == "" { + t.Fatal("no docker build ran") + } + if !strings.Contains(build, "--secret id=npmrc,src=") { + t.Fatalf("the build was not given the credential as a secret: %s", build) + } + + // The .npmrc lives under the workspace, beside the clone, never inside the source tree that is + // the docker context. + tree := filepath.Join(workspace, "source") + src := strings.SplitN(strings.SplitN(build, "--secret id=npmrc,src=", 2)[1], " ", 2)[0] + if strings.HasPrefix(src, tree+string(os.PathSeparator)) { + t.Fatalf("the credential file %s is inside the build context %s", src, tree) + } + if _, err := os.Stat(src); err != nil { + t.Fatalf("the credential file the build was pointed at does not exist: %v", err) + } +} + +func TestAnImageBuildWithoutACredentialGetsNoSecret(t *testing.T) { + r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch", "files/x": "y"}) + if _, err := Build(context.Background(), r.run, r, + "https://forge.invalid/meshboard.git", "", "", workspace, nil, Npmrc{}, nil); err != nil { + t.Fatalf("the build failed: %v", err) + } + for _, line := range r.ran { + if strings.HasPrefix(line, "docker build") && strings.Contains(line, "--secret") { + t.Fatalf("a build with no credential was still given a secret: %s", line) + } + } +} + +const aPackage = `{"module":"mesh-sdk","version":"1", + "build":{"artifacts":[{"name":"lib","kind":"package","language":"typescript"}]}, + "resources":[]}` + +// A package is compiled on a public base and published to the mesh's package registry by version, +// with nothing pushed to the artifact store and the credential mounted, not baked (novox/hq ADR 0076). +func TestAPackageIsBuiltOnAPublicBaseAndPublishedByVersion(t *testing.T) { + r, workspace := aRepository(t, aPackage, map[string]string{ + "package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`, + }) + n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"} + got, err := Build(context.Background(), r.run, r, + "https://forge.invalid/mesh-sdk.git", "", "", workspace, nil, n, nil) + if err != nil { + t.Fatalf("the package did not build: %v", err) + } + if len(got.Built) != 1 || got.Built[0].Reference != "@novox/mesh-sdk@0.1.0" { + t.Fatalf("a package is published by name and version, got %+v", got.Built) + } + if len(r.images) != 0 || len(r.archives) != 0 { + t.Fatal("a package was pushed to the artifact store, which is not where packages live") + } + var ran string + for _, line := range r.ran { + if strings.HasPrefix(line, "docker run") { + ran = line + } + } + if ran == "" { + t.Fatal("nothing ran to build the package") + } + if !strings.Contains(ran, "node:22-bookworm-slim") { + t.Fatalf("a package was not built on a public base: %s", ran) + } + if !strings.Contains(ran, ":/root/.npmrc:ro") { + t.Fatalf("the credential was not mounted read-only for the publish: %s", ran) + } +} + +func TestAPackageWithNoRegistryIsRefused(t *testing.T) { + r, workspace := aRepository(t, aPackage, map[string]string{ + "package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`, + }) + _, err := Build(context.Background(), r.run, r, + "https://forge.invalid/mesh-sdk.git", "", "", workspace, nil, Npmrc{}, nil) + if err == nil { + t.Fatal("a package built with no registry to publish to, silently") + } +} + +func TestNpmrcRendersBasicAuthWhenGivenAUserAndPassword(t *testing.T) { + n := Npmrc{ + Scope: "@novox", + Registry: "http://forge.invalid:3000/api/packages/novox/npm/", + Username: "mesh_anchor_builder", + Password: "s3cret", + } + got, err := n.File() + if err != nil { + t.Fatalf("basic-auth credential did not render: %v", err) + } + key := "//forge.invalid:3000/api/packages/novox/npm/" + if !strings.Contains(got, key+":username=mesh_anchor_builder\n") { + t.Fatalf("username line missing:\n%s", got) + } + // npm reads the password base64-encoded. + if !strings.Contains(got, key+":_password=czNjcmV0\n") { + t.Fatalf("base64 password line missing or wrong:\n%s", got) + } + if !strings.Contains(got, key+":always-auth=true\n") { + t.Fatalf("always-auth missing, so reads would go unauthenticated:\n%s", got) + } + if strings.Contains(got, "_authToken") { + t.Fatalf("a token line was rendered for a basic-auth credential:\n%s", got) + } +} + +func TestNpmrcRefusesWhenGivenNeitherTokenNorPassword(t *testing.T) { + n := Npmrc{Scope: "@novox", Registry: "http://x.invalid/npm/", Username: "u"} + if _, err := n.File(); err == nil { + t.Fatal("a username with no password rendered an .npmrc") + } +} diff --git a/internal/catalogue/build.go b/internal/catalogue/build.go index d7df29a..9ff4336 100644 --- a/internal/catalogue/build.go +++ b/internal/catalogue/build.go @@ -86,6 +86,12 @@ func (m Manifest) Resolve(built []Built) (Manifest, error) { } delete(filled, "artifact") switch artifact.Kind { + case ArtifactPackage: + // A package is not a resource on any machine; it is consumed by other builds. A + // resource that names one is a manifest error, named here rather than shipped. + return Manifest{}, fmt.Errorf( + "%s: %v uses %q, which is a package — a build input, not a resource a machine runs", + m.Module, r["id"], named) case ArtifactImage, ArtifactUpstream: filled["image"] = artifact.Reference case ArtifactArchive, ArtifactBundle: @@ -123,19 +129,19 @@ func (b *Build) problems(module string) []string { } seen[a.Name] = true switch a.Kind { - case ArtifactImage, ArtifactArchive, ArtifactUpstream, ArtifactBundle: + case ArtifactImage, ArtifactArchive, ArtifactUpstream, ArtifactBundle, ArtifactPackage: default: problems = append(problems, fmt.Sprintf( "%s: %q is a %q, and an artifact is %q, %q, %q or %q", module, a.Name, a.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream, - ArtifactBundle)) + ArtifactBundle+", "+ArtifactPackage)) } // **A bundle is built from the module itself, so it says a language instead.** Everything // else names what it is built from: a Dockerfile, a directory, somebody else's reference. // A bundle's source is the module's own directory by definition, and what it needs to say // is which compiler — because the mesh chooses that, and cannot choose for a module that // has not said. - if a.Kind == ArtifactBundle { + if a.Kind == ArtifactBundle || a.Kind == ArtifactPackage { if a.From != "" { problems = append(problems, fmt.Sprintf( "%s: %q is a bundle and names what it is built from (%q). A bundle is built "+ diff --git a/internal/catalogue/build_test.go b/internal/catalogue/build_test.go index cba3ed2..34a8694 100644 --- a/internal/catalogue/build_test.go +++ b/internal/catalogue/build_test.go @@ -142,3 +142,41 @@ func TestAnArtifactOfAnUnknownKindIsRefused(t *testing.T) { t.Fatal("an artifact of an unknown kind was accepted") } } + +func TestAPackageParsesLikeABundleAndNeedsALanguage(t *testing.T) { + // The SDK's shape: a package built from the module's own directory, naming a language. + m, err := ParseManifest([]byte(`{"module":"mesh-sdk","version":"1","slug":"sdk", + "build":{"artifacts":[{"name":"lib","kind":"package","language":"typescript"}]}, + "resources":[]}`)) + if err != nil { + t.Fatalf("the SDK's package manifest did not parse: %v", err) + } + if m.Build.Artifacts[0].Kind != ArtifactPackage { + t.Fatalf("expected a package artifact, got %q", m.Build.Artifacts[0].Kind) + } + + // A package that names what it is built from is refused, exactly as a bundle is: it is built + // from the module's own directory. + if _, err := ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[ + {"name":"lib","kind":"package","language":"typescript","from":"Dockerfile"}]}}`)); err == nil { + t.Fatal("a package naming a source was accepted") + } + // A package with no language cannot choose a toolchain. + if _, err := ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[ + {"name":"lib","kind":"package"}]}}`)); err == nil { + t.Fatal("a package with no language was accepted") + } +} + +func TestAResourceNamingAPackageIsRefused(t *testing.T) { + m, err := ParseManifest([]byte(`{"module":"a","version":"1","slug":"a", + "build":{"artifacts":[{"name":"lib","kind":"package","language":"typescript"}]}, + "resources":[{"id":"svc","type":"container","name":"a","artifact":"lib"}]}`)) + if err != nil { + t.Fatalf("parse: %v", err) + } + _, err = m.Resolve([]Built{{Name: "lib", Kind: ArtifactPackage, Reference: "@novox/a@1.0.0"}}) + if err == nil { + t.Fatal("a resource backed by a package was accepted; a package is not a resource") + } +} diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index 03ba034..38585f6 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -463,6 +463,13 @@ const ( // Mirroring is what the bootstrap already does by hand: the lab stocks upstream images into // the registry a first node pulls from. This makes that a thing a module can say. ArtifactUpstream = "upstream" + + // ArtifactPackage is this module's own code, compiled and published to the mesh's package + // registry by version, for other modules to consume when they are built — the SDK above all + // (novox/hq ADR 0076). Like a bundle it is built from the module's own directory and names a + // language; unlike a bundle it is not a resource on any machine, it is a build input. It is + // compiled on a PUBLIC base, never the mesh toolchain, because the toolchain is built from it. + ArtifactPackage = "package" ) // ArtifactStoreProvision is the name a module offers when it is the mesh's store for what modules