The builder resolves the SDK from the mesh registry, and can publish packages

A new 'package' artifact kind builds a module's own code on a public base image
and publishes it to the mesh's package registry by version (hq ADR 0076) — the
SDK above all, which the toolchain is built from and so cannot be built in the
toolchain. The credential a build needs to resolve or publish packages is
rendered as an .npmrc (basic auth, hq ADR 0048) and given to an image build as a
buildkit secret, never a layer, so a token is not baked into the toolchain image.

Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx
This commit is contained in:
2026-09-16 10:27:26 +02:00
parent 2fb700d61b
commit 4b9bc50aad
10 changed files with 588 additions and 26 deletions
+93 -5
View File
@@ -55,6 +55,11 @@ is dialled except the broker.
MESH_BROKER_FILE a file the mesh sealed to this machine holding the same MESH_BROKER_FILE a file the mesh sealed to this machine holding the same
MESH_REGISTRY host:port to publish artifacts to, when the mesh has not said MESH_REGISTRY host:port to publish artifacts to, when the mesh has not said
MESH_BINDING a file the mesh wrote saying where the artifact store is MESH_BINDING a file the mesh wrote saying where the artifact store is
MESH_PACKAGE_BINDING a file the mesh wrote saying where the package registry is
MESH_NPM_TOKEN_FILE a file the mesh sealed holding the token for it
MESH_NPM_REGISTRY a package registry URL, when the mesh has not said (a person, the bootstrap)
MESH_NPM_TOKEN the token for it, likewise
MESH_NPM_SCOPE the scope it answers for (default: @novox)
MESH_WORKSPACE where to clone and build (default: a temporary directory) MESH_WORKSPACE where to clone and build (default: a temporary directory)
It also builds one module and stops, which is how a mesh is raised — before there is a It also builds one module and stops, which is how a mesh is raised — before there is a
@@ -189,11 +194,18 @@ func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publis
} }
fmt.Fprintln(os.Stderr) fmt.Fprintln(os.Stderr)
built, err := builder.Build(ctx, builder.Command, publisher, npmrc, err := packagesFrom()
request.Repository, request.Path, request.Ref, workspace, request.Held, var built builder.Result
func(step, message string) { if err == nil {
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message) // The package-registry credential is a build input, so it is resolved before the clone: a
}) // build that could not have resolved its dependencies is refused in front of the reason,
// not after a clone that then fails at npm ci.
built, err = builder.Build(ctx, builder.Command, publisher,
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
func(step, message string) {
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
})
}
if err != nil { if err != nil {
// A failure is a result. A build that fails and says nothing is indistinguishable from a // A failure is a result. A build that fails and says nothing is indistinguishable from a
// builder that is not running, and those want completely different responses. // builder that is not running, and those want completely different responses.
@@ -279,6 +291,82 @@ func moduleOf(manifest json.RawMessage) string {
return named.Module return named.Module
} }
// packagesFrom is where a build resolves the mesh's own published packages — the SDK above all
// (novox/hq ADR 0076, issue 053).
//
// Preferably from the mesh: a package-registry binding names the endpoint the way the artifact
// store's binding does, and a sealed token file the credential the way the broker's does. The
// environment variables remain for a builder run by a person, and for the bootstrap, where there is
// no registry yet — there the result is disabled and a build that needs no mesh-published dependency
// builds anyway.
func packagesFrom() (builder.Npmrc, error) {
scope := strings.TrimSpace(os.Getenv("MESH_NPM_SCOPE"))
if scope == "" {
scope = "@novox"
}
registry := strings.TrimSpace(os.Getenv("MESH_NPM_REGISTRY"))
var username string
if path := strings.TrimSpace(os.Getenv("MESH_PACKAGE_BINDING")); path != "" {
raw, err := os.ReadFile(path)
if err != nil {
return builder.Npmrc{}, fmt.Errorf("cannot read what the mesh said about the package registry: %w", err)
}
var told struct {
From string `json:"from"`
At string `json:"at"`
As string `json:"as"`
Serves map[string]any `json:"serves"`
}
if err := json.Unmarshal(raw, &told); err != nil {
return builder.Npmrc{}, fmt.Errorf("%s is not a binding: %w", path, err)
}
if told.At == "" {
return builder.Npmrc{}, fmt.Errorf(
"%s says the package registry is on %q and gives no address for it", path, told.From)
}
// Composed from what the provider serves, so nothing here knows gitea's URL shape from
// another registry's: it states its port, the path its registry answers on, and the scheme.
scheme := "https"
if s, ok := told.Serves["scheme"]; ok {
scheme = fmt.Sprintf("%v", s)
}
port, ok := told.Serves["port"]
if !ok {
return builder.Npmrc{}, fmt.Errorf("%s says nothing about which port the package registry answers on", path)
}
npmPath, ok := told.Serves["npm-path"]
if !ok {
return builder.Npmrc{}, fmt.Errorf("%s says nothing about the path the package registry answers on", path)
}
registry = fmt.Sprintf("%s://%s:%v%v", scheme, told.At, port, npmPath)
username = told.As
}
// The credential the mesh sealed to this machine. The mesh authenticates the ordinary way — a
// generated password the provider only applies (novox/hq ADR 0048) — so with a username this is
// a password (basic auth); without one it is a bearer token a provider minted.
secret := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN"))
if path := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN_FILE")); path != "" {
raw, err := os.ReadFile(path)
if err != nil {
return builder.Npmrc{}, fmt.Errorf("cannot read this builder's package-registry credential: %w", err)
}
secret = strings.TrimSpace(string(raw))
}
if u := strings.TrimSpace(os.Getenv("MESH_NPM_USER")); u != "" {
username = u
}
if registry == "" && secret == "" {
return builder.Npmrc{}, nil
}
if username != "" {
return builder.Npmrc{Scope: scope, Registry: registry, Username: username, Password: secret}, nil
}
return builder.Npmrc{Scope: scope, Registry: registry, Token: secret}, nil
}
func short(commit string) string { func short(commit string) string {
if len(commit) > 8 { if len(commit) > 8 {
return commit[:8] return commit[:8]
+5 -1
View File
@@ -84,7 +84,11 @@ func buildOnce(ctx context.Context, args []string) error {
} }
fmt.Fprintln(os.Stderr) fmt.Fprintln(os.Stderr)
built, buildErr := builder.Build(ctx, builder.Command, publisher, repository, *path, *ref, where, bases, npmrc, err := packagesFrom()
if err != nil {
return err
}
built, buildErr := builder.Build(ctx, builder.Command, publisher, repository, *path, *ref, where, bases, npmrc,
func(step, message string) { fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message) }) func(step, message string) { fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message) })
if buildErr != nil { if buildErr != nil {
return buildErr return buildErr
+99 -3
View File
@@ -6,6 +6,7 @@ import (
"context" "context"
"crypto/sha256" "crypto/sha256"
"encoding/hex" "encoding/hex"
"encoding/json"
"fmt" "fmt"
"io" "io"
"os" "os"
@@ -68,7 +69,7 @@ type Result struct {
// archive failed would otherwise leave half of itself in the store under a digest the mesh never // archive failed would otherwise leave half of itself in the store under a digest the mesh never
// records — reachable, unreferenced, and indistinguishable from something in use. // records — reachable, unreferenced, and indistinguishable from something in use.
func Build(ctx context.Context, run Runner, publish Publisher, func Build(ctx context.Context, run Runner, publish Publisher,
repository, path, ref, workspace string, held map[string]string, log Log) (Result, error) { repository, path, ref, workspace string, held map[string]string, npmrc Npmrc, log Log) (Result, error) {
say := logging(log) say := logging(log)
say("clone", "%s%s at %s", repository, describePath(path), refOrHead(ref)) say("clone", "%s%s at %s", repository, describePath(path), refOrHead(ref))
@@ -124,6 +125,22 @@ func Build(ctx context.Context, run Runner, publish Publisher,
} }
say("manifest", "%s v%s — %d artifact(s)", manifest.Module, manifest.Version, artifactCount(manifest)) say("manifest", "%s v%s — %d artifact(s)", manifest.Module, manifest.Version, artifactCount(manifest))
// A build-time credential, written where a build can mount it but never where it can be copied
// into an image or committed: under the workspace, beside the clone, not inside it. Absent when
// this mesh has no package registry yet, which is the bootstrap case (novox/hq ADR 0076).
var npmrcPath string
if npmrc.Enabled() {
content, err := npmrc.File()
if err != nil {
return Result{}, err
}
npmrcPath = filepath.Join(workspace, "npmrc")
if err := os.WriteFile(npmrcPath, []byte(content), 0o600); err != nil {
return Result{}, fmt.Errorf("cannot write the package-registry credential for the build: %w", err)
}
say("packages", "resolving %s from the mesh's package registry", npmrc.Scope)
}
var built []catalogue.Built var built []catalogue.Built
if manifest.Build != nil { if manifest.Build != nil {
// What this module said it stands on, answered with what this mesh actually holds. Done // What this module said it stands on, answered with what this mesh actually holds. Done
@@ -143,7 +160,7 @@ func Build(ctx context.Context, run Runner, publish Publisher,
sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name }) sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name })
for _, a := range artifacts { for _, a := range artifacts {
say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a)) say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a))
made, err := one(ctx, run, publish, manifest.Module, within, commit, a, args, held, say) made, err := one(ctx, run, publish, manifest.Module, within, commit, a, args, held, npmrcPath, say)
if err != nil { if err != nil {
say("artifact", "%s FAILED: %v", a.Name, err) say("artifact", "%s FAILED: %v", a.Name, err)
return Result{}, err return Result{}, err
@@ -282,7 +299,7 @@ const ManifestName = "module.json"
func one(ctx context.Context, run Runner, publish Publisher, func one(ctx context.Context, run Runner, publish Publisher,
module, tree, commit string, a catalogue.Artifact, args []string, module, tree, commit string, a catalogue.Artifact, args []string,
held map[string]string, say func(step, format string, args ...any)) (catalogue.Built, error) { held map[string]string, npmrc string, say func(step, format string, args ...any)) (catalogue.Built, error) {
switch a.Kind { switch a.Kind {
case catalogue.ArtifactUpstream: case catalogue.ArtifactUpstream:
@@ -311,6 +328,12 @@ func one(ctx context.Context, run Runner, publish Publisher,
if a.Target != "" { if a.Target != "" {
invocation = append(invocation, "--target", a.Target) invocation = append(invocation, "--target", a.Target)
} }
if npmrc != "" {
// Given to the build as a buildkit secret, so a RUN that needs the package registry mounts
// it at that step and it is in no image layer. A Dockerfile that does not ask for it is
// unaffected; the secret is simply not read (novox/hq ADR 0076).
invocation = append(invocation, "--secret", "id=npmrc,src="+npmrc)
}
invocation = append(invocation, ".") invocation = append(invocation, ".")
say("image", "docker build -f %s", a.From) say("image", "docker build -f %s", a.From)
if _, err := run(ctx, tree, "docker", invocation...); err != nil { if _, err := run(ctx, tree, "docker", invocation...); err != nil {
@@ -365,6 +388,19 @@ func one(ctx context.Context, run Runner, publish Publisher,
} }
return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: where, Digest: digest}, nil return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: where, Digest: digest}, nil
case catalogue.ArtifactPackage:
// Built and published on a public base, to the mesh's package registry, by version
// (novox/hq ADR 0076). Not an image, not an archive: nothing on a machine references it, so
// there is no Publisher call — the container itself publishes, with the credential the
// build was handed.
say("package", "building and publishing %s (%s)", a.Name, a.Language)
reference, err := publishPackage(ctx, run, module, tree, a, npmrc, say)
if err != nil {
return catalogue.Built{}, fmt.Errorf("%s: publishing %s failed: %w", module, a.Name, err)
}
say("package", "published %s", reference)
return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil
case catalogue.ArtifactArchive: case catalogue.ArtifactArchive:
body, err := pack(filepath.Join(tree, a.From)) body, err := pack(filepath.Join(tree, a.From))
if err != nil { if err != nil {
@@ -541,6 +577,66 @@ func standingOn(manifest catalogue.Manifest, held map[string]string) ([]string,
// where its dependencies resolve upward into the base's own library directory, so what it is // where its dependencies resolve upward into the base's own library directory, so what it is
// compiled against is exactly what it will run against — the reason every hand-written Dockerfile // compiled against is exactly what it will run against — the reason every hand-written Dockerfile
// had to choose a working directory carefully, and the reason none of them has to now. // had to choose a working directory carefully, and the reason none of them has to now.
// publishPackage builds a `package` artifact on a public base image and publishes it to the mesh's
// package registry by version. The credential arrives as an .npmrc file the build was handed
// (novox/hq ADR 0076); it is mounted read-only into the container rather than baked, because a
// package build produces no image to leak it into. The reference returned is name@version, read from
// the module's own package.json — the same two fields npm publishes under.
func publishPackage(ctx context.Context, run Runner, module, dir string, a catalogue.Artifact,
npmrc string, say func(step, format string, args ...any)) (string, error) {
recipe, ok := packageRecipes[a.Language]
if !ok {
return "", fmt.Errorf(
"a package written in %q cannot be built: no public toolchain is known for it", a.Language)
}
if npmrc == "" {
// A package with nowhere to be published is not built. Said here rather than failing inside
// npm publish with a message about a registry that is simply absent.
return "", fmt.Errorf(
"%s is a package and this build was given no package registry to publish it to", a.Name)
}
raw, err := os.ReadFile(filepath.Join(dir, "package.json"))
if err != nil {
return "", fmt.Errorf("a package is published by name and version, and %s has no package.json: %w", module, err)
}
var pkg struct {
Name string `json:"name"`
Version string `json:"version"`
}
if err := json.Unmarshal(raw, &pkg); err != nil {
return "", fmt.Errorf("%s's package.json is not readable: %w", module, err)
}
if pkg.Name == "" || pkg.Version == "" {
return "", fmt.Errorf("%s's package.json names no %s to publish under",
module, either(pkg.Name == "", "name", "version"))
}
const within = "/app/module"
invocation := []string{
"run", "--rm",
"--volume", dir + ":" + within,
// Read-only, so a build cannot alter the credential, and at /root where npm reads it.
"--volume", npmrc + ":/root/.npmrc:ro",
"--workdir", within,
recipe.Base,
"sh", "-c", recipe.Script,
}
if _, err := run(ctx, dir, "docker", invocation...); err != nil {
return "", err
}
return pkg.Name + "@" + pkg.Version, nil
}
// either names whichever of two fields is the missing one, for a message that says which.
func either(first bool, a, b string) string {
if first {
return a
}
return b
}
func compile(ctx context.Context, run Runner, tree string, chain Toolchain, func compile(ctx context.Context, run Runner, tree string, chain Toolchain,
base string, a catalogue.Artifact) (string, error) { base string, a catalogue.Artifact) (string, error) {
+10 -10
View File
@@ -108,7 +108,7 @@ func TestABuildProducesAManifestThePinsAreIn(t *testing.T) {
r, workspace := aRepository(t, withBoth, map[string]string{ r, workspace := aRepository(t, withBoth, map[string]string{
"Dockerfile": "FROM scratch", "files/theme.conf": "dark", "Dockerfile": "FROM scratch", "files/theme.conf": "dark",
}) })
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, nil) got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, Npmrc{}, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -134,7 +134,7 @@ func TestTwoBuildsOfOneCommitProduceOneDigest(t *testing.T) {
}) })
// A year apart, so a packer carrying timestamps cannot accidentally agree. // A year apart, so a packer carrying timestamps cannot accidentally agree.
r.stamped = time.Date(2020+i, time.March, 3, 4, 5, 6, 0, time.UTC) r.stamped = time.Date(2020+i, time.March, 3, 4, 5, 6, 0, time.UTC)
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, nil) got, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -154,7 +154,7 @@ func TestNothingIsPublishedUntilEverythingIsBuilt(t *testing.T) {
// unreferenced, and indistinguishable from something in use. // unreferenced, and indistinguishable from something in use.
r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch"}) r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch"})
// `files` is missing, so packing the archive fails — after the image would have been pushed. // `files` is missing, so packing the archive fails — after the image would have been pushed.
_, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, nil) _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, nil)
if err == nil { if err == nil {
t.Fatal("a build with a missing input succeeded") t.Fatal("a build with a missing input succeeded")
} }
@@ -166,7 +166,7 @@ func TestNothingIsPublishedUntilEverythingIsBuilt(t *testing.T) {
func TestARepositoryWithNoManifestSaysSo(t *testing.T) { func TestARepositoryWithNoManifestSaysSo(t *testing.T) {
workspace := t.TempDir() workspace := t.TempDir()
r := &recorded{contents: map[string]string{"README.md": "nothing to see"}} r := &recorded{contents: map[string]string{"README.md": "nothing to see"}}
_, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, nil) _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, nil)
if err == nil { if err == nil {
t.Fatal("a repository with nothing saying what it is was built") t.Fatal("a repository with nothing saying what it is was built")
} }
@@ -179,7 +179,7 @@ func TestAModuleThatBuildsNothingStillProducesAManifest(t *testing.T) {
// Most of what a person installs is configuration. // Most of what a person installs is configuration.
r, workspace := aRepository(t, `{"module":"shell","version":"1","resources":[ r, workspace := aRepository(t, `{"module":"shell","version":"1","resources":[
{"id":"rc","type":"file","path":"/etc/zsh/zshrc","content":"setopt"}]}`, nil) {"id":"rc","type":"file","path":"/etc/zsh/zshrc","content":"setopt"}]}`, nil)
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/shell.git", "", "", workspace, nil, nil) got, err := Build(context.Background(), r.run, r, "https://forge.invalid/shell.git", "", "", workspace, nil, Npmrc{}, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -209,7 +209,7 @@ func TestTheTreeIsFreshEveryTime(t *testing.T) {
if err := os.WriteFile(leftover, []byte("stale"), 0o644); err != nil { if err := os.WriteFile(leftover, []byte("stale"), 0o644); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, nil); err != nil { if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, nil); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if _, err := os.Stat(leftover); err == nil { if _, err := os.Stat(leftover); err == nil {
@@ -222,7 +222,7 @@ func TestABuildThatCannotPushFails(t *testing.T) {
"Dockerfile": "FROM scratch", "files/a": "b", "Dockerfile": "FROM scratch", "files/a": "b",
}) })
r.failPush = true r.failPush = true
if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, nil); err == nil { if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, nil); err == nil {
t.Fatal("a build that could publish nothing reported success") t.Fatal("a build that could publish nothing reported success")
} }
} }
@@ -236,7 +236,7 @@ func TestAnUpstreamImageIsMirroredRatherThanBuilt(t *testing.T) {
"resources":[{"id":"db","type":"container","name":"mesh-postgres","artifact":"store"}]}` "resources":[{"id":"db","type":"container","name":"mesh-postgres","artifact":"store"}]}`
r, workspace := aRepository(t, mirrors, nil) r, workspace := aRepository(t, mirrors, nil)
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/postgres.git", "", "", workspace, nil, nil) got, err := Build(context.Background(), r.run, r, "https://forge.invalid/postgres.git", "", "", workspace, nil, Npmrc{}, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -302,7 +302,7 @@ func TestAModuleIsBuiltFromItsPathWithinTheRepository(t *testing.T) {
"modules/other/" + ManifestName: `{"module":"other","version":"1"}`, "modules/other/" + ManifestName: `{"module":"other","version":"1"}`,
}} }}
got, err := Build(context.Background(), r.run, r, got, err := Build(context.Background(), r.run, r,
"https://forge.invalid/catalogue.git", "modules/shell", "", t.TempDir(), nil, nil) "https://forge.invalid/catalogue.git", "modules/shell", "", t.TempDir(), nil, Npmrc{}, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -321,7 +321,7 @@ func TestAPathThatLeavesTheRepositoryIsRefused(t *testing.T) {
for _, escaping := range []string{"../../etc", "/etc"} { for _, escaping := range []string{"../../etc", "/etc"} {
r := &recorded{contents: map[string]string{ManifestName: withBoth}} r := &recorded{contents: map[string]string{ManifestName: withBoth}}
_, err := Build(context.Background(), r.run, r, _, err := Build(context.Background(), r.run, r,
"https://forge.invalid/x.git", escaping, "", t.TempDir(), nil, nil) "https://forge.invalid/x.git", escaping, "", t.TempDir(), nil, Npmrc{}, nil)
if err == nil { if err == nil {
t.Fatalf("%q was accepted as a module's path", escaping) t.Fatalf("%q was accepted as a module's path", escaping)
} }
+4 -4
View File
@@ -54,7 +54,7 @@ func TestABundleIsCompiledAndPackedWithNoDockerfile(t *testing.T) {
held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)}
got, err := Build(context.Background(), compiling{r}.run, r, got, err := Build(context.Background(), compiling{r}.run, r,
"https://forge.invalid/greeter.git", "", "", workspace, held, nil) "https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, nil)
if err != nil { if err != nil {
t.Fatalf("a module with a language and no Dockerfile did not build: %v", err) t.Fatalf("a module with a language and no Dockerfile did not build: %v", err)
} }
@@ -91,7 +91,7 @@ func TestABundleWhoseToolchainIsNotHeldIsRefusedFirst(t *testing.T) {
r, workspace := aRepository(t, aBundle, map[string]string{"index.ts": "console.log(1)"}) r, workspace := aRepository(t, aBundle, map[string]string{"index.ts": "console.log(1)"})
_, err := Build(context.Background(), compiling{r}.run, r, _, err := Build(context.Background(), compiling{r}.run, r,
"https://forge.invalid/greeter.git", "", "", workspace, nil, nil) "https://forge.invalid/greeter.git", "", "", workspace, nil, Npmrc{}, nil)
if err == nil { if err == nil {
t.Fatal("a bundle was built with no toolchain to compile it in") t.Fatal("a bundle was built with no toolchain to compile it in")
} }
@@ -112,7 +112,7 @@ func TestABundleInAnUnknownLanguageIsRefused(t *testing.T) {
_, err := Build(context.Background(), compiling{r}.run, r, _, err := Build(context.Background(), compiling{r}.run, r,
"https://forge.invalid/greeter.git", "", "", workspace, "https://forge.invalid/greeter.git", "", "", workspace,
map[string]string{"mesh-tools/build": "registry.invalid/x@sha256:" + strings.Repeat("c", 64)}, nil) map[string]string{"mesh-tools/build": "registry.invalid/x@sha256:" + strings.Repeat("c", 64)}, Npmrc{}, nil)
if err == nil { if err == nil {
t.Fatal("a language nothing can compile was accepted") t.Fatal("a language nothing can compile was accepted")
} }
@@ -140,7 +140,7 @@ func TestTwoBundlesInOneModuleArePackedSeparately(t *testing.T) {
held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)}
got, err := Build(context.Background(), compiling{r}.run, r, got, err := Build(context.Background(), compiling{r}.run, r,
"https://forge.invalid/greeter.git", "", "", workspace, held, nil) "https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, nil)
if err != nil { if err != nil {
t.Fatalf("a module with two bundles did not build: %v", err) t.Fatalf("a module with two bundles did not build: %v", err)
} }
+127
View File
@@ -0,0 +1,127 @@
package builder
import (
"encoding/base64"
"fmt"
"net/url"
"strings"
)
// Npmrc is what a build needs to resolve the mesh's own shared library — and any other package the
// mesh publishes — from the mesh's package registry rather than from a git URL (novox/hq ADR 0076,
// issue 053).
//
// It is a build-time credential, not a runtime one. A module compiled inside the toolchain image
// resolves the SDK there, once, when that image is built; the running container never speaks to the
// package registry. So this is given to the *builder*, the way the artifact store is
// (`whereToPublish`), and reaches a build as a secret rather than a layer — see Secret.
//
// The registry's exact URL shape is the provider's business, not the builder's: it arrives whole,
// either from the binding the mesh writes (a package-registry provider's `serves` facts) or from the
// environment when a person runs a build by hand. Nothing here knows gitea from verdaccio.
type Npmrc struct {
// Scope is the npm scope the registry answers for, e.g. "@novox". A build resolves only this
// scope from the mesh; everything else resolves the ordinary way, so a mesh with no internet
// still cannot pull the public registry's version of a name the mesh also publishes.
Scope string
// Registry is the full base URL a client uses for this scope, e.g.
// "https://<forge>/api/packages/<owner>/npm/". Trailing slash tolerated either way.
Registry string
// Token authenticates to the registry as a bearer token, when a provider mints one. Left empty
// when the mesh authenticates the ordinary way it authenticates everything — a generated
// password it applies and seals — for which see Username and Password.
Token string
// Username and Password authenticate by basic auth, which is what gitea and verdaccio both
// accept and what lets the credential be a mesh-generated password the provider's provisioner
// applies and the mesh seals to the consumer — the same shape a database password takes. The
// username is the consumer's mesh identity. Ignored when Token is set.
Username string
Password string
}
// Enabled reports whether there is a registry to resolve against at all. A bootstrap build that
// runs before any package registry exists has none, and must still build whatever needs no
// mesh-published dependency.
func (n Npmrc) Enabled() bool {
return strings.TrimSpace(n.Scope) != "" && strings.TrimSpace(n.Registry) != ""
}
// File renders the .npmrc a build mounts. Two lines: which registry answers for the scope, and the
// token to present to it. The auth line is keyed by the registry URL with its scheme removed, which
// is how npm matches a stored credential to a request.
//
// It returns an error rather than a malformed file, because an .npmrc that npm parses but points
// nowhere fails much later, inside a build, as a package that cannot be found.
func (n Npmrc) File() (string, error) {
scope := strings.TrimSpace(n.Scope)
if !strings.HasPrefix(scope, "@") {
return "", fmt.Errorf("a package-registry scope is written with its leading @, not %q", scope)
}
reg := strings.TrimSpace(n.Registry)
if !strings.HasPrefix(reg, "http://") && !strings.HasPrefix(reg, "https://") {
return "", fmt.Errorf("a package registry is reached over http(s), and %q is neither", reg)
}
if !strings.HasSuffix(reg, "/") {
// npm's per-scope registry key is matched by prefix, and the auth key below is derived from
// it; a missing trailing slash makes the two disagree and the token is never sent.
reg += "/"
}
parsed, err := url.Parse(reg)
if err != nil {
return "", fmt.Errorf("%q is not a usable registry URL: %w", reg, err)
}
// The auth key is the URL without its scheme, e.g. "//host/api/packages/owner/npm/".
authKey := "//" + parsed.Host + parsed.EscapedPath()
var auth string
switch {
case strings.TrimSpace(n.Token) != "":
auth = fmt.Sprintf("%s:_authToken=%s\n", authKey, strings.TrimSpace(n.Token))
case strings.TrimSpace(n.Username) != "" && n.Password != "":
// npm reads the password base64-encoded, and always-auth so it presents the credential to
// reads as well as writes — a private registry answers neither without it.
enc := base64.StdEncoding.EncodeToString([]byte(n.Password))
auth = fmt.Sprintf("%s:username=%s\n%s:_password=%s\n%s:always-auth=true\n",
authKey, strings.TrimSpace(n.Username), authKey, enc, authKey)
default:
return "", fmt.Errorf(
"the package registry at %s was given neither a token nor a username and password", reg)
}
return fmt.Sprintf("%s:registry=%s\n%s", scope, reg, auth), nil
}
// packageRecipe is how a `package` artifact is built and published: on a PUBLIC base image, never
// the mesh toolchain, because the toolchain is built from the package this produces (the SDK). The
// script builds the module, then publishes it to the mesh's package registry unless that exact
// version is already there — so a re-run of genesis, which must be safe, does not fail on a version
// it published a moment ago.
type packageRecipe struct {
Base string
Script string
}
var packageRecipes = map[string]packageRecipe{
"typescript": {
Base: "node:22-bookworm-slim",
Script: `set -e
npm install --no-audit --no-fund
npm run build
name="$(node -p "require('./package.json').name")"
ver="$(node -p "require('./package.json').version")"
if npm view "$name@$ver" version >/dev/null 2>&1; then
echo "mesh-builder: $name@$ver is already published, leaving it"
else
npm publish
fi`,
},
}
// PackageLanguages is the languages a package artifact can be written in, for a manifest check that
// wants to refuse one it cannot build before a build starts.
func PackageLanguages() []string {
out := make([]string, 0, len(packageRecipes))
for l := range packageRecipes {
out = append(out, l)
}
return out
}
+196
View File
@@ -0,0 +1,196 @@
package builder
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
)
func TestNpmrcRendersRegistryAndTokenForTheScope(t *testing.T) {
n := Npmrc{
Scope: "@novox",
Registry: "https://forge.invalid/api/packages/novox/npm/",
Token: "a-token",
}
got, err := n.File()
if err != nil {
t.Fatalf("a complete credential did not render: %v", err)
}
if !strings.Contains(got, "@novox:registry=https://forge.invalid/api/packages/novox/npm/") {
t.Fatalf("the scope's registry line is missing:\n%s", got)
}
// The auth line is keyed by the URL without its scheme, or npm never sends the token.
if !strings.Contains(got, "//forge.invalid/api/packages/novox/npm/:_authToken=a-token") {
t.Fatalf("the auth line does not match the registry key:\n%s", got)
}
}
func TestNpmrcAddsATrailingSlashSoTheAuthKeyMatches(t *testing.T) {
n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm", Token: "t"}
got, err := n.File()
if err != nil {
t.Fatal(err)
}
if !strings.Contains(got, "registry=https://forge.invalid/api/packages/novox/npm/\n") {
t.Fatalf("a missing trailing slash was not normalised:\n%s", got)
}
}
func TestNpmrcRefusesTheHalfConfigured(t *testing.T) {
cases := map[string]Npmrc{
"scope without @": {Scope: "novox", Registry: "https://x.invalid/", Token: "t"},
"registry not http": {Scope: "@novox", Registry: "ftp://x.invalid/", Token: "t"},
"no token": {Scope: "@novox", Registry: "https://x.invalid/", Token: ""},
}
for name, n := range cases {
if _, err := n.File(); err == nil {
t.Fatalf("%s rendered an .npmrc rather than refusing", name)
}
}
}
func TestNpmrcDisabledUntilThereIsARegistry(t *testing.T) {
if (Npmrc{}).Enabled() {
t.Fatal("an empty credential reported itself usable")
}
if (Npmrc{Scope: "@novox"}).Enabled() {
t.Fatal("a scope with no registry reported itself usable")
}
if !(Npmrc{Scope: "@novox", Registry: "https://x.invalid/"}).Enabled() {
t.Fatal("a scope and a registry did not count as usable")
}
}
// The credential reaches an image build as a buildkit secret and never as a file inside the build
// context, because a token copied into a layer is a token published (novox/hq ADR 0076).
func TestAnImageBuildGetsTheCredentialAsASecretNotALayer(t *testing.T) {
r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch", "files/x": "y"})
n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"}
if _, err := Build(context.Background(), r.run, r,
"https://forge.invalid/meshboard.git", "", "", workspace, nil, n, nil); err != nil {
t.Fatalf("the build failed: %v", err)
}
var build string
for _, line := range r.ran {
if strings.HasPrefix(line, "docker build") {
build = line
}
}
if build == "" {
t.Fatal("no docker build ran")
}
if !strings.Contains(build, "--secret id=npmrc,src=") {
t.Fatalf("the build was not given the credential as a secret: %s", build)
}
// The .npmrc lives under the workspace, beside the clone, never inside the source tree that is
// the docker context.
tree := filepath.Join(workspace, "source")
src := strings.SplitN(strings.SplitN(build, "--secret id=npmrc,src=", 2)[1], " ", 2)[0]
if strings.HasPrefix(src, tree+string(os.PathSeparator)) {
t.Fatalf("the credential file %s is inside the build context %s", src, tree)
}
if _, err := os.Stat(src); err != nil {
t.Fatalf("the credential file the build was pointed at does not exist: %v", err)
}
}
func TestAnImageBuildWithoutACredentialGetsNoSecret(t *testing.T) {
r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch", "files/x": "y"})
if _, err := Build(context.Background(), r.run, r,
"https://forge.invalid/meshboard.git", "", "", workspace, nil, Npmrc{}, nil); err != nil {
t.Fatalf("the build failed: %v", err)
}
for _, line := range r.ran {
if strings.HasPrefix(line, "docker build") && strings.Contains(line, "--secret") {
t.Fatalf("a build with no credential was still given a secret: %s", line)
}
}
}
const aPackage = `{"module":"mesh-sdk","version":"1",
"build":{"artifacts":[{"name":"lib","kind":"package","language":"typescript"}]},
"resources":[]}`
// A package is compiled on a public base and published to the mesh's package registry by version,
// with nothing pushed to the artifact store and the credential mounted, not baked (novox/hq ADR 0076).
func TestAPackageIsBuiltOnAPublicBaseAndPublishedByVersion(t *testing.T) {
r, workspace := aRepository(t, aPackage, map[string]string{
"package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`,
})
n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"}
got, err := Build(context.Background(), r.run, r,
"https://forge.invalid/mesh-sdk.git", "", "", workspace, nil, n, nil)
if err != nil {
t.Fatalf("the package did not build: %v", err)
}
if len(got.Built) != 1 || got.Built[0].Reference != "@novox/mesh-sdk@0.1.0" {
t.Fatalf("a package is published by name and version, got %+v", got.Built)
}
if len(r.images) != 0 || len(r.archives) != 0 {
t.Fatal("a package was pushed to the artifact store, which is not where packages live")
}
var ran string
for _, line := range r.ran {
if strings.HasPrefix(line, "docker run") {
ran = line
}
}
if ran == "" {
t.Fatal("nothing ran to build the package")
}
if !strings.Contains(ran, "node:22-bookworm-slim") {
t.Fatalf("a package was not built on a public base: %s", ran)
}
if !strings.Contains(ran, ":/root/.npmrc:ro") {
t.Fatalf("the credential was not mounted read-only for the publish: %s", ran)
}
}
func TestAPackageWithNoRegistryIsRefused(t *testing.T) {
r, workspace := aRepository(t, aPackage, map[string]string{
"package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`,
})
_, err := Build(context.Background(), r.run, r,
"https://forge.invalid/mesh-sdk.git", "", "", workspace, nil, Npmrc{}, nil)
if err == nil {
t.Fatal("a package built with no registry to publish to, silently")
}
}
func TestNpmrcRendersBasicAuthWhenGivenAUserAndPassword(t *testing.T) {
n := Npmrc{
Scope: "@novox",
Registry: "http://forge.invalid:3000/api/packages/novox/npm/",
Username: "mesh_anchor_builder",
Password: "s3cret",
}
got, err := n.File()
if err != nil {
t.Fatalf("basic-auth credential did not render: %v", err)
}
key := "//forge.invalid:3000/api/packages/novox/npm/"
if !strings.Contains(got, key+":username=mesh_anchor_builder\n") {
t.Fatalf("username line missing:\n%s", got)
}
// npm reads the password base64-encoded.
if !strings.Contains(got, key+":_password=czNjcmV0\n") {
t.Fatalf("base64 password line missing or wrong:\n%s", got)
}
if !strings.Contains(got, key+":always-auth=true\n") {
t.Fatalf("always-auth missing, so reads would go unauthenticated:\n%s", got)
}
if strings.Contains(got, "_authToken") {
t.Fatalf("a token line was rendered for a basic-auth credential:\n%s", got)
}
}
func TestNpmrcRefusesWhenGivenNeitherTokenNorPassword(t *testing.T) {
n := Npmrc{Scope: "@novox", Registry: "http://x.invalid/npm/", Username: "u"}
if _, err := n.File(); err == nil {
t.Fatal("a username with no password rendered an .npmrc")
}
}
+9 -3
View File
@@ -86,6 +86,12 @@ func (m Manifest) Resolve(built []Built) (Manifest, error) {
} }
delete(filled, "artifact") delete(filled, "artifact")
switch artifact.Kind { switch artifact.Kind {
case ArtifactPackage:
// A package is not a resource on any machine; it is consumed by other builds. A
// resource that names one is a manifest error, named here rather than shipped.
return Manifest{}, fmt.Errorf(
"%s: %v uses %q, which is a package — a build input, not a resource a machine runs",
m.Module, r["id"], named)
case ArtifactImage, ArtifactUpstream: case ArtifactImage, ArtifactUpstream:
filled["image"] = artifact.Reference filled["image"] = artifact.Reference
case ArtifactArchive, ArtifactBundle: case ArtifactArchive, ArtifactBundle:
@@ -123,19 +129,19 @@ func (b *Build) problems(module string) []string {
} }
seen[a.Name] = true seen[a.Name] = true
switch a.Kind { switch a.Kind {
case ArtifactImage, ArtifactArchive, ArtifactUpstream, ArtifactBundle: case ArtifactImage, ArtifactArchive, ArtifactUpstream, ArtifactBundle, ArtifactPackage:
default: default:
problems = append(problems, fmt.Sprintf( problems = append(problems, fmt.Sprintf(
"%s: %q is a %q, and an artifact is %q, %q, %q or %q", "%s: %q is a %q, and an artifact is %q, %q, %q or %q",
module, a.Name, a.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream, module, a.Name, a.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream,
ArtifactBundle)) ArtifactBundle+", "+ArtifactPackage))
} }
// **A bundle is built from the module itself, so it says a language instead.** Everything // **A bundle is built from the module itself, so it says a language instead.** Everything
// else names what it is built from: a Dockerfile, a directory, somebody else's reference. // else names what it is built from: a Dockerfile, a directory, somebody else's reference.
// A bundle's source is the module's own directory by definition, and what it needs to say // A bundle's source is the module's own directory by definition, and what it needs to say
// is which compiler — because the mesh chooses that, and cannot choose for a module that // is which compiler — because the mesh chooses that, and cannot choose for a module that
// has not said. // has not said.
if a.Kind == ArtifactBundle { if a.Kind == ArtifactBundle || a.Kind == ArtifactPackage {
if a.From != "" { if a.From != "" {
problems = append(problems, fmt.Sprintf( problems = append(problems, fmt.Sprintf(
"%s: %q is a bundle and names what it is built from (%q). A bundle is built "+ "%s: %q is a bundle and names what it is built from (%q). A bundle is built "+
+38
View File
@@ -142,3 +142,41 @@ func TestAnArtifactOfAnUnknownKindIsRefused(t *testing.T) {
t.Fatal("an artifact of an unknown kind was accepted") t.Fatal("an artifact of an unknown kind was accepted")
} }
} }
func TestAPackageParsesLikeABundleAndNeedsALanguage(t *testing.T) {
// The SDK's shape: a package built from the module's own directory, naming a language.
m, err := ParseManifest([]byte(`{"module":"mesh-sdk","version":"1","slug":"sdk",
"build":{"artifacts":[{"name":"lib","kind":"package","language":"typescript"}]},
"resources":[]}`))
if err != nil {
t.Fatalf("the SDK's package manifest did not parse: %v", err)
}
if m.Build.Artifacts[0].Kind != ArtifactPackage {
t.Fatalf("expected a package artifact, got %q", m.Build.Artifacts[0].Kind)
}
// A package that names what it is built from is refused, exactly as a bundle is: it is built
// from the module's own directory.
if _, err := ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[
{"name":"lib","kind":"package","language":"typescript","from":"Dockerfile"}]}}`)); err == nil {
t.Fatal("a package naming a source was accepted")
}
// A package with no language cannot choose a toolchain.
if _, err := ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[
{"name":"lib","kind":"package"}]}}`)); err == nil {
t.Fatal("a package with no language was accepted")
}
}
func TestAResourceNamingAPackageIsRefused(t *testing.T) {
m, err := ParseManifest([]byte(`{"module":"a","version":"1","slug":"a",
"build":{"artifacts":[{"name":"lib","kind":"package","language":"typescript"}]},
"resources":[{"id":"svc","type":"container","name":"a","artifact":"lib"}]}`))
if err != nil {
t.Fatalf("parse: %v", err)
}
_, err = m.Resolve([]Built{{Name: "lib", Kind: ArtifactPackage, Reference: "@novox/a@1.0.0"}})
if err == nil {
t.Fatal("a resource backed by a package was accepted; a package is not a resource")
}
}
+7
View File
@@ -463,6 +463,13 @@ const (
// Mirroring is what the bootstrap already does by hand: the lab stocks upstream images into // Mirroring is what the bootstrap already does by hand: the lab stocks upstream images into
// the registry a first node pulls from. This makes that a thing a module can say. // the registry a first node pulls from. This makes that a thing a module can say.
ArtifactUpstream = "upstream" ArtifactUpstream = "upstream"
// ArtifactPackage is this module's own code, compiled and published to the mesh's package
// registry by version, for other modules to consume when they are built — the SDK above all
// (novox/hq ADR 0076). Like a bundle it is built from the module's own directory and names a
// language; unlike a bundle it is not a resource on any machine, it is a build input. It is
// compiled on a PUBLIC base, never the mesh toolchain, because the toolchain is built from it.
ArtifactPackage = "package"
) )
// ArtifactStoreProvision is the name a module offers when it is the mesh's store for what modules // ArtifactStoreProvision is the name a module offers when it is the mesh's store for what modules