Merge pull request 'mesh-builder: read the genesis document until the registry binds, and refuse a registry build with neither (hq issue 121, ADR 0336)' (#289) from fix/121-genesis-package-registry into main

internal/builder/builder.go

@@ -251,6 +251,20 @@ func build(ctx context.Context, run Runner, publish Publisher,
 	// never resolve a mesh package — making their image non-deterministic (a needless rollout every
 	// build) and leaking the credential into a build stage. Absent entirely with no registry, which
 	// is the bootstrap case (novox/hq ADR 0076).
+	// **A build that needs the registry and has none is refused before anything is built or
+	// published** (novox/hq ADR 0336 §6) — before a base is copied into the store, and never built
+	// without the credential or against the public registry.
+	if !npmrc.Enabled() {
+		why, err := needsRegistry(manifest, within)
+		if err != nil {
+			return Result{}, err
+		}
+		if why != "" {
+			err := npmrc.unbound(manifest.Module, why)
+			say("packages", "REFUSED: %v", err)
+			return Result{}, err
+		}
+	}
 	var npmrcPath string
 	if npmrc.Enabled() && manifest.Build != nil && wantsPackages(manifest, within) {
 		content, err := npmrc.File()
@@ -263,7 +277,7 @@ func build(ctx context.Context, run Runner, publish Publisher,
 		}
 		// Only for as long as the build: a later check's container mounts this workspace (novox/hq issue 462).
 		defer os.Remove(npmrcPath)
-		say("packages", "resolving %s from the mesh's package registry", npmrc.Scope)
+		say("packages", "resolving %s from the mesh's package registry, from %s", npmrc.Scope, fromOr(npmrc.From))
 		src.notPinned("it resolves packages from the mesh's registry at build time")
 	}
 
@@ -701,6 +715,14 @@ func against(within string, manifest catalogue.Manifest, resolved []string) []st
 	return out
 }
 
+// fromOr is where a registry was resolved from, as a build's log says it.
+func fromOr(from string) string {
+	if from == "" {
+		return "what this build was handed"
+	}
+	return from
+}
+
 // ManifestName is the one file a module repository must have.
 //
 // At the root, and named the same in every repository. A convention somebody can look for beats a

internal/builder/packages.go

@@ -2,9 +2,16 @@ package builder
 
 import (
 	"encoding/base64"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"net/url"
+	"os"
+	"path/filepath"
+	"sort"
 	"strings"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
 )
 
 // Npmrc is what a build needs to resolve the mesh's own shared library — and any other package the
@@ -37,8 +44,36 @@ type Npmrc struct {
 	// username is the consumer's mesh identity. Ignored when Token is set.
 	Username string
 	Password string
+
+	// From says where the build agent resolved this registry from (novox/hq ADR 0336 §4): its
+	// npm-package-registry binding, the genesis document the installer sealed to it, or the
+	// environment of a builder run by a person. Empty when it resolved none.
+	From string
+	// Machine is the machine the build agent runs on, named when a build that needs the registry is
+	// refused for having none (ADR 0336 §6).
+	Machine string
 }
 
+// Where a build agent resolves the mesh's package registry from (novox/hq ADR 0336 §3, §4).
+const (
+	// FromBinding is the npm-package-registry binding the mesh wrote, with the credential it sealed.
+	FromBinding = "the npm-package-registry binding"
+	// FromGenesis is the genesis document the installer sealed at genesis's packages step, read only
+	// while no binding is present.
+	FromGenesis = "the genesis document"
+	// FromEnvironment is the environment of a builder run by a person.
+	FromEnvironment = "the environment"
+)
+
+// RegistryRequirement is the build agent's optional requirement the registry is bound through, and
+// RegistryProvider the catalogue module that provides it (novox/hq ADR 0336 §1, §6): named when a
+// build that needs the registry is refused, so the outcome says what to assign. The controller's
+// catalogue test holds the provider to the catalogue's.
+const (
+	RegistryRequirement = "npm-package-registry"
+	RegistryProvider    = "gitea"
+)
+
 // Enabled reports whether there is a registry to resolve against at all. A bootstrap build that
 // runs before any package registry exists has none, and must still build whatever needs no
 // mesh-published dependency.
@@ -90,6 +125,77 @@ func (n Npmrc) File() (string, error) {
 	return fmt.Sprintf("%s:registry=%s\n%s", scope, reg, auth), nil
 }
 
+// needsRegistry says why this module's build needs the mesh's package registry, or "" when it needs
+// none (novox/hq ADR 0336 §5): a `package` artifact, an image whose Dockerfile names .npmrc, or a
+// TypeScript artifact with an `@novox/` dependency. Every Go artifact, every image that does not name
+// .npmrc, and every image pulled upstream builds without it.
+func needsRegistry(manifest catalogue.Manifest, within string) (string, error) {
+	if manifest.Build == nil {
+		return "", nil
+	}
+	for _, a := range manifest.Build.Artifacts {
+		switch {
+		case a.Kind == catalogue.ArtifactPackage:
+			return fmt.Sprintf("%s is a package, published to the registry", a.Name), nil
+		case a.Kind == catalogue.ArtifactImage:
+			raw, err := os.ReadFile(filepath.Join(within, a.From))
+			if err == nil && strings.Contains(string(raw), ".npmrc") {
+				return fmt.Sprintf("%s's Dockerfile names .npmrc", a.Name), nil
+			}
+		case a.Language == "typescript":
+			scoped, err := scopedDependencies(within)
+			if err != nil {
+				return "", err
+			}
+			if len(scoped) > 0 {
+				return fmt.Sprintf("%s is TypeScript and depends on %s", a.Name, strings.Join(scoped, ", ")), nil
+			}
+		}
+	}
+	return "", nil
+}
+
+// scopedDependencies is every `@novox/` package the module's package.json depends on, the SDK
+// included.
+func scopedDependencies(tree string) ([]string, error) {
+	raw, err := os.ReadFile(filepath.Join(tree, "package.json"))
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	var p struct {
+		Dependencies map[string]string `json:"dependencies"`
+	}
+	if err := json.Unmarshal(raw, &p); err != nil {
+		return nil, fmt.Errorf("the module's package.json is not JSON: %w", err)
+	}
+	var names []string
+	for name := range p.Dependencies {
+		if strings.HasPrefix(name, "@novox/") {
+			names = append(names, name)
+		}
+	}
+	sort.Strings(names)
+	return names, nil
+}
+
+// unbound is the refusal of a build that needs the registry on a build agent that has none (novox/hq
+// ADR 0336 §6): it names the machine, the requirement left unbound there and the module that provides
+// it. Nothing is built, nothing is published, and the `@novox` scope is never resolved from the
+// public registry instead.
+func (n Npmrc) unbound(module, why string) error {
+	machine := n.Machine
+	if machine == "" {
+		machine = "this machine"
+	}
+	return fmt.Errorf("%s needs the mesh's package registry (%s), and %s is unbound on %s: the build "+
+		"agent there has neither its binding nor the genesis document. It is provided by the %s module; "+
+		"once %s is assigned the requirement binds (novox/hq ADR 0336 §6)",
+		module, why, RegistryRequirement, machine, RegistryProvider, RegistryProvider)
+}
+
 // packageRecipe is how a `package` artifact is built and published: on a PUBLIC base image, never
 // the mesh toolchain, because the toolchain is built from the package this produces (the SDK). The
 // script builds the module, then publishes it to the mesh's package registry unless that exact

internal/builder/registry_unbound_test.go

@@ -0,0 +1,122 @@
+package builder
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/beside"
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// **A build that needs the registry fails loudly while the build agent has none** (novox/hq ADR 0336
+// §5, §6, issue 121). On a holder with neither the npm-package-registry binding nor the genesis
+// document, a `package` artifact, an image whose Dockerfile names .npmrc and a TypeScript artifact
+// with an `@novox/` dependency each fail before anything is built or published, naming the machine,
+// the unbound requirement and the module that provides it. A Go artifact, and an image that does not
+// name .npmrc, build there as before.
+func TestABuildNeedingTheRegistryIsRefusedOnAnAgentWithoutOne(t *testing.T) {
+	none := Npmrc{Machine: "workstation"}
+	cases := map[string]struct {
+		manifest string
+		files    map[string]string
+		held     map[string]string
+	}{
+		"a package": {aPackage, map[string]string{"package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`}, nil},
+		"an image naming .npmrc": {withBoth, map[string]string{
+			"Dockerfile": "FROM scratch\nCOPY .npmrc /root/.npmrc\n", "files/x": "y"}, nil},
+		"a TypeScript bundle depending on the SDK": {aBundle, map[string]string{
+			"index.ts": "console.log(1)", "package.json": `{"dependencies":{"@novox/mesh-sdk":"^0.1.0"}}`},
+			map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)}},
+	}
+	for name, c := range cases {
+		t.Run(name, func(t *testing.T) {
+			r, workspace := aRepository(t, c.manifest, c.files)
+			var said []string
+			_, err := Build(context.Background(), compiling{r}.run, r,
+				"https://forge.invalid/m.git", "", "", workspace, c.held, none, GitCredential{},
+				func(step, message string) { said = append(said, step+": "+message) })
+			if err == nil {
+				t.Fatal("built with no package registry")
+			}
+			for _, want := range []string{"workstation", "npm-package-registry", "unbound", "gitea"} {
+				if !strings.Contains(err.Error(), want) {
+					t.Errorf("the refusal does not say %q: %v", want, err)
+				}
+			}
+			for _, line := range r.ran {
+				if strings.HasPrefix(line, "docker build") || strings.HasPrefix(line, "docker run") ||
+					strings.HasPrefix(line, "docker pull") {
+					t.Errorf("something was built after the refusal: %s", line)
+				}
+			}
+			if len(r.images) != 0 || len(r.archives) != 0 {
+				t.Errorf("something was published after the refusal: %v %v", r.images, r.archives)
+			}
+			if !strings.Contains(strings.Join(said, "\n"), "REFUSED") {
+				t.Errorf("the build's log does not say it was refused:\n%s", strings.Join(said, "\n"))
+			}
+		})
+	}
+
+	// A Go artifact builds on the same agent.
+	files := aSharedRepository("one", "1")
+	manifest := `{"module":"ctl","version":"1","build":{"artifacts":[
+	  {"name":"controller","kind":"bundle","language":"go","system":"arch","from":"cmd/ctl","binary":"ctl"}]},
+	  "resources":[{"id":"controller","type":"process","name":"ctl","artifact":"controller","run":["./ctl"]}]}`
+	r := &recorded{contents: map[string]string{ManifestName: manifest}}
+	for k, v := range files {
+		r.contents[k] = v
+	}
+	held := map[string]string{"mesh-tools-go/build": "registry.invalid/mesh-tools-go/build@sha256:" + strings.Repeat("b", 64)}
+	if _, err := Build(context.Background(), onTrunk{recorded: r}.run, r,
+		"https://forge.invalid/ctl.git", "", "", t.TempDir(), held, none, GitCredential{}, nil); err != nil {
+		t.Fatalf("a Go artifact did not build on an agent without the registry: %v", err)
+	}
+	// And an image that does not name .npmrc.
+	r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch\nCOPY . .", "files/x": "y"})
+	if _, err := Build(context.Background(), r.run, r,
+		"https://forge.invalid/meshboard.git", "", "", workspace, nil, none, GitCredential{}, nil); err != nil {
+		t.Fatalf("an image not naming .npmrc did not build on an agent without the registry: %v", err)
+	}
+}
+
+// **The build's log says which registry it resolved from** (novox/hq ADR 0336 §4).
+func TestABuildSaysWhichRegistryItResolvedFrom(t *testing.T) {
+	for _, from := range []string{FromBinding, FromGenesis} {
+		r, workspace := aRepository(t, aPackage, map[string]string{
+			"package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`,
+		})
+		n := Npmrc{Scope: "@novox", Registry: "http://127.0.0.1:3000/api/packages/novox/npm/",
+			Username: "mesh-builder", Password: "p", From: from, Machine: "anchor"}
+		var said []string
+		if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/mesh-sdk.git", "", "",
+			workspace, nil, n, GitCredential{}, func(step, message string) { said = append(said, step+": "+message) }); err != nil {
+			t.Fatalf("%s: %v", from, err)
+		}
+		if !strings.Contains(strings.Join(said, "\n"), "from "+from) {
+			t.Errorf("the log does not say the registry came from %s:\n%s", from, strings.Join(said, "\n"))
+		}
+	}
+}
+
+// The refusal names the module that provides the registry; it is the catalogue's provider.
+func TestTheRegistryABuildIsRefusedForNamesItsProvider(t *testing.T) {
+	raw, err := os.ReadFile(filepath.Join(beside.Catalogue(t), RegistryProvider, "module.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	forge, err := catalogue.ParseManifest(raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	provides := false
+	for _, p := range forge.Provides {
+		provides = provides || p.Name == RegistryRequirement
+	}
+	if !provides {
+		t.Fatalf("%s does not provide %s, which a refused build names it for", RegistryProvider, RegistryRequirement)
+	}
+}

internal/catalogue/genesis_registry_test.go

@@ -0,0 +1,85 @@
+package catalogue
+
+import (
+	"strings"
+	"testing"
+)
+
+// Genesis assigns the build agent before the package registry exists (novox/hq ADR 0336, issue 121),
+// judged over the real catalogue. Genesis's step 13 was refused with "nothing provides
+// \"npm-package-registry\", wanted by build-agent": the forge genesis raises at its packages step is a
+// container, not the gitea module, and provides nothing.
+
+// agentBuilt is the build agent as the mesh holds it once built: its image pinned.
+func agentBuilt(t *testing.T) Manifest {
+	t.Helper()
+	agent := catalogueManifest(t, "build-agent")
+	built, err := agent.Resolve([]Built{{Name: "server", Kind: ArtifactImage,
+		Reference: "mesh-builder@sha256:" + strings.Repeat("a", 64)}})
+	if err != nil {
+		t.Fatalf("the build agent does not resolve against its own build: %v", err)
+	}
+	return built
+}
+
+func TestGenesisAssignsTheBuildAgentWithNothingProvidingTheRegistry(t *testing.T) {
+	agent := agentBuilt(t)
+	if !agent.IsOptional("npm-package-registry") {
+		t.Fatalf("the build agent's npm-package-registry is hard (ADR 0336 §1): %v", agent.Requires)
+	}
+	if agent.IsOptional("artifact-store") {
+		t.Fatal("the build agent's artifact-store is optional; it stays hard")
+	}
+	store := map[string][]Provider{"artifact-store": {
+		{Node: "anchor", At: "anchor.internal", Module: "distribution", Serves: map[string]any{"port": float64(5000)}}}}
+	books := shelf(agent, catalogueManifest(t, "gitea"), catalogueManifest(t, "distribution"))
+
+	// Genesis's node at step 13: the store answers, nothing provides the registry.
+	got, err := Resolve(books, []string{"build-agent"}, reachable(), World{Offered: store})
+	if err != nil {
+		t.Fatalf("assigning the build agent with no registry was refused: %v", err)
+	}
+	if have := strings.Join(names(got), " "); have != "build-agent" {
+		t.Fatalf("assigning the build agent put %s on the machine; nothing is installed for the registry", have)
+	}
+	if len(got.Absent) != 1 || got.Absent[0] != (Absent{Consumer: "build-agent", Requirement: "npm-package-registry"}) {
+		t.Fatalf("the registry is not listed unbound for the build agent: %+v", got.Absent)
+	}
+	out, err := got.Declaration(Rendering{ArtifactStore: "anchor.internal:5000",
+		Needed: map[string]map[string]string{"build-agent": {"broker": "sealed-broker"}}})
+	if err != nil {
+		t.Fatalf("the build agent does not compose with the registry unbound: %v", err)
+	}
+	for _, id := range []string{"build-agent." + BoundID("npm-package-registry"), "build-agent." + SecretID("npm-package-registry"),
+		"build-agent." + NeedID("genesis-package-registry")} {
+		if fileNamed(out, id) != nil {
+			t.Errorf("%s was delivered with the registry unbound and no genesis document given", id)
+		}
+	}
+
+	// The genesis document, once the installer seals it at the packages step (ADR 0336 §3).
+	out, err = got.Declaration(Rendering{ArtifactStore: "anchor.internal:5000",
+		Needed: map[string]map[string]string{"build-agent": {"broker": "sealed-broker", "genesis-package-registry": "sealed-genesis"}}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	doc := fileNamed(out, "build-agent."+NeedID("genesis-package-registry"))
+	if doc == nil || doc["sealed"] != "sealed-genesis" || !strings.HasSuffix(doc["path"].(string), "/genesis-package-registry.json") {
+		t.Fatalf("the genesis document is not delivered where the agent reads it: %v", doc)
+	}
+
+	// The gitea module assigned: the requirement binds, with no act on the build agent (ADR 0336 §2).
+	registry := map[string][]Provider{"artifact-store": store["artifact-store"], "npm-package-registry": {
+		{Node: "anchor", At: "anchor.internal", Module: "gitea",
+			Serves: map[string]any{"port": float64(3000), "scheme": "http", "npm-path": "/api/packages/novox/npm/"}}}}
+	bound, err := Resolve(books, []string{"build-agent"}, reachable(), World{Offered: registry})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n := boundFrom(t, bound, "build-agent", "npm-package-registry"); n.Module != "gitea" {
+		t.Fatalf("bound to %s", n.Module)
+	}
+	if len(bound.Absent) != 0 {
+		t.Fatalf("the registry is still listed unbound once gitea is assigned: %+v", bound.Absent)
+	}
+}