Merge pull request 'Make dev-tools, lab and the build modules need the go and nodejs modules instead of declaring the toolchain packages, step B of issue 1573 (ADR 0477 §4)' (#409) from fix/1573b-toolchain-users into main

checks/toolchains/go.mod

@@ -0,0 +1,3 @@
+module git.novox.be/novox/mesh-catalog/checks/toolchains
+
+go 1.22

checks/toolchains/main.go

@@ -0,0 +1,104 @@
+// Command toolchains refuses any manifest but the owner's declaring a language's toolchain package (novox/hq
+// ADR 0477 clause 4, issue 1573): the package go only in modules/go, and nodejs and npm only in modules/nodejs.
+// Everything that compiles or runs that language requires the owner's provision instead (go-toolchain,
+// node-toolchain), so the package is declared once, and a node without the owner refuses the module that
+// needs it at resolution rather than failing at its first build. The owners must declare their packages.
+//
+//	go run . <catalogue root>
+//
+// Exits 1 on a finding.
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+)
+
+// Owners names, for each toolchain package, the one module that declares it.
+var Owners = map[string]string{
+	"go":     "go",
+	"nodejs": "nodejs",
+	"npm":    "nodejs",
+}
+
+// Finding is one refusal: Module declares Package, which Owner alone may, or (Missing) Owner does not declare
+// the Package it owns.
+type Finding struct {
+	Module, Package, Owner string
+	Missing                bool
+}
+
+// Check reads every manifest under root/modules and answers what it refuses.
+func Check(root string) ([]Finding, error) {
+	manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
+	if err != nil {
+		return nil, err
+	}
+	sort.Strings(manifests)
+	var out []Finding
+	declared := map[string]map[string]bool{} // module -> packages
+	for _, path := range manifests {
+		raw, err := os.ReadFile(path)
+		if err != nil {
+			return nil, err
+		}
+		var m struct {
+			Module    string `json:"module"`
+			Resources []struct {
+				Type    string `json:"type"`
+				Package string `json:"package"`
+			} `json:"resources"`
+		}
+		if err := json.Unmarshal(raw, &m); err != nil {
+			return nil, fmt.Errorf("%s: %w", path, err)
+		}
+		declared[m.Module] = map[string]bool{}
+		for _, r := range m.Resources {
+			if r.Type != "package" {
+				continue
+			}
+			declared[m.Module][r.Package] = true
+			if owner, ok := Owners[r.Package]; ok && owner != m.Module {
+				out = append(out, Finding{Module: m.Module, Package: r.Package, Owner: owner})
+			}
+		}
+	}
+	var pkgs []string
+	for p := range Owners {
+		pkgs = append(pkgs, p)
+	}
+	sort.Strings(pkgs)
+	for _, p := range pkgs {
+		if !declared[Owners[p]][p] {
+			out = append(out, Finding{Module: Owners[p], Package: p, Owner: Owners[p], Missing: true})
+		}
+	}
+	return out, nil
+}
+
+func main() {
+	root := "."
+	if len(os.Args) > 1 {
+		root = os.Args[1]
+	}
+	found, err := Check(root)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+	for _, f := range found {
+		if f.Missing {
+			fmt.Printf("%s does not declare the package %s it owns (novox/hq ADR 0477 clause 4)\n", f.Owner, f.Package)
+			continue
+		}
+		fmt.Printf("%s declares the package %s: only %s may; require its provision instead (novox/hq ADR 0477 clause 4)\n",
+			f.Module, f.Package, f.Owner)
+	}
+	if len(found) > 0 {
+		os.Exit(1)
+	}
+	fmt.Println("toolchains: go, nodejs and npm are declared only by their owners")
+}

checks/toolchains/main_test.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+)
+
+func catalogue(t *testing.T, manifests map[string]string) string {
+	t.Helper()
+	root := t.TempDir()
+	for name, body := range manifests {
+		dir := filepath.Join(root, "modules", name)
+		if err := os.MkdirAll(dir, 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(body), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	return root
+}
+
+func TestOnlyTheOwnerDeclaresAToolchainPackage(t *testing.T) {
+	root := catalogue(t, map[string]string{
+		"go":      `{"module":"go","resources":[{"id":"go","type":"package","package":"go"}]}`,
+		"nodejs":  `{"module":"nodejs","resources":[{"id":"nodejs","type":"package","package":"nodejs"},{"id":"npm","type":"package","package":"npm"}]}`,
+		"lab":     `{"module":"lab","resources":[{"id":"git","type":"package","package":"git"},{"id":"go","type":"package","package":"go"}]}`,
+		"builder": `{"module":"builder","resources":[{"id":"n","type":"package","package":"nodejs"},{"id":"dir","type":"directory","path":"/go"}]}`,
+	})
+	got, err := Check(root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []Finding{{Module: "builder", Package: "nodejs", Owner: "nodejs"}, {Module: "lab", Package: "go", Owner: "go"}}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("found %v, want %v", got, want)
+	}
+}
+
+func TestTheOwnersMustExistAndDeclareTheirPackages(t *testing.T) {
+	root := catalogue(t, map[string]string{
+		"go": `{"module":"go","resources":[]}`,
+	})
+	got, err := Check(root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 3 {
+		t.Errorf("an owner missing its packages gave %v, want three findings (go, nodejs, npm)", got)
+	}
+}
+
+// The catalogue as it stands: this is the check itself, run here too.
+func TestTheCatalogueGivesEachToolchainPackageToItsOwnerAlone(t *testing.T) {
+	got, err := Check("../..")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 0 {
+		t.Errorf("the catalogue declares a toolchain package outside its owner: %v", got)
+	}
+}

merge-check.sh

@@ -53,6 +53,8 @@
 set -eu
 
 (cd checks/words && go test -count=1 . && go run . ../..)
+# The toolchains (novox/hq ADR 0477 clause 4): the package go only in modules/go, nodejs and npm only in modules/nodejs.
+(cd checks/toolchains && go test -count=1 . && go run . ../..)
 # The agent's home (novox/hq ADR 0266): no archive, and no file written into, under it.
 (cd checks/agenthome && go test -count=1 . && go run . ../..)
 # The mail-store seat (novox/hq ADR 0378 §3): only mesh-mail may claim it.

modules/dev-tools/README.md

@@ -5,16 +5,16 @@ machine: an agent working on a change builds it and runs its tests there, with t
 pull request. The lab is the QA step after that: it tests the work in a sandbox, through its seat at the mesh
 scope, and is not where development happens.
 
-Assign it to every node.
+Assign it to every node, with the `go` and `nodejs` modules: it requires their provisions `go-toolchain` and `node-toolchain` (novox/hq ADR 0477 clause 4).
 
 ## Owns
 
 | what | package | class (ADR 0182) |
 |---|---|---|
-| the Go toolchain, for `go build` and `go test` in every Go module | `go` | package |
+| the Go toolchain, for `go build` and `go test` in every Go module | the `go` module's package `go` | package |
 | a C compiler, for `go test -race` and cgo | `gcc` | package |
 | make | `make` | package |
-| Node.js and npm, for the TypeScript modules' `npm ci` and `npm test` | `nodejs`, `npm` | package |
+| Node.js and npm, for the TypeScript modules' `npm ci` and `npm test` | the `nodejs` module's packages `nodejs`, `npm` | package |
 | Python, for the scripts some checks run | `python` | package |
 | git and jq | `git`, `jq` | package |
 

modules/dev-tools/module.json

@@ -4,12 +4,11 @@
   "capabilities": [
     "package-manager"
   ],
+  "requires": [
+    "go-toolchain",
+    "node-toolchain"
+  ],
   "resources": [
-    {
-      "id": "go",
-      "type": "package",
-      "package": "go"
-    },
     {
       "id": "c-compiler",
       "type": "package",
@@ -20,16 +19,6 @@
       "type": "package",
       "package": "make"
     },
-    {
-      "id": "nodejs",
-      "type": "package",
-      "package": "nodejs"
-    },
-    {
-      "id": "npm",
-      "type": "package",
-      "package": "npm"
-    },
     {
       "id": "python",
       "type": "package",

modules/go/README.md

@@ -1,14 +1,11 @@
 # go
 
-The Go toolchain as a module (novox/hq ADR 0477 §4, issue 1573).
+The Go toolchain as a module (novox/hq ADR 0477 clause 4, issue 1573).
 
-- **Declares the package `go`.** Today dev-tools, lab and mesh-build-go declare it too, which the resolver allows
-  (ADR 0303); once they require `go-toolchain` instead (issue 1573's second step), this is the only module that
-  declares it, and `checks/toolchains` refuses another.
-- **Provides `go-toolchain` with the machine's reach**: a module that requires it composes only on a machine where
-  this module is assigned. It is never pulled in for whatever asked, so `go` is assigned to a node on purpose.
-  Nothing requires it yet; dev-tools, lab and mesh-build-go will, in the second step.
+- **The only module that declares the package `go`.** `checks/toolchains` refuses any other that does.
+- **Provides `go-toolchain` with the machine's reach**: dev-tools, lab and mesh-build-go require it, so each
+  composes only on a machine where this module is assigned. It is never pulled in for whatever asked, so `go` is
+  assigned to a node on purpose, and a node without it refuses those modules at resolution.
 - Holds no long-running resource, so it adds nothing to `health-undeclared`.
 
-The node-engine never uninstalls a package, so when the users stop declaring `go`, nothing leaves a node that has
-this module.
+The node-engine never uninstalls a package, so moving the declaration here removed nothing from a node.

modules/lab/README.md

@@ -1,6 +1,6 @@
 # lab
 
-The `lab` module of the mesh catalogue: installs the package `git`; installs the package `make`; installs the package `python`; installs the package `file`; installs the package `iproute2`; installs the package `npm`; installs the package `go`; installs the package `incus`.
+The `lab` module of the mesh catalogue: installs the package `git`; installs the package `make`; installs the package `python`; installs the package `file`; installs the package `iproute2`; installs the package `incus`; requires `go-toolchain` and `node-toolchain`, so the `go` module and the `nodejs` module are assigned to a machine running it (novox/hq ADR 0477 clause 4).
 
 ## Files it does not know
 

modules/lab/module.json

@@ -6,7 +6,9 @@
     "virtualisation"
   ],
   "requires": [
-    "npm-package-registry"
+    "npm-package-registry",
+    "go-toolchain",
+    "node-toolchain"
   ],
   "contributes": {
     "npm-package-registry": {
@@ -84,16 +86,6 @@
       "type": "package",
       "package": "iproute2"
     },
-    {
-      "id": "npm",
-      "type": "package",
-      "package": "npm"
-    },
-    {
-      "id": "go",
-      "type": "package",
-      "package": "go"
-    },
     {
       "id": "incus",
       "type": "package",

modules/mesh-build-go/README.md

@@ -2,8 +2,8 @@
 
 Design 47, release R4c-3 (novox/hq ADR 0405, ADR 0466 §1, ADR 0469, ADR 0470). The module that builds and checks Go
 code on a build machine, as an account of its own (`runs-as`, no login shell, home at mode 0700), with the
-toolchain installed as the packages it declares (`go`, `git`, `openssh`, `postgresql`) and the capability
-`user-namespaces`. It claims no seat: it is assigned to the build machines (ADR 0469 §6) and runs dark until the
+toolchain installed as the packages it declares (`git`, `openssh`, `postgresql`; `go` comes from the `go` module, which this module requires as `go-toolchain`) and the capability
+`user-namespaces`. It claims no seat: it is assigned to the build machines (ADR 0469 clause 6) and runs dark until the
 builder uses it.
 
 ## The recipe

modules/mesh-build-go/cmd/mesh-build-go/forge_test.go

@@ -143,8 +143,8 @@ func TestForgeFilesRenderFromTheBoundValuesInTheModulesOwnHome(t *testing.T) {
 // template in the files/ archive (novox/hq ADR 0349 §2).
 func TestManifestRequiresForgeAccountKeepsKeysAndTakesFilesFromTemplates(t *testing.T) {
 	m := readForgeManifest(t)
-	if len(m.Requires) != 1 || m.Requires[0] != "forge-account" {
-		t.Errorf("requires = %v, want [forge-account]", m.Requires)
+	if len(m.Requires) != 2 || m.Requires[0] != "forge-account" || m.Requires[1] != "go-toolchain" {
+		t.Errorf("requires = %v, want [forge-account go-toolchain]", m.Requires)
 	}
 	keeps := false
 	for _, s := range m.State {

modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go

@@ -64,7 +64,7 @@ func TestManifestIsValidAndDeclaresTheAccountTheCapabilityAndThePackages(t *test
 		}
 	}
 	slices.Sort(packages)
-	if want := []string{"git", "go", "openssh", "postgresql"}; !slices.Equal(packages, want) {
+	if want := []string{"git", "openssh", "postgresql"}; !slices.Equal(packages, want) {
 		t.Errorf("packages = %v, want %v", packages, want)
 	}
 }

modules/mesh-build-go/module.json

@@ -27,7 +27,8 @@
     }
   ],
   "requires": [
-    "forge-account"
+    "forge-account",
+    "go-toolchain"
   ],
   "own-secrets": {
     "broker": "${dir:state}/broker"
@@ -105,11 +106,6 @@
       "mode": "0644",
       "template": "files/gitconfig"
     },
-    {
-      "id": "go",
-      "type": "package",
-      "package": "go"
-    },
     {
       "id": "git",
       "type": "package",

modules/mesh-build-node/README.md

@@ -2,8 +2,8 @@
 
 Design 47, release R4c-3 (novox/hq ADR 0405, ADR 0466 §1, ADR 0469 §1 and §4). The module that builds and checks
 TypeScript code on a build machine, as an account of its own (`runs-as`, no login shell, home at mode 0700), with the
-packages it declares (`nodejs`, `npm`, `git`, `openssh`) and the capability `user-namespaces`. The bundle is Go
-(`cmd/mesh-build-node`). It claims no seat: it is assigned to the build machines (ADR 0469 §6) and runs dark until the
+packages it declares (`git`, `openssh`; `nodejs` and `npm` come from the `nodejs` module, which this module requires as `node-toolchain`) and the capability `user-namespaces`. The bundle is Go
+(`cmd/mesh-build-node`). It claims no seat: it is assigned to the build machines (ADR 0469 clause 6) and runs dark until the
 builder uses it. Its forge key and ssh files are the same as mesh-build-go's (issue 1484), filled on the node by the
 provisioner's client (issue 1537).
 

modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go

@@ -66,7 +66,7 @@ func TestManifestIsValidAndDeclaresTheAccountTheCapabilityAndThePackages(t *test
 		}
 	}
 	slices.Sort(packages)
-	if want := []string{"git", "nodejs", "npm", "openssh"}; !slices.Equal(packages, want) {
+	if want := []string{"git", "openssh"}; !slices.Equal(packages, want) {
 		t.Errorf("packages = %v, want %v", packages, want)
 	}
 }
@@ -134,6 +134,9 @@ func TestTheModuleCarriesItsToolchainLockAndTheManifestPlacesIt(t *testing.T) {
 			t.Errorf("the module carries no toolchain/%s: %v", name, err)
 		}
 	}
+	if !slices.Contains(m.Requires, "node-toolchain") {
+		t.Errorf("requires = %v lack node-toolchain, which provides the nodejs and npm packages (issue 1573)", m.Requires)
+	}
 	if !slices.Contains(m.Requires, "npm-package-registry") {
 		t.Errorf("requires = %v lack npm-package-registry, which the install resolves @novox from", m.Requires)
 	}

modules/mesh-build-node/module.json

@@ -29,7 +29,8 @@
   ],
   "requires": [
     "forge-account",
-    "npm-package-registry"
+    "npm-package-registry",
+    "node-toolchain"
   ],
   "contributes": {
     "npm-package-registry": {
@@ -126,16 +127,6 @@
       "artifact": "toolchain",
       "unknown": "program"
     },
-    {
-      "id": "nodejs",
-      "type": "package",
-      "package": "nodejs"
-    },
-    {
-      "id": "npm",
-      "type": "package",
-      "package": "npm"
-    },
     {
       "id": "git",
       "type": "package",

modules/nodejs/README.md

@@ -1,14 +1,12 @@
 # nodejs
 
-Node.js and npm as a module (novox/hq ADR 0477 §4, issue 1573).
+Node.js and npm as a module (novox/hq ADR 0477 clause 4, issue 1573).
 
-- **Declares the packages `nodejs` and `npm`.** Today dev-tools, lab and mesh-build-node declare some of them too,
-  which the resolver allows (ADR 0303); once they require `node-toolchain` instead (issue 1573's second step), this
-  is the only module that declares them, and `checks/toolchains` refuses another.
-- **Provides `node-toolchain` with the machine's reach**: a module that requires it composes only on a machine where
-  this module is assigned. It is never pulled in for whatever asked, so `nodejs` is assigned to a node on purpose.
-  Nothing requires it yet; dev-tools, lab and mesh-build-node will, in the second step.
+- **The only module that declares the packages `nodejs` and `npm`.** `checks/toolchains` refuses any other that
+  does.
+- **Provides `node-toolchain` with the machine's reach**: dev-tools, lab and mesh-build-node require it, so each
+  composes only on a machine where this module is assigned. It is never pulled in for whatever asked, so `nodejs`
+  is assigned to a node on purpose, and a node without it refuses those modules at resolution.
 - Holds no long-running resource, so it adds nothing to `health-undeclared`.
 
-The node-engine never uninstalls a package, so when the users stop declaring `nodejs` and `npm`, nothing leaves a
-node that has this module.
+The node-engine never uninstalls a package, so moving the declarations here removed nothing from a node.