diff --git a/modules/mesh-build-node/cmd/mesh-build-node/asks.go b/modules/mesh-build-node/cmd/mesh-build-node/asks.go index ddcf77176..402ffcce3 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/asks.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/asks.go @@ -43,6 +43,7 @@ func (m *Module) askSteps(steps []StepTool) steprun.AskSteps { Release: func(ws string) error { return m.spaces().Release(ws) }, Logf: m.Say, Has: m.Has, + Install: m.installTreeStep, Lint: m.lintStep, Node: m.Node, SayStep: m.sayStep(), diff --git a/modules/mesh-build-node/cmd/mesh-build-node/check.go b/modules/mesh-build-node/cmd/mesh-build-node/check.go index 0ecdd56bb..4f0b5b244 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/check.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/check.go @@ -26,13 +26,28 @@ import ( "git.novox.be/novox/mesh-sdk/go/forgekey" ) -// The bounds of a level (ADR 0445 §4). +// The bounds of a level (ADR 0445 §4). The integration level's setup bound is half of its bound, never a value of its +// own (design 47 H2 §800: the setup bound in code goes), as mesh-sdk's test helpers take theirs from LevelBoundEnv. const ( - UnitBound = 5 * time.Second - IntegrationSetupBound = 30 * time.Second - IntegrationRunBound = 60 * time.Second + UnitBound = 5 * time.Second + IntegrationRunBound = 60 * time.Second ) +// LevelBoundEnv is the word each level's tests get its bound in, as a duration ("1m0s"; design 47 H2 §800): mesh-sdk's +// test helpers take a setup bound of half of it. +const LevelBoundEnv = "MESH_TEST_LEVEL_BOUND" + +// levelBound is a level's bound. +func levelBound(level string) time.Duration { + if level == "unit" { + return UnitBound + } + return IntegrationRunBound +} + +// integrationSetupBound is the integration level's setup bound: half of its bound (§800). +func integrationSetupBound() time.Duration { return IntegrationRunBound / 2 } + // TestResult is what a test step passes with, with the field names of mesh-build-go's (issue 1290). The step ends // passed whenever the level ran, whatever its verdict, and Passed is the verdict; it ends failed only when the tests // could not be run (no toolchain, a command that cannot start, a workspace that was never fetched). @@ -191,7 +206,7 @@ func (m *Module) testLevel(ctx context.Context, args map[string]any) (*TestResul report := filepath.Join(root, ".mesh-node-test-"+level+".tap") _ = os.Remove(report) env := []string{"PATH=" + binDir(root) + ":" + pathOrDefault(), "npm_config_cache=" + filepath.Join(m.Toolchain.Paths.Cache, "npm"), - "NODE_OPTIONS=" + steprun.NodeTestOptions(report)} + "NODE_OPTIONS=" + steprun.NodeTestOptions(report), LevelBoundEnv + "=" + levelBound(level).String()} r := m.runIn(ctx, root, false, tg.dir, env, "npm", scriptArgs(script)...) run += m.now().Sub(start) tap, _ := os.ReadFile(report) @@ -223,11 +238,12 @@ func (m *Module) testLevel(ctx context.Context, args map[string]any) (*TestResul } } else { res.Setup, res.Elapsed = check.Seconds(), run.Seconds() - res.SetupBound, res.Bound = IntegrationSetupBound.Seconds(), IntegrationRunBound.Seconds() - res.OverBound = check > IntegrationSetupBound || run > IntegrationRunBound + setup := integrationSetupBound() + res.SetupBound, res.Bound = setup.Seconds(), IntegrationRunBound.Seconds() + res.OverBound = check > setup || run > IntegrationRunBound if res.OverBound { why = append(why, fmt.Sprintf("the integration level took %.2f s of setup and %.2f s of running, over its bounds of %s and %s", - res.Setup, res.Elapsed, IntegrationSetupBound, IntegrationRunBound)) + res.Setup, res.Elapsed, setup, IntegrationRunBound)) } } res.Passed = len(failed) == 0 diff --git a/modules/mesh-build-node/cmd/mesh-build-node/check_test.go b/modules/mesh-build-node/cmd/mesh-build-node/check_test.go index fcc3a7ba4..ca66028cf 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/check_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/check_test.go @@ -123,6 +123,10 @@ func TestTheIntegrationLevelRunsItsScriptAndReportsSetupAndRunningApart(t *testi if strings.Join(last.args, " ") != "run test:integration" { t.Errorf("ran %s, want npm run test:integration", joined(last)) } + // Design 47 H2 §800: the level's tests get its bound in the environment, the setup bound being half of it. + if !slices.Contains(last.env, "MESH_TEST_LEVEL_BOUND=1m0s") { + t.Errorf("the integration script ran with %v, not the level's bound MESH_TEST_LEVEL_BOUND=1m0s", last.env) + } if tsc := f.calls[0]; strings.Join(tsc.args[1:], " ") != "--noEmit -p ." { t.Errorf("a package with a tsconfig was checked with %s", joined(tsc)) } @@ -133,7 +137,7 @@ func TestTheIntegrationLevelRunsItsScriptAndReportsSetupAndRunningApart(t *testi } } -// The commands of a step get an environment built from scratch: no word of this process's. +// The commands of a step get an environment built from scratch: no word of this process's, the level's bound alone (§800). func TestTheStepsCommandsGetAnEnvironmentOfTheirOwn(t *testing.T) { t.Setenv("MESH_SECRET_TOKEN", "hunter2") m, f, work := newMachine(t) @@ -143,7 +147,7 @@ func TestTheStepsCommandsGetAnEnvironmentOfTheirOwn(t *testing.T) { } for _, c := range f.calls { for _, e := range c.env { - if strings.Contains(e, "hunter2") || strings.HasPrefix(e, "MESH_") { + if strings.Contains(e, "hunter2") || (strings.HasPrefix(e, "MESH_") && !strings.HasPrefix(e, LevelBoundEnv+"=")) { t.Errorf("%s was given %s", joined(c), e) } } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/install.go b/modules/mesh-build-node/cmd/mesh-build-node/install.go new file mode 100644 index 000000000..25921b4ec --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/install.go @@ -0,0 +1,121 @@ +package main + +// **An install of a module's npm tree** (design 47 H2 §800; H-A-399 (2), H-A-403): the runner asks the seat node-build-node +// a StepAsk of the kind install, on the seat's install queue, for a module whose Go artifact's check-needs names this +// seat. The install takes the module's package lock at the commit, installs it from the npm registry seat with an +// integrity-checked install (npm ci) that runs no install script and makes no link (--ignore-scripts, --no-bin-links: the +// SDK's Pack and the unpack refuse a symlink, and the mesh creates none), packs the tree, publishes the archive to the +// artifact store as the module's artifact :node-modules under the ask's version, and answers its reference, which +// the step's end carries and the integration ask names for the Go build module to unpack. + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + artifactref "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/containerimageregistryclient/reference" + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" +) + +// installKind is the artifact kind the install archive is published as (H-A-403: kind archive). +const installKind = "archive" + +// installTree is where the installed tree lies in the archive, from the workspace's tree: the module's directory's +// node_modules, which the Go build module unpacks at the tree's root (layInstall). +func installTree(module string) string { return filepath.Join("modules", module, "node_modules") } + +// installTreeStep is AskSteps.Install: the install body of an install ask (steprun's installStep gives its arguments). +func (m *Module) installTreeStep(ctx context.Context, args map[string]any) (any, error) { + module, _ := args["module"].(string) + lock, _ := args["lock"].(string) + artifact, _ := args["artifact"].(string) + version, _ := args["version"].(string) + name, ok := strings.CutPrefix(artifact, module+":") + switch { + case module == "" || !ok || artifact != sdkstep.InstallArtifact(module): + return nil, fmt.Errorf("the install of %q names the artifact %q, not %s", module, artifact, sdkstep.InstallArtifact(module)) + case version == "": + return nil, fmt.Errorf("the install of %s names no version to publish its archive under", module) + } + pkg, err := lockRoot(lock) + if err != nil { + return nil, fmt.Errorf("the install of %s: %w", module, err) + } + if err := os.MkdirAll(m.Work, 0o755); err != nil { + return nil, err + } + root, err := os.MkdirTemp(m.Work, "install-") + if err != nil { + return nil, err + } + defer os.RemoveAll(root) + dir := filepath.Join(root, "npm") + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, err + } + if err := os.WriteFile(filepath.Join(dir, "package.json"), pkg, 0o600); err != nil { + return nil, err + } + if err := os.WriteFile(filepath.Join(dir, "package-lock.json"), []byte(lock), 0o600); err != nil { + return nil, err + } + // The registry is given through a config file of its own, outside the workspace, removed after (npmrcIn). + npmrc, err := m.npmrcIn(root) + if err != nil { + return nil, fmt.Errorf("the install of %s: %w", module, err) + } + defer os.Remove(npmrc) + env := append(m.baseEnv(), "npm_config_cache="+m.npmCache()) + r := m.Run(ctx, env, "npm", "ci", "--ignore-scripts", "--no-bin-links", "--no-audit", "--no-fund", "--prefix", dir, "--userconfig", npmrc) + if why := ranFailed(r); why != "" { + return nil, steprun.Transport(fmt.Errorf("installing the npm tree of %s: %s", module, why)) + } + // Only the tree is archived, under the module's directory: never the package.json written here from the lock, which + // would lie over the module's own when unpacked. + packed := filepath.Join(root, "packed") + into := filepath.Join(packed, installTree(module)) + if err := os.MkdirAll(filepath.Dir(into), 0o755); err != nil { + return nil, err + } + if err := os.Rename(filepath.Join(dir, "node_modules"), into); err != nil { + return nil, fmt.Errorf("the install of %s made no node_modules: %w", module, err) + } + archive, digest, err := steprun.Pack(packed) + if err != nil { + return nil, fmt.Errorf("packing the npm tree of %s: %w", module, err) + } + if err := m.publishOn(ctx, steprun.ArtifactSeat, module+"/"+name, installKind, version, digest, archive); err != nil { + return nil, fmt.Errorf("publishing the npm tree of %s: %w", module, err) + } + reference := artifactref.SeatReference(steprun.ArtifactSeat, module, name, digest) + if m.Say != nil { + m.Say("installed the npm tree of %s at %s (%d bytes) as %s", module, version, len(archive), reference) + } + return map[string]any{"archive": reference}, nil +} + +// lockRoot is the package.json npm ci needs beside a lock, written from the lock's root package (packages[""]): its +// name, version and dependency maps, never a script. A lock with no root package (lockfileVersion 1) is refused. +func lockRoot(lock string) ([]byte, error) { + var l struct { + Packages map[string]map[string]any `json:"packages"` + } + if err := json.Unmarshal([]byte(lock), &l); err != nil { + return nil, fmt.Errorf("the package lock is not JSON: %w", err) + } + root, ok := l.Packages[""] + if !ok { + return nil, fmt.Errorf("the package lock names no root package (packages[\"\"]): a lock of lockfileVersion 2 or later is needed") + } + pkg := map[string]any{} + for _, k := range []string{"name", "version", "dependencies", "devDependencies", "optionalDependencies", "peerDependencies"} { + if v, ok := root[k]; ok { + pkg[k] = v + } + } + return json.Marshal(pkg) +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/push_test.go b/modules/mesh-build-node/cmd/mesh-build-node/push_test.go index fa5b44c01..6a2b2d95a 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/push_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/push_test.go @@ -1,6 +1,9 @@ package main import ( + "archive/tar" + "bytes" + "compress/gzip" "context" "crypto/sha256" "encoding/base64" @@ -9,6 +12,8 @@ import ( "errors" "os" "path/filepath" + "reflect" + "slices" "strings" "sync" "testing" @@ -18,6 +23,7 @@ import ( builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" + "git.novox.be/novox/mesh-sdk/go/forgekey" ) // pushed are the blobs the fake holder took, by digest, for the tests that read an archive back. @@ -51,7 +57,7 @@ func withStore(t *testing.T, m *Module) *fakeStore { case "publish": var in registry.PublishInput _ = json.Unmarshal(raw, &in) - if (seat == steprun.ArtifactSeat) != (in.Kind == "bundle") || in.Transfer == "" || strings.ContainsAny(in.Transfer, "/. *>") { + if (seat == steprun.ArtifactSeat) != (in.Kind == "bundle" || in.Kind == "archive") || in.Transfer == "" || strings.ContainsAny(in.Transfer, "/. *>") { t.Errorf("publish on %s of kind %q, transfer %q", seat, in.Kind, in.Transfer) } s.open[in.Transfer] = in @@ -115,6 +121,72 @@ func TestADoneBundleAnswersTheReferenceOfTheArchiveItPushed(t *testing.T) { if len(store.published) != 1 || store.published[0].Version != "1.4.3" { t.Errorf("the bundle was published as %+v, want the ask's version 1.4.3", store.published) } + // Design 47 H2 §800, H-A-403, H-A-427: an install runs no script and makes no link; its tree alone, under the module's + // directory, is published as the archive :node-modules under the ask's version, and the end names its reference. + t.Run("an install runs no script", func(t *testing.T) { + m := &Module{Work: t.TempDir(), Toolchain: &Toolchain{Paths: Paths{Home: t.TempDir(), Cache: t.TempDir()}, + Registry: func() (Registry, error) { return Registry{}, nil }}} + store := withStore(t, m) + var ran []string + var written map[string]any + m.Run = func(_ context.Context, _ []string, name string, args ...string) forgekey.Ran { + ran = append([]string{name}, args...) + dir := args[slices.Index(args, "--prefix")+1] + raw, _ := os.ReadFile(filepath.Join(dir, "package.json")) + _ = json.Unmarshal(raw, &written) + os.MkdirAll(filepath.Join(dir, "node_modules", "left-pad"), 0o755) + os.WriteFile(filepath.Join(dir, "node_modules", "left-pad", "index.js"), []byte("module.exports = 1\n"), 0o644) + return forgekey.Ran{} + } + lock := `{"lockfileVersion": 3, "packages": {"": {"name": "@novox/module-gitea", "scripts": {"install": "touch ran"}, + "dependencies": {"left-pad": "^1.3.0"}}, "node_modules/left-pad": {"version": "1.3.0", "hasInstallScript": true}}}` + install := func(lock string) (any, error) { + return m.installTreeStep(context.Background(), map[string]any{"module": "gitea", "commit": strings.Repeat("a", 40), "fingerprint": "fp", + "pipeline": "p1", "lock": lock, "artifact": sdkstep.InstallArtifact("gitea"), "version": "0.4.2"}) + } + out, err := install(lock) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(ran, "--ignore-scripts") || !slices.Contains(ran, "--no-bin-links") || ran[0] != "npm" || ran[1] != "ci" { + t.Errorf("the install ran %v, want npm ci --ignore-scripts --no-bin-links", ran) + } + if _, scripts := written["scripts"]; scripts || written["name"] != "@novox/module-gitea" { + t.Errorf("the package.json beside the lock is %v, want the lock's root with no script", written) + } + if len(store.published) != 1 || store.published[0].Name != "gitea/node-modules" || store.published[0].Kind != "archive" || store.published[0].Version != "0.4.2" { + t.Fatalf("published %+v, want gitea/node-modules of kind archive at 0.4.2", store.published) + } + digest := store.published[0].Digest + if want := map[string]any{"archive": "oci-artifact-registry/gitea/node-modules@" + digest}; !reflect.DeepEqual(out, want) { + t.Errorf("the install answered %v, want %v", out, want) + } + if names := archiveNames(t, store.blobs["gitea/node-modules@"+digest]); !slices.Equal(names, []string{"modules/gitea/node_modules/left-pad/index.js"}) { + t.Errorf("the archive holds %v, want the tree alone under the module's directory", names) + } + if _, err := install(`{"lockfileVersion": 1, "dependencies": {}}`); err == nil { + t.Error("a lock with no root package was installed") + } + }) +} + +// archiveNames are the file names a packed archive holds (steprun.Pack: tar in gzip). +func archiveNames(t *testing.T, archive []byte) []string { + t.Helper() + z, err := gzip.NewReader(bytes.NewReader(archive)) + if err != nil { + t.Fatal(err) + } + var names []string + r := tar.NewReader(z) + for { + h, err := r.Next() + if err != nil { + break + } + names = append(names, h.Name) + } + return names } // 1587-S7 and S8 (ADR 0498 §2, the lead's decision L9): a publish the holder refuses fails the step `not stored: