Merge pull request 'mesh-build-go says which packages it cannot test here, and mesh-build-node's tests need the Node toolchain (issue 890, ADR 0552)' (#498) from feat/890-placement into main

modules/mesh-build-go/cmd/mesh-build-go/asks.go

@@ -43,6 +43,7 @@ func (m *Module) askSteps(steps []StepTool) buildstep.AskSteps {
 			return delivery.ToolchainUsed{Version: installed, Recipe: m.Recipe.Version()}, nil
 		},
 		Kept:    m.KeptLookup,
+		Has:     m.Has,
 		Release: func(ws string) error { return m.spaces().Release(ws) },
 		Logf:    m.Say,
 	}
@@ -96,7 +97,7 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep
 		}
 	}
 	return &buildstep.Puller{Source: source, Language: delivery.AskGo, Node: m.Node, Put: m.Steps, PausePut: m.Pauses, Load: load,
-		Capacity: capacity, WatchConfig: watch, Logf: m.Say,
+		Capacity: capacity, WatchConfig: watch, Logf: m.Say, Has: m.Has,
 		Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
 			if err := m.Check(ctx); err != nil {
 				return nil, err

modules/mesh-build-go/cmd/mesh-build-go/digest.go

@@ -11,6 +11,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/fs"
@@ -19,6 +20,8 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
 )
 
 // DigestVersion is the form of the digest's text: changed whenever what the digest hashes changes in kind.
@@ -39,6 +42,33 @@ const besideVariable = "MESH_CHECK_BESIDE"
 // test log) with each verdict.
 const ReadsOutsideMarker = "reads-outside"
 
+// NeedsMarker is the name of a file in a package's directory that names, one per line, what its integration tests need
+// of the node they run on (issue 890; mesh-sdk delivery.AskNeeds): `node-build-node` for tests that read the Node build
+// module's installed toolchain. The digest reports it (DigestResult.Needs); a check ask leaves such a package untested on
+// a node that lacks it, and the checker asks it again of a node that has it. A marker beside the tests, as
+// ReadsOutsideMarker is, and not a manifest field: it is a property of the package's tests, read only where they are
+// run, and the manifest describes the module on the machines. The unit level is never given anything of the node (ADR
+// 0533 §1), so a marker binds the integration level alone. Blank lines and lines starting with # are not read.
+const NeedsMarker = "check-needs"
+
+// readNeeds is what the marker in dir names; nil when there is none.
+func readNeeds(dir string) ([]string, error) {
+	raw, err := os.ReadFile(filepath.Join(dir, NeedsMarker))
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	var needs []string
+	for _, l := range strings.Split(string(raw), "\n") {
+		if l = strings.TrimSpace(l); l != "" && !strings.HasPrefix(l, "#") {
+			needs = append(needs, l)
+		}
+	}
+	return needs, nil
+}
+
 // integrationTags are the build tags of the integration level, its run and its digest alike: integration, and isolation
 // for the bubblewrap proofs (mesh-sdk's and the build modules'), which a build node can run.
 var integrationTags = []string{"integration", "isolation"}
@@ -101,6 +131,8 @@ type DigestResult struct {
 	// ReadsOutside names the packages that carry the marker: each has a digest that no kept verdict can have (a fresh
 	// random one), so the check runs it every time.
 	ReadsOutside []string `json:"reads_outside,omitempty"`
+	// Needs are the packages whose tests at this level need something of the node, as their NeedsMarker names it.
+	Needs []delivery.PackageNeeds `json:"needs,omitempty"`
 	// JudgedBy is the program that ran the digest (ADR 0535 §5): "installed", or "candidate <merge commit>" under the
 	// runner workspace.
 	JudgedBy string `json:"judged_by,omitempty"`
@@ -206,12 +238,13 @@ func (m *Module) digestHere(ctx context.Context, workspace, level string) (Diges
 		if err != nil {
 			return DigestResult{}, err
 		}
-		if err := digestModule(&res, pkgs, tree, header, besideHeader); err != nil {
+		if err := digestModule(&res, pkgs, tree, header, besideHeader, level == LevelIntegration); err != nil {
 			return DigestResult{}, err
 		}
 	}
 	sort.Strings(res.WithoutTests)
 	sort.Strings(res.ReadsOutside)
+	sort.Slice(res.Needs, func(i, j int) bool { return res.Needs[i].Package < res.Needs[j].Package })
 	for _, p := range res.ReadsOutside {
 		if m.Say != nil {
 			m.Say("the %s level of %s always runs: its directory carries %s, so its tests read files outside it that no digest names", level, p, ReadsOutsideMarker)
@@ -256,7 +289,7 @@ func parsePackages(out string) (map[string]*goPackage, error) {
 
 // digestModule adds to res the digest of each root package of one go.mod directory. A root is a package of the main
 // module that is listed, not only a dependency, and not a variant of another.
-func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, header, beside []string) error {
+func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, header, beside []string, needs bool) error {
 	roots := []string{}
 	for path, p := range pkgs {
 		if p.ForTest == "" && !p.DepOnly && !p.Standard && !strings.HasSuffix(path, ".test") && p.Module != nil && p.Module.Main {
@@ -274,6 +307,15 @@ func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, he
 		if testMain == nil {
 			return fmt.Errorf("go list gave no test binary for %s", path)
 		}
+		if needs {
+			n, err := readNeeds(root.Dir)
+			if err != nil {
+				return fmt.Errorf("%s: its %s: %w", path, NeedsMarker, err)
+			}
+			if len(n) > 0 {
+				res.Needs = append(res.Needs, delivery.PackageNeeds{Package: path, Needs: n})
+			}
+		}
 		if _, err := os.Stat(filepath.Join(root.Dir, ReadsOutsideMarker)); err == nil {
 			// The digest of a package that reads outside itself is one nothing can have kept: the key is a fresh random
 			// hex, so the ask finds no kept verdict and runs it, and what the checker writes under it is never read again.

modules/mesh-build-go/cmd/mesh-build-go/digest_test.go

@@ -213,6 +213,16 @@ func TestAPackageWithTheReadsOutsideMarkerIsNeverKept(t *testing.T) {
 	if first.Packages[a] != plain.Packages[a] || first.Packages[a] != second.Packages[a] {
 		t.Errorf("a has no marker and its digest moved: %s, %s, before %s", first.Packages[a], second.Packages[a], plain.Packages[a])
 	}
+
+	// Issue 890/AC3: a package whose directory carries the needs marker is named with its needs at the integration level,
+	// comments and blank lines not read, and at the unit level not at all.
+	edit(t, tree, "a/"+NeedsMarker, "# its integration tests copy the installed Node toolchain\n\nnode-build-node\n")
+	if n := digests(t, m, LevelIntegration).Needs; len(n) != 1 || n[0].Package != a || strings.Join(n[0].Needs, ",") != "node-build-node" {
+		t.Errorf("the integration level's needs = %+v, want %s needing node-build-node", n, a)
+	}
+	if n := digests(t, m, LevelUnit).Needs; len(n) != 0 {
+		t.Errorf("the unit level names needs: %+v", n)
+	}
 }
 
 // Issue 1935: a catalogue module's packages read ../../module.json and the module's files: a change to them changes the

modules/mesh-build-go/cmd/mesh-build-go/level.go

@@ -11,11 +11,13 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 	"time"
 
 	"git.novox.be/novox/mesh-sdk/go/buildstep"
+	"git.novox.be/novox/mesh-sdk/go/delivery"
 )
 
 // levelArg is the first argument that makes this binary a step's shut-off process: the integration level's, and, when
@@ -140,6 +142,49 @@ func (m *Module) nodeToolchains() []string {
 	return []string{dir}
 }
 
+// nodeToolchainMark is the file the Node build module writes in a toolchain directory once it is installed whole
+// (mesh-build-node toolchain.go installedMark).
+const nodeToolchainMark = ".installed"
+
+// Has says whether this node has a need a check ask or a package's tests name (issue 890, ADR 0552; mesh-sdk
+// delivery.AskNeeds), read from the node itself at every ask: node-build-node when the Node build module's toolchains
+// directory, given to the integration level as nodeToolchains gives it, holds a toolchain installed whole (its mark and
+// its package-lock.json readable by this account), and node, npm and bwrap are installed: what mesh-build-node's
+// integration tests (realMachine) look for before they run. Any other need it lacks.
+func (m *Module) Has(need string) bool {
+	if m.HasNeed != nil {
+		return m.HasNeed(need)
+	}
+	switch need {
+	case delivery.NeedNodeBuildNode:
+		dirs := m.nodeToolchains()
+		if len(dirs) == 0 {
+			return false
+		}
+		for _, p := range []string{"node", "npm", "bwrap"} {
+			if _, err := exec.LookPath(p); err != nil {
+				return false
+			}
+		}
+		entries, err := os.ReadDir(dirs[0])
+		if err != nil {
+			return false
+		}
+		for _, e := range entries {
+			dir := filepath.Join(dirs[0], e.Name())
+			if _, err := os.Stat(filepath.Join(dir, nodeToolchainMark)); err != nil {
+				continue
+			}
+			if f, err := os.Open(filepath.Join(dir, "package-lock.json")); err == nil {
+				f.Close()
+				return true
+			}
+		}
+		return false
+	}
+	return false
+}
+
 func lastLine(s string) string {
 	lines := strings.Split(strings.TrimSpace(s), "\n")
 	return strings.TrimSpace(lines[len(lines)-1])

modules/mesh-build-go/cmd/mesh-build-go/module.go

@@ -75,6 +75,8 @@ type Module struct {
 	// NodeToolchains is where the Node build module keeps its installed toolchains, bound read-only into the
 	// integration level (ADR 0533 §1); NodeToolchains when empty. A test points it elsewhere.
 	NodeToolchains string
+	// HasNeed answers Has in place of the node's own reading; a test's.
+	HasNeed func(need string) bool
 	// StartBus starts the throwaway bus of a run in dir and answers its address and what stops it (the module's own
 	// program, StartBus in bus.go, when nil).
 	StartBus func(ctx context.Context, dir string) (url string, stop func(), err error)

modules/mesh-build-go/cmd/mesh-build-go/node_toolchains_test.go

@@ -71,6 +71,47 @@ func TestANodeWithoutNodeToolchainsRunsTheLevelWithNothingMoreBound(t *testing.T
 	if slices.Contains(s.roBinds, missing) {
 		t.Errorf("a missing %s is bound: %v", missing, s.roBinds)
 	}
+
+	// Issue 890/AC4: such a node lacks node-build-node; one with an installed toolchain and node and npm has it; nothing
+	// else is had.
+	bin := t.TempDir()
+	for _, p := range []string{"node", "npm", "bwrap"} {
+		if err := os.WriteFile(filepath.Join(bin, p), []byte("#!/bin/sh\n"), 0o755); err != nil {
+			t.Fatal(err)
+		}
+	}
+	t.Setenv("PATH", bin)
+	m := &Module{NodeToolchains: missing}
+	if m.Has(delivery.NeedNodeBuildNode) {
+		t.Error("a node with no Node toolchain has node-build-node")
+	}
+	installed := t.TempDir()
+	m.NodeToolchains = installed
+	if m.Has(delivery.NeedNodeBuildNode) {
+		t.Error("a node with an empty toolchains directory has node-build-node")
+	}
+	half := filepath.Join(installed, "0.1.0")
+	if err := os.Mkdir(half, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(half, "package-lock.json"), []byte("{}"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if m.Has(delivery.NeedNodeBuildNode) {
+		t.Error("a toolchain without its installed mark counts")
+	}
+	if err := os.WriteFile(filepath.Join(half, nodeToolchainMark), nil, 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if !m.Has(delivery.NeedNodeBuildNode) || m.Has("gpu") {
+		t.Errorf("with a toolchain installed: node-build-node %v, gpu %v; want true, false", m.Has(delivery.NeedNodeBuildNode), m.Has("gpu"))
+	}
+	if err := os.Remove(filepath.Join(bin, "bwrap")); err != nil {
+		t.Fatal(err)
+	}
+	if m.Has(delivery.NeedNodeBuildNode) {
+		t.Error("a node with no bubblewrap installed has node-build-node")
+	}
 }
 
 // 1891 (ADR 0533 §1): the unit level is not given them.

modules/mesh-build-go/go.mod

@@ -3,7 +3,7 @@ module meshbuildgo
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125154-6b8497314297
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026130530-33395f68b309
 	github.com/distribution/distribution/v3 v3.1.2
 	github.com/nats-io/nats-server/v2 v2.11.17
 	github.com/nats-io/nats.go v1.54.0

modules/mesh-build-go/go.sum

@@ -1,5 +1,5 @@
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125154-6b8497314297 h1:k31FtFp/DGXGatRjKV1S0z1gBAHfVgtvBX6xF0g1Vgw=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026130530-33395f68b309 h1:mKJToG8E6ZnZj9700mtfu6YUNNnxzSj0KTfZplQkUCo=
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125154-6b8497314297/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026130530-33395f68b309/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
 github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=

modules/mesh-build-node/cmd/mesh-build-node/check-needs

@@ -0,0 +1,3 @@
+# Its integration tests copy the Node toolchain the Node build module installed on this node and run node and npm
+# (realMachine, build_integration_test.go): they run only where the node holds node-build-node (issue 890).
+node-build-node