diff --git a/modules/mesh-build-go/cmd/mesh-build-go/asks.go b/modules/mesh-build-go/cmd/mesh-build-go/asks.go index f7ba3cea..a4053606 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/asks.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/asks.go @@ -43,6 +43,7 @@ func (m *Module) askSteps(steps []StepTool) buildstep.AskSteps { return delivery.ToolchainUsed{Version: installed, Recipe: m.Recipe.Version()}, nil }, Kept: m.KeptLookup, + Has: m.Has, Release: func(ws string) error { return m.spaces().Release(ws) }, Logf: m.Say, } @@ -96,7 +97,7 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep } } return &buildstep.Puller{Source: source, Language: delivery.AskGo, Node: m.Node, Put: m.Steps, PausePut: m.Pauses, Load: load, - Capacity: capacity, WatchConfig: watch, Logf: m.Say, + Capacity: capacity, WatchConfig: watch, Logf: m.Say, Has: m.Has, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { if err := m.Check(ctx); err != nil { return nil, err diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest.go b/modules/mesh-build-go/cmd/mesh-build-go/digest.go index bce625ba..746a6f85 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/digest.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest.go @@ -11,6 +11,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "io/fs" @@ -19,6 +20,8 @@ import ( "sort" "strconv" "strings" + + "git.novox.be/novox/mesh-sdk/go/delivery" ) // DigestVersion is the form of the digest's text: changed whenever what the digest hashes changes in kind. @@ -39,6 +42,33 @@ const besideVariable = "MESH_CHECK_BESIDE" // test log) with each verdict. const ReadsOutsideMarker = "reads-outside" +// NeedsMarker is the name of a file in a package's directory that names, one per line, what its integration tests need +// of the node they run on (issue 890; mesh-sdk delivery.AskNeeds): `node-build-node` for tests that read the Node build +// module's installed toolchain. The digest reports it (DigestResult.Needs); a check ask leaves such a package untested on +// a node that lacks it, and the checker asks it again of a node that has it. A marker beside the tests, as +// ReadsOutsideMarker is, and not a manifest field: it is a property of the package's tests, read only where they are +// run, and the manifest describes the module on the machines. The unit level is never given anything of the node (ADR +// 0533 §1), so a marker binds the integration level alone. Blank lines and lines starting with # are not read. +const NeedsMarker = "check-needs" + +// readNeeds is what the marker in dir names; nil when there is none. +func readNeeds(dir string) ([]string, error) { + raw, err := os.ReadFile(filepath.Join(dir, NeedsMarker)) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + var needs []string + for _, l := range strings.Split(string(raw), "\n") { + if l = strings.TrimSpace(l); l != "" && !strings.HasPrefix(l, "#") { + needs = append(needs, l) + } + } + return needs, nil +} + // integrationTags are the build tags of the integration level, its run and its digest alike: integration, and isolation // for the bubblewrap proofs (mesh-sdk's and the build modules'), which a build node can run. var integrationTags = []string{"integration", "isolation"} @@ -101,6 +131,8 @@ type DigestResult struct { // ReadsOutside names the packages that carry the marker: each has a digest that no kept verdict can have (a fresh // random one), so the check runs it every time. ReadsOutside []string `json:"reads_outside,omitempty"` + // Needs are the packages whose tests at this level need something of the node, as their NeedsMarker names it. + Needs []delivery.PackageNeeds `json:"needs,omitempty"` // JudgedBy is the program that ran the digest (ADR 0535 §5): "installed", or "candidate " under the // runner workspace. JudgedBy string `json:"judged_by,omitempty"` @@ -206,12 +238,13 @@ func (m *Module) digestHere(ctx context.Context, workspace, level string) (Diges if err != nil { return DigestResult{}, err } - if err := digestModule(&res, pkgs, tree, header, besideHeader); err != nil { + if err := digestModule(&res, pkgs, tree, header, besideHeader, level == LevelIntegration); err != nil { return DigestResult{}, err } } sort.Strings(res.WithoutTests) sort.Strings(res.ReadsOutside) + sort.Slice(res.Needs, func(i, j int) bool { return res.Needs[i].Package < res.Needs[j].Package }) for _, p := range res.ReadsOutside { if m.Say != nil { m.Say("the %s level of %s always runs: its directory carries %s, so its tests read files outside it that no digest names", level, p, ReadsOutsideMarker) @@ -256,7 +289,7 @@ func parsePackages(out string) (map[string]*goPackage, error) { // digestModule adds to res the digest of each root package of one go.mod directory. A root is a package of the main // module that is listed, not only a dependency, and not a variant of another. -func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, header, beside []string) error { +func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, header, beside []string, needs bool) error { roots := []string{} for path, p := range pkgs { if p.ForTest == "" && !p.DepOnly && !p.Standard && !strings.HasSuffix(path, ".test") && p.Module != nil && p.Module.Main { @@ -274,6 +307,15 @@ func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, he if testMain == nil { return fmt.Errorf("go list gave no test binary for %s", path) } + if needs { + n, err := readNeeds(root.Dir) + if err != nil { + return fmt.Errorf("%s: its %s: %w", path, NeedsMarker, err) + } + if len(n) > 0 { + res.Needs = append(res.Needs, delivery.PackageNeeds{Package: path, Needs: n}) + } + } if _, err := os.Stat(filepath.Join(root.Dir, ReadsOutsideMarker)); err == nil { // The digest of a package that reads outside itself is one nothing can have kept: the key is a fresh random // hex, so the ask finds no kept verdict and runs it, and what the checker writes under it is never read again. diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go index 9b6fd78c..a2bdd155 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go @@ -213,6 +213,16 @@ func TestAPackageWithTheReadsOutsideMarkerIsNeverKept(t *testing.T) { if first.Packages[a] != plain.Packages[a] || first.Packages[a] != second.Packages[a] { t.Errorf("a has no marker and its digest moved: %s, %s, before %s", first.Packages[a], second.Packages[a], plain.Packages[a]) } + + // Issue 890/AC3: a package whose directory carries the needs marker is named with its needs at the integration level, + // comments and blank lines not read, and at the unit level not at all. + edit(t, tree, "a/"+NeedsMarker, "# its integration tests copy the installed Node toolchain\n\nnode-build-node\n") + if n := digests(t, m, LevelIntegration).Needs; len(n) != 1 || n[0].Package != a || strings.Join(n[0].Needs, ",") != "node-build-node" { + t.Errorf("the integration level's needs = %+v, want %s needing node-build-node", n, a) + } + if n := digests(t, m, LevelUnit).Needs; len(n) != 0 { + t.Errorf("the unit level names needs: %+v", n) + } } // Issue 1935: a catalogue module's packages read ../../module.json and the module's files: a change to them changes the diff --git a/modules/mesh-build-go/cmd/mesh-build-go/level.go b/modules/mesh-build-go/cmd/mesh-build-go/level.go index 6ff18aa6..9043b1db 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/level.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/level.go @@ -11,11 +11,13 @@ import ( "encoding/json" "fmt" "os" + "os/exec" "path/filepath" "strings" "time" "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/delivery" ) // levelArg is the first argument that makes this binary a step's shut-off process: the integration level's, and, when @@ -140,6 +142,33 @@ func (m *Module) nodeToolchains() []string { return []string{dir} } +// Has says whether this node has a need a check ask or a package's tests name (issue 890; mesh-sdk delivery.AskNeeds), +// read from the node itself at every ask: node-build-node when the Node build module's toolchains directory holds an +// installed toolchain this account may read, given to the integration level as nodeToolchains gives it, and node and npm +// are installed, as mesh-build-node's integration tests look for them before they run. Any other need it lacks. +func (m *Module) Has(need string) bool { + if m.HasNeed != nil { + return m.HasNeed(need) + } + switch need { + case delivery.NeedNodeBuildNode: + dirs := m.nodeToolchains() + if len(dirs) == 0 { + return false + } + if entries, err := os.ReadDir(dirs[0]); err != nil || len(entries) == 0 { + return false + } + for _, p := range []string{"node", "npm"} { + if _, err := exec.LookPath(p); err != nil { + return false + } + } + return true + } + return false +} + func lastLine(s string) string { lines := strings.Split(strings.TrimSpace(s), "\n") return strings.TrimSpace(lines[len(lines)-1]) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module.go b/modules/mesh-build-go/cmd/mesh-build-go/module.go index f7ef5317..184f4f21 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/module.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/module.go @@ -73,6 +73,8 @@ type Module struct { // NodeToolchains is where the Node build module keeps its installed toolchains, bound read-only into the // integration level (ADR 0533 §1); NodeToolchains when empty. A test points it elsewhere. NodeToolchains string + // HasNeed answers Has in place of the node's own reading; a test's. + HasNeed func(need string) bool // StartBus starts the throwaway bus of a run in dir and answers its address and what stops it (the module's own // program, StartBus in bus.go, when nil). StartBus func(ctx context.Context, dir string) (url string, stop func(), err error) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/node_toolchains_test.go b/modules/mesh-build-go/cmd/mesh-build-go/node_toolchains_test.go index 0c48ac73..bd032546 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/node_toolchains_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/node_toolchains_test.go @@ -71,6 +71,35 @@ func TestANodeWithoutNodeToolchainsRunsTheLevelWithNothingMoreBound(t *testing.T if slices.Contains(s.roBinds, missing) { t.Errorf("a missing %s is bound: %v", missing, s.roBinds) } + + // Issue 890/AC4: such a node lacks node-build-node; one with an installed toolchain and node and npm has it; nothing + // else is had. + bin := t.TempDir() + for _, p := range []string{"node", "npm"} { + if err := os.WriteFile(filepath.Join(bin, p), []byte("#!/bin/sh\n"), 0o755); err != nil { + t.Fatal(err) + } + } + t.Setenv("PATH", bin) + m := &Module{NodeToolchains: missing} + if m.Has(delivery.NeedNodeBuildNode) { + t.Error("a node with no Node toolchain has node-build-node") + } + installed := t.TempDir() + m.NodeToolchains = installed + if m.Has(delivery.NeedNodeBuildNode) { + t.Error("a node with an empty toolchains directory has node-build-node") + } + if err := os.Mkdir(filepath.Join(installed, "0.1.0"), 0o755); err != nil { + t.Fatal(err) + } + if !m.Has(delivery.NeedNodeBuildNode) || m.Has("gpu") { + t.Errorf("with a toolchain installed: node-build-node %v, gpu %v; want true, false", m.Has(delivery.NeedNodeBuildNode), m.Has("gpu")) + } + t.Setenv("PATH", t.TempDir()) + if m.Has(delivery.NeedNodeBuildNode) { + t.Error("a node with no node installed has node-build-node") + } } // 1891 (ADR 0533 §1): the unit level is not given them. diff --git a/modules/mesh-build-go/go.mod b/modules/mesh-build-go/go.mod index 728a1896..d44cc621 100644 --- a/modules/mesh-build-go/go.mod +++ b/modules/mesh-build-go/go.mod @@ -3,7 +3,7 @@ module meshbuildgo go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026123512-bb713103371a github.com/distribution/distribution/v3 v3.1.2 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/modules/mesh-build-go/go.sum b/modules/mesh-build-go/go.sum index 36b59fa1..5c5c5ca0 100644 --- a/modules/mesh-build-go/go.sum +++ b/modules/mesh-build-go/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4 h1:05sVel/UOvz4/g0dxrDOeVUL7BGakVrxPBlqRq02ZhM= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026123512-bb713103371a h1:ZxKOtc8LusZEEpbbu41lYxJiaITN8lZCIcasA5kdgGQ= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026123512-bb713103371a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI= github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= diff --git a/modules/mesh-build-node/cmd/mesh-build-node/check-needs b/modules/mesh-build-node/cmd/mesh-build-node/check-needs new file mode 100644 index 00000000..7d69db71 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/check-needs @@ -0,0 +1,3 @@ +# Its integration tests copy the Node toolchain the Node build module installed on this node and run node and npm +# (realMachine, build_integration_test.go): they run only where the node holds node-build-node (issue 890). +node-build-node