BM2: a test that cannot run says NOT RUN; the level's bound and the install archive reach the tests

The Go build module's level result carries not-run (the SDK reader's NOT RUN
skips), its tests get MESH_TEST_LEVEL_BOUND, the level's bound, from which
mesh-sdk's helpers take the setup bound; a test call naming an install archive
reads it from the artifact store by its reference and unpacks it into the
workspace's tree before go test. check-needs is read from the module's
manifest, on the Go artifact whose path holds the package, and the marker
file goes: the Node build module's manifest declares it on its bundle.

Builds: design 47 H2 §800; H-A-294, H-A-350 DMC-6, H-A-368, H-A-399 (1) (2) (3)
This commit is contained in:
2026-11-22 01:07:45 +01:00
parent 681de09f04
commit c63b4ae31d
14 changed files with 220 additions and 35 deletions
@@ -10,13 +10,18 @@ import (
"archive/zip"
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"net/http"
"os"
"path"
"path/filepath"
"strings"
artifactref "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/containerimageregistryclient/reference"
imageseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun"
)
@@ -108,3 +113,65 @@ func unzipBuild(data []byte, dir string) error {
}
return nil
}
// **A step's install archive, unpacked into the workspace** (design 47 H2 §800; H-A-399 (2)): an integration step whose
// module's Go artifact needs node-build-node names the archive the Node build module's install published to the artifact
// store, as its reference (`oci-artifact-registry/<module>/<artifact>@<digest>`); the level's tests run with its tree
// laid into the workspace's tree. It is read from the store by its digest, as a pinned archive is fetched by its pin:
// bytes that are not the digest's are refused, and a store that does not answer is a transport fault (§799).
var installForms = artifactref.Seats{Image: imageseat.Seat, Object: ArtifactSeat}
// layInstall fetches the archive reference names from the artifact store and unpacks it into tree.
func (m *Module) layInstall(ctx context.Context, workspace, tree, reference string) error {
where, kept := installForms.InArtifactStore(reference)
_, digest, _ := strings.Cut(where, "/blobs/")
if !kept || !strings.HasPrefix(digest, "sha256:") {
return fmt.Errorf("the install archive %q is no reference of an archive in the artifact store", reference)
}
base, err := steprun.StoreURL(m.ArtifactStoreFile)
if err != nil {
return fmt.Errorf("the install archive %s is read from the artifact store: %w", reference, err)
}
request, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(base, "/")+"/v2/"+where, nil)
if err != nil {
return err
}
response, err := m.httpClient().Do(request)
if err != nil {
return steprun.Transport(fmt.Errorf("cannot fetch the install archive %s: %w", reference, err))
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return fmt.Errorf("the artifact store answered %s for the install archive %s", response.Status, reference)
}
dir := filepath.Join(m.Work, workspace, "upstream")
if err := os.MkdirAll(dir, 0o755); err != nil {
return err
}
part, err := os.CreateTemp(dir, "install.*.part")
if err != nil {
return err
}
defer os.Remove(part.Name())
hash := sha256.New()
n, err := io.Copy(io.MultiWriter(part, hash), io.LimitReader(response.Body, maxUpstream+1))
if cerr := part.Close(); err == nil {
err = cerr
}
if err != nil {
return steprun.Transport(fmt.Errorf("fetching the install archive %s: %w", reference, err))
}
if n > maxUpstream {
return fmt.Errorf("the install archive %s is larger than %d bytes, more than a machine unpacks", reference, maxUpstream)
}
if got := "sha256:" + hex.EncodeToString(hash.Sum(nil)); got != digest {
return fmt.Errorf("the install archive %s read as %s, not its digest", reference, got)
}
if err := unpackUpstream(part.Name(), tree, ""); err != nil {
return fmt.Errorf("the install archive %s: %w", reference, err)
}
if m.Say != nil {
m.Say("laid the install archive %s into the workspace %s", reference, workspace)
}
return nil
}
@@ -14,6 +14,7 @@ import (
"testing"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun"
)
// Design 47 G §465, §373 (BG7): a fetch naming builds reads each member from the artifact store by its digest, checks the
@@ -28,9 +29,21 @@ func TestAFetchLaysAGroupMembersBuildFromTheArtifactStore(t *testing.T) {
zw.Close()
sum := sha256.Sum256(zbuf.Bytes())
digest := "sha256:" + hex.EncodeToString(sum[:])
// Design 47 H2 §800: an install archive, as the Node build module packs it, its paths from the workspace's tree.
npm := t.TempDir()
os.MkdirAll(filepath.Join(npm, "modules", "gitea", "node_modules", "x"), 0o755)
os.WriteFile(filepath.Join(npm, "modules", "gitea", "node_modules", "x", "index.js"), []byte("module.exports = 1\n"), 0o644)
install, installDigest, err := steprun.Pack(npm)
if err != nil {
t.Fatal(err)
}
var asked []string
store := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
asked = append(asked, r.URL.Path)
if r.URL.Path == "/v2/gitea/node-modules/blobs/"+installDigest {
rw.Write(install)
return
}
if r.URL.Path != "/v2/lib/pkg/blobs/"+digest {
http.NotFound(rw, r)
return
@@ -68,7 +81,21 @@ func TestAFetchLaysAGroupMembersBuildFromTheArtifactStore(t *testing.T) {
t.Errorf("member %+v: error %v", c, err)
}
}
// Design 47 H2 §800: the step's install archive is read from the store by its reference and unpacked into the tree; a
// reference whose bytes are not its digest is refused, and a store that does not answer is a transport fault (§799).
if err := m.layInstall(context.Background(), "ws1", filepath.Join(root, "tree"), "oci-artifact-registry/gitea/node-modules@"+installDigest); err != nil {
t.Fatal(err)
}
if b, err := os.ReadFile(filepath.Join(root, "tree", "modules", "gitea", "node_modules", "x", "index.js")); err != nil || string(b) != "module.exports = 1\n" {
t.Errorf("the install archive was not unpacked into the tree: %q, %v", b, err)
}
if err := m.layInstall(context.Background(), "ws1", filepath.Join(root, "tree"), "oci-artifact-registry/gitea/node-modules@"+digest); err == nil {
t.Error("an install archive whose bytes are not its digest was laid")
}
store.Close()
if err := m.layInstall(context.Background(), "ws1", filepath.Join(root, "tree"), "oci-artifact-registry/gitea/node-modules@"+installDigest); !sdkstep.IsPassing(err) {
t.Errorf("an install archive from a store that does not answer: %v, want a transport fault", err)
}
if err := m.layBuilds(context.Background(), root, []string{"."}, []sdkstep.FetchBuild{member}); err == nil || !strings.Contains(err.Error(), "example.test/lib") {
t.Errorf("a store that does not answer: %v", err)
}
@@ -18,12 +18,14 @@ import (
"os"
"path"
"path/filepath"
"slices"
"sort"
"strconv"
"strings"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
trunk "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// DigestVersion is the form of the digest's text: changed whenever what the digest hashes changes in kind.
@@ -56,31 +58,44 @@ const namesVariable = "MESH_NODE_NAMES"
// from the Go test log with each verdict, replaces the lists (issue 1936).
const ReadsOutsideMarker = "reads-outside"
// NeedsMarker is the name of a file in a package's directory that names, one per line, what its integration tests need
// of the node they run on (issue 890; mesh-delivery-sdk steprun.AskNeeds): `node-build-node` for tests that read the Node build
// module's installed toolchain. The digest reports it (DigestResult.Needs); a check ask leaves such a package untested on
// a node that lacks it, and the checker asks it again of a node that has it. A marker beside the tests, as
// ReadsOutsideMarker is, and not a manifest field: it is a property of the package's tests, read only where they are
// run, and the manifest describes the module on the machines. The unit level is never given anything of the node (ADR
// 0533 §1), so a marker binds the integration level alone. Blank lines and lines starting with # are not read.
const NeedsMarker = "check-needs"
// readNeeds is what the marker in dir names; nil when there is none.
func readNeeds(dir string) ([]string, error) {
raw, err := os.ReadFile(filepath.Join(dir, NeedsMarker))
if errors.Is(err, fs.ErrNotExist) {
return nil, nil
}
if err != nil {
return nil, err
}
var needs []string
for _, l := range strings.Split(string(raw), "\n") {
if l = strings.TrimSpace(l); l != "" && !strings.HasPrefix(l, "#") {
needs = append(needs, l)
// readNeeds is what a package's integration tests need of the node they run on (issue 890; mesh-delivery-sdk
// steprun.AskNeeds): the seats its module's manifest names in `check-needs` on each Go artifact whose path holds the
// package's directory (design 47 H2 §800; H-A-350 DMC-6, H-A-399 (3)), `node-build-node` for tests that read the Node
// build module's installed toolchain. The manifest is the nearest module.json in dir or above it within the tree; nil
// when there is none, or none of its Go artifacts holds dir. The digest reports it (DigestResult.Needs); a check ask
// leaves such a package untested on a node that lacks it, and the checker asks it again of a node that has it. The unit
// level is never given anything of the node (ADR 0533 §1), so the field binds the integration level alone.
func readNeeds(tree, dir string) ([]string, error) {
for at := dir; within(tree, at); at = filepath.Dir(at) {
raw, err := os.ReadFile(filepath.Join(at, "module.json"))
if errors.Is(err, fs.ErrNotExist) {
if at == tree {
break
}
continue
}
if err != nil {
return nil, err
}
manifest, err := catalogue.Decode(raw)
if err != nil {
return nil, fmt.Errorf("the manifest %s: %w", filepath.Join(at, "module.json"), err)
}
var needs []string
if manifest.Build != nil {
for _, a := range manifest.Build.Artifacts {
if a.Language == "go" && len(a.CheckNeeds) > 0 && within(filepath.Join(at, filepath.FromSlash(a.From)), dir) {
for _, n := range a.CheckNeeds {
if !slices.Contains(needs, n) {
needs = append(needs, n)
}
}
}
}
}
return needs, nil
}
return needs, nil
return nil, nil
}
// integrationTags are the build tags of the integration level, its run and its digest alike: integration, and isolation
@@ -149,7 +164,7 @@ type DigestResult struct {
// of the files markers name, which several packages share (the catalogue's manifests).
neverKept map[string]string
hashedFiles map[string]string
// Needs are the packages whose tests at this level need something of the node, as their NeedsMarker names it.
// Needs are the packages whose tests at this level need something of the node, as their module's manifest names them (check-needs).
Needs []sdkstep.PackageNeeds `json:"needs,omitempty"`
// JudgedBy is the program that ran the digest (ADR 0535 §5): "installed", or "candidate <merge commit>" under the
// runner workspace.
@@ -340,9 +355,9 @@ func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, he
return fmt.Errorf("go list gave no test binary for %s", path)
}
if needs {
n, err := readNeeds(root.Dir)
n, err := readNeeds(tree, root.Dir)
if err != nil {
return fmt.Errorf("%s: its %s: %w", path, NeedsMarker, err)
return fmt.Errorf("%s: its check-needs: %w", path, err)
}
if len(n) > 0 {
res.Needs = append(res.Needs, sdkstep.PackageNeeds{Package: path, Needs: n})
@@ -353,9 +353,12 @@ func TestAListedReadsOutsideMarkerKeepsTheDigestOfOnlyWhatItLists(t *testing.T)
before = after
}
// Issue 890/AC3: a package whose directory carries the needs marker is named with its needs at the integration level,
// comments and blank lines not read, and at the unit level not at all.
edit(t, tree, "a/"+NeedsMarker, "# its integration tests copy the installed Node toolchain\n\nnode-build-node\n")
// Issue 890/AC3, design 47 H2 §800 (H-A-350 DMC-6): a package a Go artifact of its module's manifest holds is named
// with the artifact's check-needs at the integration level, a package no such artifact holds (c) not, and at the
// unit level not at all.
edit(t, tree, "module.json", `{"module":"digest","build":{"artifacts":[`+
`{"name":"tools","kind":"bundle","language":"go","from":"a","check-needs":["node-build-node"]},`+
`{"name":"other","kind":"bundle","language":"go","from":"c"}]}}`)
if n := digests(t, m, LevelIntegration).Needs; len(n) != 1 || n[0].Package != a || strings.Join(n[0].Needs, ",") != "node-build-node" {
t.Errorf("the integration level's needs = %+v, want %s needing node-build-node", n, a)
}
@@ -59,6 +59,10 @@ type TestResult struct {
SetupBound float64 `json:"setup_bound,omitempty"`
Slowest []TestTime `json:"slowest,omitempty"`
Failed []Failure `json:"failed,omitempty"`
// NotRun are the tests that said they did not run (`t.Skip("NOT RUN: <reason>")`), each with its reason (design 47 H2
// §800, H-A-294): counted apart from passed and failed, they leave the level passing, listed here, where the delivery
// SDK's test step reads them into the step's end.
NotRun []sdkstep.NotRun `json:"not-run,omitempty"`
// Store and Bus are what the integration level ran against: the postgres major version and the nats-server
// version (ADR 0470 §4).
// OverBound says a bound was kept no longer, and OverWhich which: setup, run or both. The level's verdict, Passed, is
@@ -86,6 +90,7 @@ func readGoTest(res *TestResult, dir, out string) (noise []string, packages int)
res.Tests += read.Tests
res.Skipped += read.Skipped
res.Failed = append(res.Failed, read.Failed...)
res.NotRun = append(res.NotRun, read.NotRun...)
res.Packages = append(res.Packages, read.Packages...)
res.Slowest = append(res.Slowest, read.Times...)
res.Elapsed = max(res.Elapsed, read.Elapsed)
@@ -42,7 +42,8 @@ func (m *Module) testTool() StepTool {
"run": map[string]any{"type": "string", "description": "optional go test -run pattern: only the tests it matches run (a try's tests)"},
sdkstep.NodeNamesArg: map[string]any{"type": "array", "items": map[string]any{"type": "string"},
"description": "optional, the names of the mesh's nodes: the tests get them comma-separated in " + sdkstep.NodeNamesEnv + " (a check that keeps node names out of the repository reads them there)"},
"runner": runnerProperty,
"runner": runnerProperty,
"archive": map[string]any{"type": "string", "description": "optional, at the integration level: the reference of the install archive in the artifact store (oci-artifact-registry/<module>/<artifact>@<digest>), unpacked into the workspace's tree before the tests run (a package whose module's Go artifact names check-needs)"},
},
"required": []string{"workspace", "level"},
},
@@ -85,6 +86,12 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
if err != nil {
return nil, err
}
if archive, _ := args["archive"].(string); archive != "" {
// Design 47 H2 §800: the step's install archive is laid into the tree before go test.
if err := m.layInstall(ctx, workspace, tree, archive); err != nil {
return nil, err
}
}
var res *TestResult
switch {
case runner == RunnerWorkspace && (level == LevelUnit || level == LevelIntegration):
@@ -312,7 +319,7 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR
args = append(args, byDir[d]...)
dr := &runs[i]
dr.res = &TestResult{}
dr.r = run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), o.env...), "go", args...)
dr.r = run(ctx, append(append(m.goEnvFor(filepath.Join(tree, d)), o.env...), levelBoundEnv(o.bounds)...), "go", args...)
dr.noise, dr.packages = readGoTest(dr.res, d, dr.r.Stdout)
}()
}
@@ -329,6 +336,7 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR
res.Tests += dr.res.Tests
res.Skipped += dr.res.Skipped
res.Failed = append(res.Failed, dr.res.Failed...)
res.NotRun = append(res.NotRun, dr.res.NotRun...)
res.Packages = append(res.Packages, dr.res.Packages...)
res.Slowest = append(res.Slowest, dr.res.Slowest...)
if dr.res.Elapsed > res.Elapsed {
@@ -342,6 +350,18 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR
return res, nil
}
// LevelBoundEnv is the word each level's tests get its bound in, as a duration ("10m0s"; design 47 H2 §800): mesh-sdk's
// test helpers take a setup bound of half of it, never a value in the test (testbus.SetupBound).
const LevelBoundEnv = "MESH_TEST_LEVEL_BOUND"
// levelBoundEnv is the level's run bound as LevelBoundEnv, nothing when the level has none.
func levelBoundEnv(b Bounds) []string {
if b.Run <= 0 {
return nil
}
return []string{LevelBoundEnv + "=" + b.Run.String()}
}
// runAnswer is what a step runner answers.
type runAnswer = forgekey.Ran
@@ -5,6 +5,7 @@ import (
"encoding/json"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
@@ -101,6 +102,14 @@ func TestUnitLevelNamesTheFailingTestAndFailsTheLevel(t *testing.T) {
if !good || res.Tests != 2 {
t.Errorf("packages = %+v, tests = %d; want the good package passed with its test, 2 tests", res.Packages, res.Tests)
}
// Design 47 H2 §800: a test that says NOT RUN leaves its level passing, listed in the result's not-run with its reason.
m, states, ws, _ = workspaceFor(t, "notrun")
res, st = runUnit(t, m, states, ws)
want := []sdkstep.NotRun{{Test: "fixture/notrun/npm TestWithNpm", Reason: "npm is not on this node"}}
if st.State != sdkstep.StepPassed || !res.Passed || !slices.Equal(res.NotRun, want) {
t.Errorf("step %s, passed %v, not run %+v; want the level passing with not run %+v", st.State, res.Passed, res.NotRun, want)
}
}
// Issue 1921: a fixture with a failing test in each of two packages: one level's run names both, each with its own
@@ -271,7 +280,8 @@ func TestUnitLevelRunsGoTestJsonInTheModulesDirectoryWithTheStepsEnvironment(t *
if line != want {
t.Errorf("command = %q, want %q", line, want)
}
for _, w := range []string{"GOTOOLCHAIN=local", "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod")} {
// Design 47 H2 §800: the level's bound reaches the tests as MESH_TEST_LEVEL_BOUND, a duration.
for _, w := range []string{"GOTOOLCHAIN=local", "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), "MESH_TEST_LEVEL_BOUND=" + m.unitBound().String()} {
if !strings.Contains(env, w) {
t.Errorf("environment %s lacks %s", env, w)
}
@@ -310,6 +320,7 @@ func TestATestOfAnUnknownWorkspaceOrAnotherLevelIsRefused(t *testing.T) {
{"workspace": "ws1", "level": "stress"},
{"workspace": "ws1", "level": "unit", "packages": []any{"-run=x"}},
{"workspace": "ws1", "level": "unit", "packages": []any{"../x"}},
{"workspace": "ws1", "level": "integration", "archive": "gitea/node-modules"}, // §800: an install archive the store does not name
} {
run(args)
if st := states.waitEnd(t); st.State != sdkstep.StepFailed {
@@ -29,3 +29,6 @@ other digests; the tests therefore compare the stream always and the digest only
The go-test-json-*.jsonl captures of issue 1901 (2026-10-23 and 2026-10-25) moved to mesh-sdk's conformance kit
(go/buildstep/conformance/captured/go, with their description) on 2026-10-26, with the reader they test: the build module
reads go test -json through buildstep.ReadGoTest and runs the kit (kit_test.go).
notrun.json was captured the same way on 2026-11-22 (go1.27.3) for design 47 H2 §800 in repos/notrun: a passing test
beside one that skips with "NOT RUN: <reason>".
@@ -0,0 +1,13 @@
{"Time":"2026-11-22T01:01:16.828741416+01:00","Action":"start","Package":"fixture/notrun/npm"}
{"Time":"2026-11-22T01:01:16.830715274+01:00","Action":"run","Package":"fixture/notrun/npm","Test":"TestTwo"}
{"Time":"2026-11-22T01:01:16.830773544+01:00","Action":"output","Package":"fixture/notrun/npm","Test":"TestTwo","Output":"=== RUN TestTwo\n","OutputType":"frame"}
{"Time":"2026-11-22T01:01:16.830797205+01:00","Action":"output","Package":"fixture/notrun/npm","Test":"TestTwo","Output":"--- PASS: TestTwo (0.00s)\n","OutputType":"frame"}
{"Time":"2026-11-22T01:01:16.830801905+01:00","Action":"pass","Package":"fixture/notrun/npm","Test":"TestTwo","Elapsed":0}
{"Time":"2026-11-22T01:01:16.830807995+01:00","Action":"run","Package":"fixture/notrun/npm","Test":"TestWithNpm"}
{"Time":"2026-11-22T01:01:16.830811005+01:00","Action":"output","Package":"fixture/notrun/npm","Test":"TestWithNpm","Output":"=== RUN TestWithNpm\n","OutputType":"frame"}
{"Time":"2026-11-22T01:01:16.830814835+01:00","Action":"output","Package":"fixture/notrun/npm","Test":"TestWithNpm","Output":" npm_test.go:12: NOT RUN: npm is not on this node\n"}
{"Time":"2026-11-22T01:01:16.830822885+01:00","Action":"output","Package":"fixture/notrun/npm","Test":"TestWithNpm","Output":"--- SKIP: TestWithNpm (0.00s)\n","OutputType":"frame"}
{"Time":"2026-11-22T01:01:16.830826595+01:00","Action":"skip","Package":"fixture/notrun/npm","Test":"TestWithNpm","Elapsed":0}
{"Time":"2026-11-22T01:01:16.830830225+01:00","Action":"output","Package":"fixture/notrun/npm","Output":"PASS\n","OutputType":"frame"}
{"Time":"2026-11-22T01:01:16.831057928+01:00","Action":"output","Package":"fixture/notrun/npm","Output":"ok \tfixture/notrun/npm\t0.002s\n"}
{"Time":"2026-11-22T01:01:16.831077789+01:00","Action":"pass","Package":"fixture/notrun/npm","Elapsed":0.002}
@@ -0,0 +1,3 @@
module fixture/notrun
go 1.25.0
@@ -0,0 +1,5 @@
// Package npm is a fixture whose one test lacks the tooling its step gives.
package npm
// Two is two.
func Two() int { return 2 }
@@ -0,0 +1,13 @@
package npm
import "testing"
func TestTwo(t *testing.T) {
if Two() != 2 {
t.Errorf("Two() = %d, want 2", Two())
}
}
func TestWithNpm(t *testing.T) {
t.Skip("NOT RUN: npm is not on this node")
}
@@ -1,3 +0,0 @@
# Its integration tests copy the Node toolchain the Node build module installed on this node and run node and npm
# (realMachine, build_integration_test.go): they run only where the node holds node-build-node (issue 890).
node-build-node
+3
View File
@@ -211,6 +211,9 @@
"binary": "mesh-build-node",
"loads": [
"mesh-build-node"
],
"check-needs": [
"node-build-node"
]
},
{