diff --git a/checks/agenthome/reads-outside b/checks/agenthome/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/checks/agenthome/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/checks/config/reads-outside b/checks/config/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/checks/config/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/checks/deps/reads-outside b/checks/deps/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/checks/deps/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/checks/health/reads-outside b/checks/health/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/checks/health/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/checks/imagebuild/reads-outside b/checks/imagebuild/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/checks/imagebuild/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/checks/imageplace/reads-outside b/checks/imageplace/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/checks/imageplace/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/checks/locks/reads-outside b/checks/locks/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/checks/locks/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/checks/mailstore/reads-outside b/checks/mailstore/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/checks/mailstore/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/checks/noscripts/reads-outside b/checks/noscripts/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/checks/noscripts/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/checks/owned/reads-outside b/checks/owned/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/checks/owned/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/checks/particulars/reads-outside b/checks/particulars/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/checks/particulars/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/checks/paths/main.go b/checks/paths/main.go index 122b7aa2..8c4c9b3d 100644 --- a/checks/paths/main.go +++ b/checks/paths/main.go @@ -251,7 +251,8 @@ func main() { } var grown []string if baseDir == "" { - fmt.Println("NOT COMPARED: no base branch's lists were given, so additions to them were not judged") + fmt.Fprintln(os.Stderr, "NOT COMPARED: no base branch's lists were given (-base ), so additions to them were not judged; that is a failure, not a pass (issue 1295)") + os.Exit(2) } else { baseShared, _, err1 := list(filepath.Join(baseDir, "shared")) baseKnown, _, err2 := list(filepath.Join(baseDir, "known")) diff --git a/checks/paths/reads-outside b/checks/paths/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/checks/paths/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/checks/paths/realtree_test.go b/checks/paths/realtree_test.go index d42f7f5a..c689267d 100644 --- a/checks/paths/realtree_test.go +++ b/checks/paths/realtree_test.go @@ -32,8 +32,8 @@ func TestNoModuleNamesAnothersState(t *testing.T) { } } -// And neither list gains a line the trunk's copies at the merge base do not hold, unless the line says why after `# allow:`. Skipped, -// and said, where no trunk is beside. +// And neither list gains a line the trunk's copies at the merge base do not hold, unless the line says why after `# allow:`. Fails, saying +// NOT COMPARED, where no trunk is beside (issue 1295). func TestTheListsOfPathsHoldNoLineTheTrunksDoNot(t *testing.T) { sharedPath, okShared := trunk.Copy(t, "checks/paths/shared") knownPath, okKnown := trunk.Copy(t, "checks/paths/known") diff --git a/checks/predefined/reads-outside b/checks/predefined/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/checks/predefined/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/checks/secrets/reads-outside b/checks/secrets/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/checks/secrets/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/checks/sshdir/reads-outside b/checks/sshdir/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/checks/sshdir/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/checks/templates/reads-outside b/checks/templates/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/checks/templates/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/checks/toolchains/reads-outside b/checks/toolchains/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/checks/toolchains/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/checks/trunk/reads-outside b/checks/trunk/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/checks/trunk/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/checks/trunk/trunk.go b/checks/trunk/trunk.go index e3bc86f3..1c4b0da4 100644 --- a/checks/trunk/trunk.go +++ b/checks/trunk/trunk.go @@ -3,14 +3,14 @@ // // The checker fetches the change's own repository at its base commit beside the change, as it fetches a sibling, and says where // in MESH_CHECK_BESIDE; the copy is under the repository's name (the same convention the other repositories' tests use). With -// MESH_CHECK_BESIDE unset, or set without the catalogue's copy in it, the comparison is skipped and the test says NOT COMPARED -// WITH THE TRUNK, naming why: the checker that runs before R4g's delivery places siblings but not the change's own repository, -// so R4g's own check cannot hold the copy (issue 1295 notes that a missing copy becomes a failure once that checker is gone). A trunk that holds no such file had +// MESH_CHECK_BESIDE unset, or set without the catalogue's copy in it, the test fails saying NOT COMPARED WITH THE TRUNK and +// why: the checker places the change's own repository since R4g (issue 1295), so a missing copy is a fault, never a pass. A trunk that holds no such file had // none: the list's first copy, which nothing was added to. package trunk import ( "errors" + "fmt" "os" "path/filepath" "testing" @@ -23,13 +23,11 @@ const Repository = "mesh-catalog" const BesideEnv = "MESH_CHECK_BESIDE" // Copy is the trunk's copy of the file at rel (a path from the catalogue's root), written to a file of the test's own: its path, -// and whether the trunk holds one (false: its first copy). It skips the test, saying so, when no copy of the catalogue is beside. +// and whether the trunk holds one (false: its first copy). It fails the test when no copy of the catalogue is beside. func Copy(t testing.TB, rel string) (string, bool) { t.Helper() - raw, found, skip, err := Read(rel) + raw, found, err := Read(rel) switch { - case skip != "": - t.Skip(skip) case err != nil: t.Fatal(err) case !found: @@ -43,27 +41,25 @@ func Copy(t testing.TB, rel string) (string, bool) { return path, true } -// Read is the trunk's copy of rel and whether the trunk holds one. skip is the words to skip the comparison with, when no trunk is -// named; err is a trunk named and not there, or unreadable. -func Read(rel string) (content []byte, found bool, skip string, err error) { +// Read is the trunk's copy of rel and whether the trunk holds one. err is a trunk not beside the check (MESH_CHECK_BESIDE unset, +// or without the catalogue's directory), or a copy unreadable: a comparison that cannot be made is never a pass. +func Read(rel string) (content []byte, found bool, err error) { beside := os.Getenv(BesideEnv) if beside == "" { - return nil, false, "NOT COMPARED WITH THE TRUNK: " + BesideEnv + " is not set, so the trunk's copy of " + rel + - " is not beside this run; the checker places it", nil + return nil, false, fmt.Errorf("NOT COMPARED WITH THE TRUNK: %s is not set, so the trunk's copy of %s is not beside this run; the checker places the change's own repository there (issue 1295)", BesideEnv, rel) } dir := filepath.Join(beside, Repository) if _, err := os.Stat(dir); errors.Is(err, os.ErrNotExist) { - return nil, false, "NOT COMPARED WITH THE TRUNK: " + Repository + " is not beside this check in " + BesideEnv + "=" + beside + - ", so " + rel + " is not judged against the trunk's copy (the checker before R4g places no copy of the change's own repository)", nil + return nil, false, fmt.Errorf("NOT COMPARED WITH THE TRUNK: %s is not beside this check in %s=%s, so %s is not judged against the trunk's copy; the checker places the change's own repository there at its trunk (issue 1295)", Repository, BesideEnv, beside, rel) } else if err != nil { - return nil, false, "", err + return nil, false, err } raw, err := os.ReadFile(filepath.Join(dir, rel)) if errors.Is(err, os.ErrNotExist) { - return nil, false, "", nil + return nil, false, nil } if err != nil { - return nil, false, "", err + return nil, false, err } - return raw, true, "", nil + return raw, true, nil } diff --git a/checks/trunk/trunk_test.go b/checks/trunk/trunk_test.go index d8283f4d..aa0ace00 100644 --- a/checks/trunk/trunk_test.go +++ b/checks/trunk/trunk_test.go @@ -17,27 +17,27 @@ func TestTheCloneBesideTheCheckIsTheTrunk(t *testing.T) { t.Fatal(err) } t.Setenv(BesideEnv, beside) - raw, found, skip, err := Read("checks/x/known") - if err != nil || skip != "" || !found || string(raw) != "a\nb\n" { - t.Fatalf("read %q found %v skip %q err %v", raw, found, skip, err) + raw, found, err := Read("checks/x/known") + if err != nil || !found || string(raw) != "a\nb\n" { + t.Fatalf("read %q found %v err %v", raw, found, err) } - if _, found, skip, err := Read("checks/x/absent"); err != nil || skip != "" || found { - t.Fatalf("a file the trunk does not hold: found %v skip %q err %v; wanted its first copy", found, skip, err) + if _, found, err := Read("checks/x/absent"); err != nil || found { + t.Fatalf("a file the trunk does not hold: found %v err %v; wanted its first copy", found, err) } } -func TestWithNothingBesideTheComparisonIsSkippedAndSaidNotMade(t *testing.T) { +func TestWithNothingBesideTheComparisonFailsAndSaysNotMade(t *testing.T) { t.Setenv(BesideEnv, "") - _, found, skip, err := Read("checks/x/known") - if found || err != nil || len(skip) < 12 || skip[:12] != "NOT COMPARED" { - t.Fatalf("found %v, skip %q, err %v; wanted NOT COMPARED", found, skip, err) + _, found, err := Read("checks/x/known") + if found || err == nil || !strings.Contains(err.Error(), "NOT COMPARED") { + t.Fatalf("found %v, err %v; wanted a failure saying NOT COMPARED", found, err) } } -func TestACloneBesideWithoutTheCatalogueIsSkippedAndSaidNotMade(t *testing.T) { - t.Setenv(BesideEnv, t.TempDir()) // the siblings are there, the catalogue's own base is not (the checker before R4g) - _, found, skip, err := Read("checks/x/known") - if found || err != nil || len(skip) < 12 || skip[:12] != "NOT COMPARED" || !strings.Contains(skip, "not beside") { - t.Fatalf("found %v, skip %q, err %v; wanted NOT COMPARED naming the missing copy", found, skip, err) +func TestACloneBesideWithoutTheCatalogueFailsAndSaysNotMade(t *testing.T) { + t.Setenv(BesideEnv, t.TempDir()) // the siblings are there, the catalogue's own base is not + _, found, err := Read("checks/x/known") + if found || err == nil || !strings.Contains(err.Error(), "NOT COMPARED") || !strings.Contains(err.Error(), "not beside") { + t.Fatalf("found %v, err %v; wanted a failure naming the missing copy", found, err) } } diff --git a/checks/words/reads-outside b/checks/words/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/checks/words/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/modules/adwaita/internal/desktop/reads-outside b/modules/adwaita/internal/desktop/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/modules/adwaita/internal/desktop/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/modules/blueman/cmd/blueman-tools/reads-outside b/modules/blueman/cmd/blueman-tools/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/modules/blueman/cmd/blueman-tools/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/modules/brother/cmd/brother/reads-outside b/modules/brother/cmd/brother/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/modules/brother/cmd/brother/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/modules/forticlient/cmd/forticlient-tools/reads-outside b/modules/forticlient/cmd/forticlient-tools/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/modules/forticlient/cmd/forticlient-tools/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/modules/i3/internal/desktop/reads-outside b/modules/i3/internal/desktop/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/modules/i3/internal/desktop/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/modules/lemurs/internal/desktop/reads-outside b/modules/lemurs/internal/desktop/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/modules/lemurs/internal/desktop/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/modules/mesh-build-go/README.md b/modules/mesh-build-go/README.md index f2493e35..7b2c14f3 100644 --- a/modules/mesh-build-go/README.md +++ b/modules/mesh-build-go/README.md @@ -102,7 +102,11 @@ integration level, the postgres major and the nats-server version; and, when the workspace (`beside`), each one's repository and commit, which the answer echoes as `beside` (the checker keeps a verdict only for the siblings it was judged with). The text's form is `DigestVersion` 3. The Go version is not in it: the toolchain enters a fingerprint once, as the version the build nodes agree on (ADR 0477 §3), and the standard library's packages are that -version's. A package with no tests at the level has no digest and is named in `without_tests`. A directory that vendors +version's. A package with no tests at the level has no digest and is named in `without_tests`. A catalogue module's packages also hash the module's non-Go files (the nearest ancestor directory holding `module.json`: its manifest, README, `files/` and the rest, so `../../module.json` is covered; digest form 5). A package whose directory holds a file named `reads-outside` (any content; say what the tests read) is never kept (issue 1935): its tests read +files outside it, which a digest from `go list` cannot name, so its digest is a fresh random one on every call, the ask finds no kept +verdict for it and runs it, and the answer names it in `reads_outside` and the module's log says why. The checker still writes a +verdict under that digest, which nothing ever reads again. The marker is the plain way to say it until each verdict keeps the files +its tests opened (Go's test log). A directory that vendors is listed and tested from its vendor directory (`-mod=vendor`). ## A change to this module is judged by its candidate diff --git a/modules/mesh-build-go/cmd/mesh-build-go/asks.go b/modules/mesh-build-go/cmd/mesh-build-go/asks.go index dd516941..bb0713da 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/asks.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/asks.go @@ -100,6 +100,9 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep if err := m.room(); err != nil { return nil, err } + if a.Compose != nil { + return m.compose(ctx, ask.Fetch, a) + } return buildstep.RunAsk(ctx, ask, a) }} } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go index 3682a61b..22aebaca 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go @@ -116,7 +116,7 @@ func TestTheIntegrationLevelDigestsUnderBothTags(t *testing.T) { t.Errorf("the integration digest listed %q", s.command) } } - if DigestVersion != "4" { + if DigestVersion != "5" { t.Errorf("the digest's form changed in kind and is still version %s", DigestVersion) } } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/compose.go b/modules/mesh-build-go/cmd/mesh-build-go/compose.go new file mode 100644 index 00000000..c733a15a --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/compose.go @@ -0,0 +1,210 @@ +package main + +// **A compose ask runs mesh-sdk's composer at a commit** (novox/hq ADR 0482 clause 4, issues 1621 and 1622): the +// assigner asks it, to compose a group with its mesh-sdk member's head, and to read the group's manifests with the +// mesh-sdk the running controller and node-engine pin (ADR 0482 clause 5, issue 1623). The module fetches the commit +// (its fetch step: the forge key and the network, none of the change's code), builds the composer package in the Go +// module the ask names, in a run step shut off with nothing fetched, and keeps the program under its key +// (delivery.ComposerKey), so each commit is built once on a node. It fills the request's manifests from the delivery +// indexer's manifests state, by digest, and runs the program shut off: no network, none of the account, the request a +// file of a workspace of its own. The answer is the composer's verdict whole (delivery.ComposeAnswer); the module +// judges nothing of it. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + + "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/composer" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// ManifestsState is the delivery indexer's manifests by digest, which a compose ask's request names (the module reads +// it, module.json reads). +const ManifestsState = delivery.IndexerModule + ".manifests" + +// composerLocks hold one mutex per composer key, so a composer two asks need at once is built once. +var composerLocks sync.Map + +// composerPath is where the composer of key is kept: the cache directory, beside Go's own caches. +func (m *Module) composerPath(key string) string { + return filepath.Join(m.Cache, "composers", key, "mesh-compose") +} + +// compose runs a compose ask: the composer of its commit, built once, over its request with the manifests filled. +func (m *Module) compose(ctx context.Context, fetch buildstep.StepFunc, a delivery.StepAsk) (any, error) { + job := *a.Compose + repository, _ := job.Fetch["repository"].(string) + commit, _ := job.Fetch["commit"].(string) + installed, err := m.Probe(ctx) + if err != nil { + return nil, err + } + key := delivery.ComposerKey(repository, commit, job.Dir, job.Package, delivery.ToolchainUsed{Version: installed, Recipe: m.Recipe.Version()}) + exe, built, err := m.composerOf(ctx, fetch, job, key) + if err != nil { + return nil, err + } + req, err := composer.Decode(job.Request) + if err != nil { + return nil, buildstep.Refuse("the compose ask %s: %v", a.ID, err) + } + if err := composer.Fill(&req, m.manifestOf); err != nil { + return nil, err + } + verdict, err := m.runComposer(ctx, exe, req) + if err != nil { + return nil, err + } + return delivery.ComposeAnswer{Built: built, Key: key, Verdict: verdict}, nil +} + +// manifestOf is the manifest the indexer keeps under digest, as its repository holds it. +func (m *Module) manifestOf(digest string) ([]byte, error) { + if m.ReadManifest == nil { + return nil, errors.New("this module reads no manifests state") + } + key, err := delivery.ManifestsKey(digest) + if err != nil { + return nil, err + } + raw, err := m.ReadManifest(key) + if err != nil { + return nil, err + } + if raw == nil { + return nil, fmt.Errorf("the indexer keeps no manifest of digest %s", digest) + } + man, err := delivery.DecodeManifest(raw) + if err != nil { + return nil, err + } + return []byte(man.JSON), nil +} + +// composerOf is the composer kept under key, or, when none is, the one built now from the fetched commit (built true). +func (m *Module) composerOf(ctx context.Context, fetch buildstep.StepFunc, job delivery.ComposeJob, key string) (string, bool, error) { + exe := m.composerPath(key) + mu, _ := composerLocks.LoadOrStore(exe, &sync.Mutex{}) + mu.(*sync.Mutex).Lock() + defer mu.(*sync.Mutex).Unlock() + if st, err := os.Stat(exe); err == nil && st.Mode().IsRegular() { + return exe, false, nil + } + if fetch == nil { + return "", false, buildstep.Refuse("this build module fetches nothing") + } + out, err := fetch(ctx, job.Fetch) + if err != nil { + return "", false, err + } + raw, err := json.Marshal(out) + if err != nil { + return "", false, err + } + var f struct { + Workspace string `json:"workspace"` + } + if err := json.Unmarshal(raw, &f); err != nil || f.Workspace == "" { + return "", false, fmt.Errorf("the fetch answered no workspace: %s", raw) + } + defer m.releaseSaid(f.Workspace) + done := m.spaces().Hold(f.Workspace) + defer done() + dir := filepath.Join(m.Work, f.Workspace, "tree", job.Dir) + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err != nil { + return "", false, fmt.Errorf("%s at %s holds no go.mod in %s, so no composer can be built there", job.Fetch["repository"], job.Fetch["commit"], job.Dir) + } + into := filepath.Join(m.Work, f.Workspace, "composer", "mesh-compose") + if err := os.MkdirAll(filepath.Dir(into), 0o700); err != nil { + return "", false, err + } + run, err := m.stepRunner(f.Workspace, false) + if err != nil { + return "", false, err + } + env := append(m.goEnvFor(dir), "GOPROXY=off", "GOTOOLCHAIN=local", "CGO_ENABLED=0") + r := run(ctx, env, "go", "-C", dir, "build", "-trimpath", "-o", into, job.Package) + if r.Status != 0 || r.Err != "" { + // The words Go says when the module holds no such package (a mesh-sdk from before the composer) are kept whole, + // so the asker tells that apart from a fault. + return "", false, fmt.Errorf("the composer %s does not build at %s: %s", job.Package, job.Fetch["commit"], compileError(r.Err+"\n"+r.Stderr+"\n"+r.Stdout)) + } + if err := copyExecutable(into, exe); err != nil { + return "", false, fmt.Errorf("the composer could not be kept: %w", err) + } + return exe, true, nil +} + +// copyExecutable copies a built program into place through a file beside it, so a half-written program is never kept. +func copyExecutable(from, to string) error { + if err := os.MkdirAll(filepath.Dir(to), 0o700); err != nil { + return err + } + src, err := os.Open(from) + if err != nil { + return err + } + defer src.Close() + tmp := to + ".new" + dst, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o700) + if err != nil { + return err + } + if _, err := io.Copy(dst, src); err != nil { + dst.Close() + return err + } + if err := dst.Close(); err != nil { + return err + } + return os.Rename(tmp, to) +} + +// releaseSaid removes a workspace once its steps ended, saying a failure: the sweep removes it later. +func (m *Module) releaseSaid(workspace string) { + if err := m.spaces().Release(workspace); err != nil && m.Say != nil { + m.Say("the compose ask's workspace %s: %v", workspace, err) + } +} + +// runComposer runs the composer over req in a workspace of its own, shut off, and answers its verdict: the last line of +// its standard output, a composer.Verdict of the version this module reads. +func (m *Module) runComposer(ctx context.Context, exe string, req composer.Request) (json.RawMessage, error) { + id := buildstep.NewID() + done := m.spaces().Hold(id) + defer m.releaseSaid(id) + defer done() + ws := filepath.Join(m.Work, id) + if err := os.MkdirAll(ws, 0o700); err != nil { + return nil, err + } + raw, err := json.Marshal(req) + if err != nil { + return nil, err + } + file := filepath.Join(ws, "request.json") + if err := os.WriteFile(file, raw, 0o600); err != nil { + return nil, err + } + run := m.Run + if !m.Inside { + iso := m.isolation(id, false) + iso.ReadOnly = append(iso.ReadOnly, exe) + run = buildstep.Isolated(m.Run, iso) + } + r := run(ctx, nil, exe, "-request", file) + line := strings.TrimSpace(lastLine(r.Stdout)) + var v composer.Verdict + if r.Status != 0 || r.Err != "" || json.Unmarshal([]byte(line), &v) != nil || v.V != composer.Version { + return nil, fmt.Errorf("the composer answered no verdict (status %d %s): %s", r.Status, r.Err, firstLine(r.Stderr+"\n"+r.Stdout)) + } + return json.RawMessage(line), nil +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/compose_integration_test.go b/modules/mesh-build-go/cmd/mesh-build-go/compose_integration_test.go new file mode 100644 index 00000000..ad1fe4d0 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/compose_integration_test.go @@ -0,0 +1,156 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/composer" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/forgekey" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +// sdkPin is the mesh-sdk version this module's go.mod requires: the composer the test builds is that version's. +func sdkPin(t *testing.T) string { + t.Helper() + raw, err := os.ReadFile("../../go.mod") + if err != nil { + t.Fatal(err) + } + for _, l := range strings.Split(string(raw), "\n") { + if f := strings.Fields(l); len(f) >= 2 && f[0] == "git.novox.be/novox/mesh-sdk/go" { + return f[1] + } + } + t.Fatal("go.mod requires no mesh-sdk") + return "" +} + +// goEnvOf is one value of this machine's go env. +func goEnvOf(t *testing.T, name string) string { + t.Helper() + out, err := exec.Command("go", "env", name).Output() + if err != nil { + t.Fatalf("go env %s: %v", name, err) + } + return strings.TrimSpace(string(out)) +} + +// 1622/AC1, AC3 against the real bus: a compose ask put on the Go build seat's check stream is pulled by the module +// started whole on that bus, which fetches the commit (here a tree of a module requiring mesh-sdk at this module's pin, +// as a reader repository is), builds mesh-sdk's composer there with nothing fetched, fills the request's manifests from +// the indexer's state and runs the composer; the step key . passes with the composer's verdict. A second +// ask of the same commit runs the kept composer: fetched and built once. +func TestAComposeAskIsBuiltOnceAndAnsweredOnTheBus(t *testing.T) { + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + ctx, stop := context.WithCancel(context.Background()) + defer stop() + if err := buildstep.DeclareAskWorkers(ctx, js, delivery.AskGo, 2*time.Minute); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = js.DeleteStream(context.Background(), "SEAT_NODE_BUILD_GO") }) + + states := newStates() + m := moduleWith(states, "go1.26.0") + m.Work, m.Cache, m.Inside = t.TempDir(), t.TempDir(), true + m.Run = StepRunner(t.TempDir()) + m.GoModCache = goEnvOf(t, "GOMODCACHE") // the module cache this module's own build filled, read and not fetched + run, gocache := m.Run, goEnvOf(t, "GOCACHE") // this machine's build cache, so the composer's packages compile once + m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + return run(ctx, append(env, "GOCACHE="+gocache), name, args...) + } + read, digests := indexerHolds(t, `{"module":"demo","version":"1"}`) + m.ReadManifest = read + + gosum, err := os.ReadFile("../../go.sum") + if err != nil { + t.Fatal(err) + } + fetches := 0 + steps := fakeSteps(new([]string)) + for i := range steps { + if steps[i].Name == "build_go_fetch" { + steps[i].Run = func(ctx context.Context, args map[string]any) (any, error) { + fetches++ + ws := buildstep.NewID() + tree := filepath.Join(m.Work, ws, "tree") + if err := os.MkdirAll(tree, 0o700); err != nil { + return nil, err + } + mod := "module reader\n\ngo 1.26.0\n\nrequire git.novox.be/novox/mesh-sdk/go " + sdkPin(t) + "\n" + if err := os.WriteFile(filepath.Join(tree, "go.mod"), []byte(mod), 0o600); err != nil { + return nil, err + } + return map[string]any{"workspace": ws}, os.WriteFile(filepath.Join(tree, "go.sum"), gosum, 0o600) + } + } + } + p := m.Puller(steps, &buildstep.JetStreamAsks{JS: js, Language: delivery.AskGo}) + p.PullWait = 200 * time.Millisecond + go p.Pull(ctx) + + ask := func() delivery.Step { + a := composeAskOf(t, digests[0]) + a.Compose.Fetch["repository"], a.Compose.Dir = "novox/mesh-controller", delivery.RootPath + raw, err := delivery.EncodeStepAsk(a) + if err != nil { + t.Fatal(err) + } + subject, _ := delivery.AskSubject(delivery.AskGo, delivery.AskCheck) + if _, err := js.Publish(ctx, subject, raw); err != nil { + t.Fatal(err) + } + key, _ := delivery.AskStepKey("ace", a.ID) + for deadline := time.Now().Add(90 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + states.mu.Lock() + v := states.steps[key] + states.mu.Unlock() + if len(v) > 0 && v[len(v)-1].State != delivery.StepRunning { + return v[len(v)-1] + } + } + t.Fatalf("the compose ask %s never ended", a.ID) + return delivery.Step{} + } + for n, wantBuilt := range []bool{true, false} { + step := ask() + if step.State != delivery.StepPassed { + t.Fatalf("ask %d: %s: %s", n+1, step.State, step.Reason) + } + var ans delivery.ComposeAnswer + var v composer.Verdict + if err := json.Unmarshal(step.Result, &ans); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(ans.Verdict, &v); err != nil { + t.Fatal(err) + } + if ans.Built != wantBuilt || !strings.HasPrefix(ans.Key, "composer-") || len(v.Cases) != 1 || !v.Cases[0].Manifests["demo"].Accepted || + !strings.Contains(v.SDK, sdkPin(t)) { + t.Fatalf("ask %d: answer built %v key %s, verdict %+v; want built %v and demo accepted by mesh-sdk %s", n+1, ans.Built, ans.Key, v, wantBuilt, sdkPin(t)) + } + } + if fetches != 1 { + t.Errorf("two asks of one commit fetched %d times, want once", fetches) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/compose_test.go b/modules/mesh-build-go/cmd/mesh-build-go/compose_test.go new file mode 100644 index 00000000..14a70ea0 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/compose_test.go @@ -0,0 +1,125 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/composer" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +const composeCommit = "4444444444444444444444444444444444444444" + +// indexerHolds is the delivery indexer's manifests state holding each body under its digest, as the indexer writes it. +func indexerHolds(t *testing.T, bodies ...string) (func(string) (json.RawMessage, error), []string) { + t.Helper() + kept, digests := map[string]json.RawMessage{}, []string{} + for _, b := range bodies { + d := delivery.ManifestDigest([]byte(b)) + key, _ := delivery.ManifestsKey(d) + raw, err := delivery.EncodeManifest(delivery.Manifest{Module: "demo", JSON: b}) + if err != nil { + t.Fatal(err) + } + kept[key], digests = raw, append(digests, d) + } + return func(key string) (json.RawMessage, error) { return kept[key], nil }, digests +} + +func composeAskOf(t *testing.T, digest string) delivery.StepAsk { + t.Helper() + req, _ := json.Marshal(composer.Request{V: composer.Version, Cases: []composer.Case{{Name: "change", + Manifests: map[string]composer.Manifest{"demo": {Digest: digest}}}}}) + return delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: "mesh-delivery-assigner", + At: time.Now(), Compose: &delivery.ComposeJob{Fetch: map[string]any{"repository": "novox/mesh-sdk", "commit": composeCommit}, + Dir: "go", Package: composer.Package, Request: req}} +} + +// composeModule is a module with work and cache directories of its own, running its commands as they are (Inside: a +// unit test has no bubblewrap to shut them off in). +func composeModule(t *testing.T) *Module { + t.Helper() + m := moduleWith(newStates(), "go1.26.0") + m.Work, m.Cache, m.Inside, m.Run = t.TempDir(), t.TempDir(), true, StepRunner(t.TempDir()) + return m +} + +// 1622/AC3: a composer kept under the ask's key is run again, nothing fetched or built: the answer says it was not +// built, names the key, and carries the composer's verdict whole; the request it read had its manifests filled from the +// indexer's state. +func TestAKeptComposerIsRunWithTheManifestsFilled(t *testing.T) { + m := composeModule(t) + read, digests := indexerHolds(t, `{"module":"demo","version":"1"}`) + m.ReadManifest = read + a := composeAskOf(t, digests[0]) + key := delivery.ComposerKey("novox/mesh-sdk", composeCommit, "go", composer.Package, delivery.ToolchainUsed{Version: "go1.26.0", Recipe: m.Recipe.Version()}) + exe := m.composerPath(key) + if err := os.MkdirAll(filepath.Dir(exe), 0o700); err != nil { + t.Fatal(err) + } + // A composer that answers which manifests its request held, filled or not, as a verdict. + script := "#!/bin/sh\n" + + `if grep -q '"digest"' "$2"; then echo '{"v":1,"sdk":"stand-in","cases":[{"name":"unfilled","manifests":{}}]}'; ` + + `else echo '{"v":1,"sdk":"stand-in","cases":[{"name":"change","manifests":{"demo":{"accepted":true}}}]}'; fi` + "\n" + if err := os.WriteFile(exe, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + fetched := 0 + fetch := func(context.Context, map[string]any) (any, error) { fetched++; return nil, nil } + out, err := m.compose(context.Background(), fetch, a) + if err != nil { + t.Fatal(err) + } + ans := out.(delivery.ComposeAnswer) + var v composer.Verdict + if err := json.Unmarshal(ans.Verdict, &v); err != nil { + t.Fatal(err) + } + if ans.Built || ans.Key != key || fetched != 0 || len(v.Cases) != 1 || v.Cases[0].Name != "change" || !v.Cases[0].Manifests["demo"].Accepted { + t.Fatalf("answer = %+v (fetched %d), verdict %+v", ans, fetched, v) + } + if entries, _ := os.ReadDir(m.Work); len(entries) != 0 { + t.Errorf("the run's workspace was left: %v", entries) + } +} + +// 1622: a manifest the indexer does not keep fails the ask, naming its digest; a fetched commit with no go.mod where +// the ask builds fails it naming the directory, and its workspace is released. +func TestAComposeAskFailsLoudly(t *testing.T) { + m := composeModule(t) + m.ReadManifest, _ = indexerHolds(t) + a := composeAskOf(t, "feedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeed") + key := delivery.ComposerKey("novox/mesh-sdk", composeCommit, "go", composer.Package, delivery.ToolchainUsed{Version: "go1.26.0", Recipe: m.Recipe.Version()}) + os.MkdirAll(filepath.Dir(m.composerPath(key)), 0o700) + os.WriteFile(m.composerPath(key), []byte("#!/bin/sh\nexit 0\n"), 0o700) + if _, err := m.compose(context.Background(), nil, a); err == nil || !strings.Contains(err.Error(), "feedfeed") { + t.Errorf("an unkept manifest: %v", err) + } + m2 := composeModule(t) + fetch := func(context.Context, map[string]any) (any, error) { + if err := os.MkdirAll(filepath.Join(m2.Work, "ws0000000000a", "tree", "go"), 0o700); err != nil { + return nil, err + } + return map[string]any{"workspace": "ws0000000000a"}, nil + } + if _, err := m2.compose(context.Background(), fetch, composeAskOf(t, "x")); err == nil || !strings.Contains(err.Error(), "no go.mod in go") { + t.Errorf("a commit with no go.mod: %v", err) + } + if entries, _ := os.ReadDir(m2.Work); len(entries) != 0 { + t.Errorf("the fetched workspace was left: %v", entries) + } +} + +// 1622: the module reads the delivery indexer's manifests, which a compose ask names by digest. +func TestTheModuleReadsTheIndexersManifests(t *testing.T) { + if m := readManifest(t); !slices.Contains(m.Reads, ManifestsState) { + t.Errorf("reads = %v, lacks %s", m.Reads, ManifestsState) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest.go b/modules/mesh-build-go/cmd/mesh-build-go/digest.go index 1c92d318..bce625ba 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/digest.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest.go @@ -7,6 +7,7 @@ package main import ( "bytes" "context" + "crypto/rand" "crypto/sha256" "encoding/hex" "encoding/json" @@ -24,12 +25,20 @@ import ( // 2: no Go version in the text; the toolchain enters a fingerprint as the agreed version (ADR 0477 §3). 3: the siblings' // commits (the lead's decision L5) and the integration level's tags integration,isolation (the checker's agreement). // 4: the siblings' commits enter only the digest of a package whose tests reach a file naming MESH_CHECK_BESIDE (ADR 0536 §1). -const DigestVersion = "4" +// 5: a catalogue module's non-Go files enter the digest of its packages (issue 1935). +const DigestVersion = "5" // besideVariable is the name a file bears when it reads where the siblings lie; a package whose tests reach such a file // depends on the siblings' commits. const besideVariable = "MESH_CHECK_BESIDE" +// ReadsOutsideMarker is the name of a file in a package's directory that says its tests read files outside it (a +// repository's documents, a sibling directory): such a package is never kept (issue 1935). The digest hashes only what +// the Go toolchain lists, so it cannot name what a test opens elsewhere; a marker file, beside the Go files where a +// reader of the package sees it, is the plain way to say so until the checker keeps the files a test opened (the Go +// test log) with each verdict. +const ReadsOutsideMarker = "reads-outside" + // integrationTags are the build tags of the integration level, its run and its digest alike: integration, and isolation // for the bubblewrap proofs (mesh-sdk's and the build modules'), which a build node can run. var integrationTags = []string{"integration", "isolation"} @@ -89,6 +98,9 @@ type DigestResult struct { // only when they are the siblings it gave the fetch. Beside []string `json:"beside,omitempty"` WithoutTests []string `json:"without_tests,omitempty"` + // ReadsOutside names the packages that carry the marker: each has a digest that no kept verdict can have (a fresh + // random one), so the check runs it every time. + ReadsOutside []string `json:"reads_outside,omitempty"` // JudgedBy is the program that ran the digest (ADR 0535 §5): "installed", or "candidate " under the // runner workspace. JudgedBy string `json:"judged_by,omitempty"` @@ -199,6 +211,12 @@ func (m *Module) digestHere(ctx context.Context, workspace, level string) (Diges } } sort.Strings(res.WithoutTests) + sort.Strings(res.ReadsOutside) + for _, p := range res.ReadsOutside { + if m.Say != nil { + m.Say("the %s level of %s always runs: its directory carries %s, so its tests read files outside it that no digest names", level, p, ReadsOutsideMarker) + } + } return res, nil } @@ -256,6 +274,17 @@ func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, he if testMain == nil { return fmt.Errorf("go list gave no test binary for %s", path) } + if _, err := os.Stat(filepath.Join(root.Dir, ReadsOutsideMarker)); err == nil { + // The digest of a package that reads outside itself is one nothing can have kept: the key is a fresh random + // hex, so the ask finds no kept verdict and runs it, and what the checker writes under it is never read again. + var nonce [32]byte + if _, err := rand.Read(nonce[:]); err != nil { + return err + } + res.Packages[path] = hex.EncodeToString(nonce[:]) + res.ReadsOutside = append(res.ReadsOutside, path) + continue + } d, err := packageDigest(root, testMain, pkgs, tree, header, beside) if err != nil { return fmt.Errorf("%s: %w", path, err) @@ -303,6 +332,15 @@ func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree s } } } + // A catalogue module's tests read its manifest, its README and its files from the package's directory (../../module.json): + // every non-Go file of the module's directory is an input (issue 1935). The module is the nearest ancestor with a module.json. + if modDir := enclosingModule(tree, root.Dir); modDir != "" { + more, err := moduleFileLines(tree, root, modDir) + if err != nil { + return "", err + } + lines = append(lines, more...) + } data := filepath.Join(root.Dir, "testdata") if _, err := os.Stat(data); err == nil { err := filepath.WalkDir(data, func(path string, e fs.DirEntry, err error) error { @@ -328,6 +366,41 @@ func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree s return hex.EncodeToString(h.Sum(nil)), nil } +// enclosingModule is the nearest ancestor of dir, inside tree, that holds a module.json, or "" when there is none. +func enclosingModule(tree, dir string) string { + tree = filepath.Clean(tree) + for d := filepath.Clean(dir); d != tree && strings.HasPrefix(d, tree+string(filepath.Separator)); d = filepath.Dir(d) { + if st, err := os.Stat(filepath.Join(d, "module.json")); err == nil && !st.IsDir() { + return d + } + } + return "" +} + +// moduleFileLines are the content lines of every file of a module's directory that is not Go source (Go source is the +// packages' own, listed by go list), skipping .git and node_modules. +func moduleFileLines(tree string, p *goPackage, modDir string) ([]string, error) { + var lines []string + err := filepath.WalkDir(modDir, func(path string, e fs.DirEntry, err error) error { + if err != nil { + return err + } + if e.IsDir() { + if n := e.Name(); n == ".git" || n == "node_modules" { + return filepath.SkipDir + } + return nil + } + if strings.HasSuffix(path, ".go") { + return nil + } + l, err := contentLine(tree, p, path) + lines = append(lines, l) + return err + }) + return lines, err +} + func fileLine(tree string, p *goPackage, name string) (string, error) { return contentLine(tree, p, filepath.Join(p.Dir, name)) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go index 5e2d81d0..9b6fd78c 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go @@ -189,3 +189,71 @@ func TestAModuleOutsideTheTreeCountsByVersionAndChecksum(t *testing.T) { t.Errorf("a new version left the digest as it was") } } + +// Issue 1935: a package whose directory carries the reads-outside marker is never kept: its digest is one no kept verdict +// can have (it differs on every call, though nothing changed), and the answer names it. A package without the marker +// keeps its digest. +func TestAPackageWithTheReadsOutsideMarkerIsNeverKept(t *testing.T) { + t.Parallel() + m, tree := digestFixture(t) + plain := digests(t, m, "unit") + if len(plain.ReadsOutside) != 0 { + t.Fatalf("a package carries the marker in the fixture: %v", plain.ReadsOutside) + } + edit(t, tree, "c/"+ReadsOutsideMarker, "the tests read ../docs\n") + first, second := digests(t, m, "unit"), digests(t, m, "unit") + c := "fixture/digest/c" + if len(first.Packages[c]) != 64 || first.Packages[c] == second.Packages[c] || first.Packages[c] == plain.Packages[c] { + t.Errorf("c carries the marker and its digest is stable: %s, %s, before %s", first.Packages[c], second.Packages[c], plain.Packages[c]) + } + if strings.Join(first.ReadsOutside, ",") != c { + t.Errorf("the answer does not name the package that reads outside: %v", first.ReadsOutside) + } + a := "fixture/digest/a" + if first.Packages[a] != plain.Packages[a] || first.Packages[a] != second.Packages[a] { + t.Errorf("a has no marker and its digest moved: %s, %s, before %s", first.Packages[a], second.Packages[a], plain.Packages[a]) + } +} + +// Issue 1935: a catalogue module's packages read ../../module.json and the module's files: a change to them changes the +// digest of the module's package, and a change to another module's files does not. +func TestACatalogueModulesFilesChangeTheDigestOfItsPackagesOnly(t *testing.T) { + t.Parallel() + m, tree := digestFixture(t) + for _, mod := range []string{"mod1", "mod2"} { + dir := filepath.Join(tree, mod, "cmd", "p") + if err := os.MkdirAll(filepath.Join(tree, mod, "files"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + edit(t, tree, mod+"/module.json", `{"module":"`+mod+`"}`) + edit(t, tree, mod+"/README.md", "# "+mod+"\n") + edit(t, tree, mod+"/files/f.env", "A=1\n") + edit(t, tree, mod+"/cmd/p/p.go", "package p\n") + edit(t, tree, mod+"/cmd/p/p_test.go", "package p\n\nimport (\n\t\"os\"\n\t\"testing\"\n)\n\nfunc TestReads(t *testing.T) {\n\tif _, err := os.ReadFile(\"../../module.json\"); err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n") + } + p1, p2 := "fixture/digest/mod1/cmd/p", "fixture/digest/mod2/cmd/p" + before := digests(t, m, "unit") + if len(before.Packages[p1]) != 64 || len(before.Packages[p2]) != 64 { + t.Fatalf("no digest for the module packages: %+v", before.Packages) + } + if before.Packages[p1] == before.Packages[p2] { + t.Errorf("two modules, one digest") + } + for _, file := range []string{"mod1/module.json", "mod1/README.md", "mod1/files/f.env"} { + edit(t, tree, file, "changed "+file+"\n") + after := digests(t, m, "unit") + if after.Packages[p1] == before.Packages[p1] { + t.Errorf("a change to %s did not change the module's digest", file) + } + if after.Packages[p2] != before.Packages[p2] { + t.Errorf("a change to %s changed the other module's digest", file) + } + before = after + } + if got := digests(t, m, "unit"); got.Packages["fixture/digest/c"] == "" { + t.Errorf("a package outside any module lost its digest") + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index e5a1b49b..6dc0e424 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -288,10 +288,14 @@ func inTestdata(path string) bool { // git, through the account's rewrite of the forge's address to ssh), and nothing that asks a question. For a directory // that vendors, see goEnvFor. func (m *Module) goEnv() []string { + modcache := filepath.Join(m.Cache, "gomod") + if m.GoModCache != "" { + modcache = m.GoModCache + } return []string{ "GOTOOLCHAIN=local", "GOFLAGS=-mod=mod", - "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), + "GOMODCACHE=" + modcache, "GOCACHE=" + filepath.Join(m.Cache, "gobuild"), "GOPRIVATE=git.novox.be", "GIT_TERMINAL_PROMPT=0", diff --git a/modules/mesh-build-go/cmd/mesh-build-go/main.go b/modules/mesh-build-go/cmd/mesh-build-go/main.go index 3becd9d0..d9e0da20 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/main.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/main.go @@ -54,6 +54,7 @@ func main() { m.RegistryFile = m.Home + "/registry" m.Settings = m.Home + "/settings.json" m.ReadKept = readOf(stdio.State(KeptState)).Get + m.ReadManifest = readOf(stdio.State(ManifestsState)).Get m.Probe = ProbeGo(m.Home, m.Cache) m.Bubblewrap = ProbeBubblewrap // Outside a launch by the runtime — a build, a check — there is no node and no state to write. diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module.go b/modules/mesh-build-go/cmd/mesh-build-go/module.go index 3db6e3fb..48903962 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/module.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/module.go @@ -82,6 +82,10 @@ type Module struct { Settings string Capacity func() int ReadKept func(key string) (json.RawMessage, error) + // ReadManifest reads a key of the delivery indexer's manifests state (ManifestsState), which a compose ask's request + // names its manifests by (compose.go); GoModCache, when set, is the Go module cache in place of Cache's (a test). + ReadManifest func(key string) (json.RawMessage, error) + GoModCache string spacesOnce sync.Once workspaces *buildstep.Workspaces diff --git a/modules/mesh-build-go/cmd/mesh-build-go/reads-outside b/modules/mesh-build-go/cmd/mesh-build-go/reads-outside new file mode 100644 index 00000000..c1b869c0 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/reads-outside @@ -0,0 +1 @@ +These tests read other catalogue modules' files, outside this module (issue 1935): never kept. diff --git a/modules/mesh-build-go/go.mod b/modules/mesh-build-go/go.mod index 79763029..a5118d4a 100644 --- a/modules/mesh-build-go/go.mod +++ b/modules/mesh-build-go/go.mod @@ -3,7 +3,7 @@ module meshbuildgo go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024125710-bfc3883af980 github.com/distribution/distribution/v3 v3.1.2 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/modules/mesh-build-go/go.sum b/modules/mesh-build-go/go.sum index 81e122b8..f32f243f 100644 --- a/modules/mesh-build-go/go.sum +++ b/modules/mesh-build-go/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93 h1:FAiFY2KBKEoZSH49OZTt7/zRBIQxyl5g/KhYrpggbN4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024125710-bfc3883af980 h1:ZEEeYibEPD1rngOAqbODzRY5rUStGC46OpyQwtNCYbs= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024125710-bfc3883af980/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI= github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index 4d2b080a..600345e3 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -20,7 +20,8 @@ } ], "reads": [ - "mesh-delivery-checker.kept" + "mesh-delivery-checker.kept", + "mesh-delivery-indexer.manifests" ], "config": { "capacity": { diff --git a/modules/mesh-build-image/cmd/mesh-build-image/reads-outside b/modules/mesh-build-image/cmd/mesh-build-image/reads-outside new file mode 100644 index 00000000..c1b869c0 --- /dev/null +++ b/modules/mesh-build-image/cmd/mesh-build-image/reads-outside @@ -0,0 +1 @@ +These tests read other catalogue modules' files, outside this module (issue 1935): never kept. diff --git a/modules/mesh-build-node/cmd/mesh-build-node/reads-outside b/modules/mesh-build-node/cmd/mesh-build-node/reads-outside new file mode 100644 index 00000000..c1b869c0 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/reads-outside @@ -0,0 +1 @@ +These tests read other catalogue modules' files, outside this module (issue 1935): never kept. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reads-outside b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/modules/mesh-issues/cmd/mesh-issues/reads-outside b/modules/mesh-issues/cmd/mesh-issues/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/modules/mesh-issues/cmd/mesh-issues/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/reads-outside b/modules/mesh-knowledge/cmd/mesh-knowledge/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/modules/mesh-mail-notifier/cmd/mesh-mail-notifier/reads-outside b/modules/mesh-mail-notifier/cmd/mesh-mail-notifier/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/modules/mesh-mail-notifier/cmd/mesh-mail-notifier/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/modules/mesh-mail-reader/cmd/mesh-mail-reader/reads-outside b/modules/mesh-mail-reader/cmd/mesh-mail-reader/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/modules/mesh-mail-reader/cmd/mesh-mail-reader/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/modules/mesh-mail/cmd/mesh-mail/reads-outside b/modules/mesh-mail/cmd/mesh-mail/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/modules/mesh-mail/cmd/mesh-mail/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/reads-outside b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/reads-outside new file mode 100644 index 00000000..c1b869c0 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/reads-outside @@ -0,0 +1 @@ +These tests read other catalogue modules' files, outside this module (issue 1935): never kept. diff --git a/modules/minio/cmd/object-store/reads-outside b/modules/minio/cmd/object-store/reads-outside new file mode 100644 index 00000000..c1b869c0 --- /dev/null +++ b/modules/minio/cmd/object-store/reads-outside @@ -0,0 +1 @@ +These tests read other catalogue modules' files, outside this module (issue 1935): never kept. diff --git a/modules/networkmanager/cmd/uplink-tools/reads-outside b/modules/networkmanager/cmd/uplink-tools/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/modules/networkmanager/cmd/uplink-tools/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/modules/nextcloud-client/cmd/nextcloud-client-tools/reads-outside b/modules/nextcloud-client/cmd/nextcloud-client-tools/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/modules/nextcloud-client/cmd/nextcloud-client-tools/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/modules/nm-applet/cmd/nm-applet-tools/reads-outside b/modules/nm-applet/cmd/nm-applet-tools/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/modules/nm-applet/cmd/nm-applet-tools/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/modules/openrazer/cmd/openrazer-tools/reads-outside b/modules/openrazer/cmd/openrazer-tools/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/modules/openrazer/cmd/openrazer-tools/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/modules/pacman/cmd/pacman-tools/reads-outside b/modules/pacman/cmd/pacman-tools/reads-outside new file mode 100644 index 00000000..2c69a776 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/reads-outside @@ -0,0 +1 @@ +These tests read the repository's root predefined.json, outside this module (issue 1935): never kept. diff --git a/modules/polychromatic/cmd/polychromatic-tools/reads-outside b/modules/polychromatic/cmd/polychromatic-tools/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/modules/polychromatic/cmd/polychromatic-tools/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/modules/postgres/cmd/postgres-provider/reads-outside b/modules/postgres/cmd/postgres-provider/reads-outside new file mode 100644 index 00000000..c1b869c0 --- /dev/null +++ b/modules/postgres/cmd/postgres-provider/reads-outside @@ -0,0 +1 @@ +These tests read other catalogue modules' files, outside this module (issue 1935): never kept. diff --git a/modules/slack/cmd/slack-tools/reads-outside b/modules/slack/cmd/slack-tools/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/modules/slack/cmd/slack-tools/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/modules/systemd-networkd/cmd/uplink-tools/reads-outside b/modules/systemd-networkd/cmd/uplink-tools/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/modules/systemd-networkd/cmd/uplink-tools/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/modules/xorg/internal/desktop/reads-outside b/modules/xorg/internal/desktop/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/modules/xorg/internal/desktop/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/modules/xterm/internal/desktop/reads-outside b/modules/xterm/internal/desktop/reads-outside new file mode 100644 index 00000000..f6b2d871 --- /dev/null +++ b/modules/xterm/internal/desktop/reads-outside @@ -0,0 +1 @@ +These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).