Compare commits
13
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f910b0ceb8 | ||
|
|
91c3e954ae | ||
|
|
545c72bb59 | ||
|
|
7afcd0282d | ||
|
|
16d7b1898d | ||
|
|
f04465537b | ||
|
|
e870a7f65f | ||
|
|
e8a59171c7 | ||
|
|
ebb25757a2 | ||
|
|
6a70735489 | ||
|
|
a715c8821f | ||
|
|
b6ed8599c8 | ||
|
|
ae6f7d5b54 |
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -251,7 +251,8 @@ func main() {
|
||||
}
|
||||
var grown []string
|
||||
if baseDir == "" {
|
||||
fmt.Println("NOT COMPARED: no base branch's lists were given, so additions to them were not judged")
|
||||
fmt.Fprintln(os.Stderr, "NOT COMPARED: no base branch's lists were given (-base <dir>), so additions to them were not judged; that is a failure, not a pass (issue 1295)")
|
||||
os.Exit(2)
|
||||
} else {
|
||||
baseShared, _, err1 := list(filepath.Join(baseDir, "shared"))
|
||||
baseKnown, _, err2 := list(filepath.Join(baseDir, "known"))
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -32,8 +32,8 @@ func TestNoModuleNamesAnothersState(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// And neither list gains a line the trunk's copies at the merge base do not hold, unless the line says why after `# allow:`. Skipped,
|
||||
// and said, where no trunk is beside.
|
||||
// And neither list gains a line the trunk's copies at the merge base do not hold, unless the line says why after `# allow:`. Fails, saying
|
||||
// NOT COMPARED, where no trunk is beside (issue 1295).
|
||||
func TestTheListsOfPathsHoldNoLineTheTrunksDoNot(t *testing.T) {
|
||||
sharedPath, okShared := trunk.Copy(t, "checks/paths/shared")
|
||||
knownPath, okKnown := trunk.Copy(t, "checks/paths/known")
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
+14
-18
@@ -3,14 +3,14 @@
|
||||
//
|
||||
// The checker fetches the change's own repository at its base commit beside the change, as it fetches a sibling, and says where
|
||||
// in MESH_CHECK_BESIDE; the copy is under the repository's name (the same convention the other repositories' tests use). With
|
||||
// MESH_CHECK_BESIDE unset, or set without the catalogue's copy in it, the comparison is skipped and the test says NOT COMPARED
|
||||
// WITH THE TRUNK, naming why: the checker that runs before R4g's delivery places siblings but not the change's own repository,
|
||||
// so R4g's own check cannot hold the copy (issue 1295 notes that a missing copy becomes a failure once that checker is gone). A trunk that holds no such file had
|
||||
// MESH_CHECK_BESIDE unset, or set without the catalogue's copy in it, the test fails saying NOT COMPARED WITH THE TRUNK and
|
||||
// why: the checker places the change's own repository since R4g (issue 1295), so a missing copy is a fault, never a pass. A trunk that holds no such file had
|
||||
// none: the list's first copy, which nothing was added to.
|
||||
package trunk
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
@@ -23,13 +23,11 @@ const Repository = "mesh-catalog"
|
||||
const BesideEnv = "MESH_CHECK_BESIDE"
|
||||
|
||||
// Copy is the trunk's copy of the file at rel (a path from the catalogue's root), written to a file of the test's own: its path,
|
||||
// and whether the trunk holds one (false: its first copy). It skips the test, saying so, when no copy of the catalogue is beside.
|
||||
// and whether the trunk holds one (false: its first copy). It fails the test when no copy of the catalogue is beside.
|
||||
func Copy(t testing.TB, rel string) (string, bool) {
|
||||
t.Helper()
|
||||
raw, found, skip, err := Read(rel)
|
||||
raw, found, err := Read(rel)
|
||||
switch {
|
||||
case skip != "":
|
||||
t.Skip(skip)
|
||||
case err != nil:
|
||||
t.Fatal(err)
|
||||
case !found:
|
||||
@@ -43,27 +41,25 @@ func Copy(t testing.TB, rel string) (string, bool) {
|
||||
return path, true
|
||||
}
|
||||
|
||||
// Read is the trunk's copy of rel and whether the trunk holds one. skip is the words to skip the comparison with, when no trunk is
|
||||
// named; err is a trunk named and not there, or unreadable.
|
||||
func Read(rel string) (content []byte, found bool, skip string, err error) {
|
||||
// Read is the trunk's copy of rel and whether the trunk holds one. err is a trunk not beside the check (MESH_CHECK_BESIDE unset,
|
||||
// or without the catalogue's directory), or a copy unreadable: a comparison that cannot be made is never a pass.
|
||||
func Read(rel string) (content []byte, found bool, err error) {
|
||||
beside := os.Getenv(BesideEnv)
|
||||
if beside == "" {
|
||||
return nil, false, "NOT COMPARED WITH THE TRUNK: " + BesideEnv + " is not set, so the trunk's copy of " + rel +
|
||||
" is not beside this run; the checker places it", nil
|
||||
return nil, false, fmt.Errorf("NOT COMPARED WITH THE TRUNK: %s is not set, so the trunk's copy of %s is not beside this run; the checker places the change's own repository there (issue 1295)", BesideEnv, rel)
|
||||
}
|
||||
dir := filepath.Join(beside, Repository)
|
||||
if _, err := os.Stat(dir); errors.Is(err, os.ErrNotExist) {
|
||||
return nil, false, "NOT COMPARED WITH THE TRUNK: " + Repository + " is not beside this check in " + BesideEnv + "=" + beside +
|
||||
", so " + rel + " is not judged against the trunk's copy (the checker before R4g places no copy of the change's own repository)", nil
|
||||
return nil, false, fmt.Errorf("NOT COMPARED WITH THE TRUNK: %s is not beside this check in %s=%s, so %s is not judged against the trunk's copy; the checker places the change's own repository there at its trunk (issue 1295)", Repository, BesideEnv, beside, rel)
|
||||
} else if err != nil {
|
||||
return nil, false, "", err
|
||||
return nil, false, err
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(dir, rel))
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, false, "", nil
|
||||
return nil, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, "", err
|
||||
return nil, false, err
|
||||
}
|
||||
return raw, true, "", nil
|
||||
return raw, true, nil
|
||||
}
|
||||
|
||||
+14
-14
@@ -17,27 +17,27 @@ func TestTheCloneBesideTheCheckIsTheTrunk(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv(BesideEnv, beside)
|
||||
raw, found, skip, err := Read("checks/x/known")
|
||||
if err != nil || skip != "" || !found || string(raw) != "a\nb\n" {
|
||||
t.Fatalf("read %q found %v skip %q err %v", raw, found, skip, err)
|
||||
raw, found, err := Read("checks/x/known")
|
||||
if err != nil || !found || string(raw) != "a\nb\n" {
|
||||
t.Fatalf("read %q found %v err %v", raw, found, err)
|
||||
}
|
||||
if _, found, skip, err := Read("checks/x/absent"); err != nil || skip != "" || found {
|
||||
t.Fatalf("a file the trunk does not hold: found %v skip %q err %v; wanted its first copy", found, skip, err)
|
||||
if _, found, err := Read("checks/x/absent"); err != nil || found {
|
||||
t.Fatalf("a file the trunk does not hold: found %v err %v; wanted its first copy", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithNothingBesideTheComparisonIsSkippedAndSaidNotMade(t *testing.T) {
|
||||
func TestWithNothingBesideTheComparisonFailsAndSaysNotMade(t *testing.T) {
|
||||
t.Setenv(BesideEnv, "")
|
||||
_, found, skip, err := Read("checks/x/known")
|
||||
if found || err != nil || len(skip) < 12 || skip[:12] != "NOT COMPARED" {
|
||||
t.Fatalf("found %v, skip %q, err %v; wanted NOT COMPARED", found, skip, err)
|
||||
_, found, err := Read("checks/x/known")
|
||||
if found || err == nil || !strings.Contains(err.Error(), "NOT COMPARED") {
|
||||
t.Fatalf("found %v, err %v; wanted a failure saying NOT COMPARED", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACloneBesideWithoutTheCatalogueIsSkippedAndSaidNotMade(t *testing.T) {
|
||||
t.Setenv(BesideEnv, t.TempDir()) // the siblings are there, the catalogue's own base is not (the checker before R4g)
|
||||
_, found, skip, err := Read("checks/x/known")
|
||||
if found || err != nil || len(skip) < 12 || skip[:12] != "NOT COMPARED" || !strings.Contains(skip, "not beside") {
|
||||
t.Fatalf("found %v, skip %q, err %v; wanted NOT COMPARED naming the missing copy", found, skip, err)
|
||||
func TestACloneBesideWithoutTheCatalogueFailsAndSaysNotMade(t *testing.T) {
|
||||
t.Setenv(BesideEnv, t.TempDir()) // the siblings are there, the catalogue's own base is not
|
||||
_, found, err := Read("checks/x/known")
|
||||
if found || err == nil || !strings.Contains(err.Error(), "NOT COMPARED") || !strings.Contains(err.Error(), "not beside") {
|
||||
t.Fatalf("found %v, err %v; wanted a failure naming the missing copy", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -589,7 +589,11 @@ if (gitea) {
|
||||
};
|
||||
// **The forge's events first, the polls as the safety net** (novox/hq issue 729): while verified deliveries arrive the
|
||||
// polls slow to every five minutes and announce only what no event said (Pace); otherwise they keep 30 s.
|
||||
await listenForForge(client);
|
||||
// **Not awaited** (novox/hq issue 1726): the receiver's set-up asks the forge's admin API, and the runtime kills a bundle
|
||||
// whose import has not ended in its 10 s handshake and never starts it again; a forge slow to answer while every bundle
|
||||
// of the node starts left gitea_commit_status, gitea_merge_pull_request and gitea_events served nowhere. Until it is
|
||||
// listening, Pace keeps the polls at 30 s, so nothing is missed meanwhile; listenForForge says its own failure.
|
||||
void listenForForge(client);
|
||||
// Main takes no push and only the tools' account merges, in every repository, kept and not only when a token is
|
||||
// minted: at the start and every ten minutes after, so a repository made later gets it too (issue 948, ADR 0398 §2).
|
||||
// Only while the setting keep-main-rules is true (mainrules.ts); false, the default, leaves every repository's rule as set.
|
||||
|
||||
@@ -109,3 +109,53 @@ test("a bundle whose read of the checker's owed is granted late writes the lates
|
||||
assert.match(got.statuses[0], /"state":"success".*pass at 4/);
|
||||
assert.match(err, /mesh-delivery-checker\.owed could not be bound/, "the refusal is said");
|
||||
});
|
||||
|
||||
// **The bundle's import does not wait on the forge** (novox/hq issue 1726, 2026-10-24 12:37). The webhook receiver's set-up
|
||||
// asks the forge's admin API for its system webhook; awaited at the top of index.ts, a forge slow to answer (busy while
|
||||
// every bundle of the node starts) held the import past node-tools' 10 s handshake, the bundle was killed and never
|
||||
// started again, and gitea_commit_status, gitea_merge_pull_request and gitea_events were served nowhere: the forge
|
||||
// bridge's calls found no responders and no new pull request was announced. Here the forge stand-in takes every
|
||||
// request and never answers: the import resolves at once all the same, with the bundle's tools registered.
|
||||
test("importing the bundle resolves, its tools registered, while the forge never answers the webhook's set-up", async () => {
|
||||
const index = pathToFileURL(join(here, "..", "dist", "index.js")).href;
|
||||
const script = `
|
||||
const http = await import("node:http");
|
||||
const fs = await import("node:fs");
|
||||
const os = await import("node:os");
|
||||
const path = await import("node:path");
|
||||
const forge = http.createServer(() => {});
|
||||
await new Promise((r) => forge.listen(0, "127.0.0.1", r));
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "gitea-1726-"));
|
||||
fs.writeFileSync(path.join(dir, "admin.password"), "pw");
|
||||
process.env.MESH_GITEA_URL = "http://127.0.0.1:" + forge.address().port;
|
||||
process.env.MESH_GITEA_STATE_DIR = dir;
|
||||
process.env.MESH_GITEA_ADMIN_PASSWORD_FILE = path.join(dir, "admin.password");
|
||||
process.env.MESH_GITEA_HOOK_PORT = "0";
|
||||
process.env.MESH_GITEA_HOOK_HOST = "127.0.0.1";
|
||||
const { useBroker } = await import("@novox/mesh-sdk/messaging");
|
||||
useBroker(() => ({
|
||||
subscribe: async () => () => {}, publish: async () => {}, request: async () => { throw new Error("no responders"); },
|
||||
handle: async () => () => {}, close: async () => {},
|
||||
}));
|
||||
await import(${JSON.stringify(index)});
|
||||
const { collectTools } = await import("@novox/mesh-sdk/tools");
|
||||
console.log(JSON.stringify({ imported: true, tools: collectTools().flatMap((r) => r.tools.map((t) => t.name)) }));
|
||||
process.exit(0);`;
|
||||
const env = { ...process.env, MESH_GITEA_TOKEN: "x", MESH_GITEA_ADMIN_USER: "admin", MESH_GITEA_HOOK_URL: "http://127.0.0.1:1" };
|
||||
for (const k of ["MESH_SERVED_MODULE", "MESH_GITEA_ADMIN_PASSWORD", "MESH_GITEA_CONFIG_FILE"]) delete env[k];
|
||||
const child = spawn(process.execPath, ["--input-type=module", "-e", script], { cwd: join(here, ".."), env, stdio: ["ignore", "pipe", "pipe"] });
|
||||
let out = "";
|
||||
let err = "";
|
||||
child.stdout.on("data", (d) => (out += d));
|
||||
child.stderr.on("data", (d) => (err += d));
|
||||
const ended = new Promise<void>((r) => child.on("exit", () => r()));
|
||||
const timer = setTimeout(() => child.kill("SIGKILL"), 3000);
|
||||
await ended;
|
||||
clearTimeout(timer);
|
||||
const line = out.split("\n").find((l) => l.startsWith("{"));
|
||||
assert.ok(line, `the import did not resolve within 3 s while the forge did not answer; stdout: ${out.slice(0, 400)} stderr: ${err.slice(0, 600)}`);
|
||||
const got = JSON.parse(line!) as { tools: string[] };
|
||||
for (const t of ["gitea_events", "gitea_commit_status", "gitea_merge_pull_request"]) {
|
||||
assert.ok(got.tools.includes(t), `${t} is registered: ${got.tools}`);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -102,7 +102,11 @@ integration level, the postgres major and the nats-server version; and, when the
|
||||
workspace (`beside`), each one's repository and commit, which the answer echoes as `beside` (the checker keeps a verdict only
|
||||
for the siblings it was judged with). The text's form is `DigestVersion` 3. The Go version is not in it: the toolchain enters a
|
||||
fingerprint once, as the version the build nodes agree on (ADR 0477 §3), and the standard library's packages are that
|
||||
version's. A package with no tests at the level has no digest and is named in `without_tests`. A directory that vendors
|
||||
version's. A package with no tests at the level has no digest and is named in `without_tests`. A catalogue module's packages also hash the module's non-Go files (the nearest ancestor directory holding `module.json`: its manifest, README, `files/` and the rest, so `../../module.json` is covered; digest form 5). A package whose directory holds a file named `reads-outside` (any content; say what the tests read) is never kept (issue 1935): its tests read
|
||||
files outside it, which a digest from `go list` cannot name, so its digest is a fresh random one on every call, the ask finds no kept
|
||||
verdict for it and runs it, and the answer names it in `reads_outside` and the module's log says why. The checker still writes a
|
||||
verdict under that digest, which nothing ever reads again. The marker is the plain way to say it until each verdict keeps the files
|
||||
its tests opened (Go's test log). A directory that vendors
|
||||
is listed and tested from its vendor directory (`-mod=vendor`).
|
||||
|
||||
## A change to this module is judged by its candidate
|
||||
|
||||
@@ -100,6 +100,9 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep
|
||||
if err := m.room(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if a.Compose != nil {
|
||||
return m.compose(ctx, ask.Fetch, a)
|
||||
}
|
||||
return buildstep.RunAsk(ctx, ask, a)
|
||||
}}
|
||||
}
|
||||
|
||||
@@ -116,7 +116,7 @@ func TestTheIntegrationLevelDigestsUnderBothTags(t *testing.T) {
|
||||
t.Errorf("the integration digest listed %q", s.command)
|
||||
}
|
||||
}
|
||||
if DigestVersion != "4" {
|
||||
if DigestVersion != "5" {
|
||||
t.Errorf("the digest's form changed in kind and is still version %s", DigestVersion)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
package main
|
||||
|
||||
// **A compose ask runs mesh-sdk's composer at a commit** (novox/hq ADR 0482 clause 4, issues 1621 and 1622): the
|
||||
// assigner asks it, to compose a group with its mesh-sdk member's head, and to read the group's manifests with the
|
||||
// mesh-sdk the running controller and node-engine pin (ADR 0482 clause 5, issue 1623). The module fetches the commit
|
||||
// (its fetch step: the forge key and the network, none of the change's code), builds the composer package in the Go
|
||||
// module the ask names, in a run step shut off with nothing fetched, and keeps the program under its key
|
||||
// (delivery.ComposerKey), so each commit is built once on a node. It fills the request's manifests from the delivery
|
||||
// indexer's manifests state, by digest, and runs the program shut off: no network, none of the account, the request a
|
||||
// file of a workspace of its own. The answer is the composer's verdict whole (delivery.ComposeAnswer); the module
|
||||
// judges nothing of it.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ManifestsState is the delivery indexer's manifests by digest, which a compose ask's request names (the module reads
|
||||
// it, module.json reads).
|
||||
const ManifestsState = delivery.IndexerModule + ".manifests"
|
||||
|
||||
// composerLocks hold one mutex per composer key, so a composer two asks need at once is built once.
|
||||
var composerLocks sync.Map
|
||||
|
||||
// composerPath is where the composer of key is kept: the cache directory, beside Go's own caches.
|
||||
func (m *Module) composerPath(key string) string {
|
||||
return filepath.Join(m.Cache, "composers", key, "mesh-compose")
|
||||
}
|
||||
|
||||
// compose runs a compose ask: the composer of its commit, built once, over its request with the manifests filled.
|
||||
func (m *Module) compose(ctx context.Context, fetch buildstep.StepFunc, a delivery.StepAsk) (any, error) {
|
||||
job := *a.Compose
|
||||
repository, _ := job.Fetch["repository"].(string)
|
||||
commit, _ := job.Fetch["commit"].(string)
|
||||
installed, err := m.Probe(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key := delivery.ComposerKey(repository, commit, job.Dir, job.Package, delivery.ToolchainUsed{Version: installed, Recipe: m.Recipe.Version()})
|
||||
exe, built, err := m.composerOf(ctx, fetch, job, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req, err := composer.Decode(job.Request)
|
||||
if err != nil {
|
||||
return nil, buildstep.Refuse("the compose ask %s: %v", a.ID, err)
|
||||
}
|
||||
if err := composer.Fill(&req, m.manifestOf); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
verdict, err := m.runComposer(ctx, exe, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return delivery.ComposeAnswer{Built: built, Key: key, Verdict: verdict}, nil
|
||||
}
|
||||
|
||||
// manifestOf is the manifest the indexer keeps under digest, as its repository holds it.
|
||||
func (m *Module) manifestOf(digest string) ([]byte, error) {
|
||||
if m.ReadManifest == nil {
|
||||
return nil, errors.New("this module reads no manifests state")
|
||||
}
|
||||
key, err := delivery.ManifestsKey(digest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := m.ReadManifest(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if raw == nil {
|
||||
return nil, fmt.Errorf("the indexer keeps no manifest of digest %s", digest)
|
||||
}
|
||||
man, err := delivery.DecodeManifest(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []byte(man.JSON), nil
|
||||
}
|
||||
|
||||
// composerOf is the composer kept under key, or, when none is, the one built now from the fetched commit (built true).
|
||||
func (m *Module) composerOf(ctx context.Context, fetch buildstep.StepFunc, job delivery.ComposeJob, key string) (string, bool, error) {
|
||||
exe := m.composerPath(key)
|
||||
mu, _ := composerLocks.LoadOrStore(exe, &sync.Mutex{})
|
||||
mu.(*sync.Mutex).Lock()
|
||||
defer mu.(*sync.Mutex).Unlock()
|
||||
if st, err := os.Stat(exe); err == nil && st.Mode().IsRegular() {
|
||||
return exe, false, nil
|
||||
}
|
||||
if fetch == nil {
|
||||
return "", false, buildstep.Refuse("this build module fetches nothing")
|
||||
}
|
||||
out, err := fetch(ctx, job.Fetch)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
raw, err := json.Marshal(out)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
var f struct {
|
||||
Workspace string `json:"workspace"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &f); err != nil || f.Workspace == "" {
|
||||
return "", false, fmt.Errorf("the fetch answered no workspace: %s", raw)
|
||||
}
|
||||
defer m.releaseSaid(f.Workspace)
|
||||
done := m.spaces().Hold(f.Workspace)
|
||||
defer done()
|
||||
dir := filepath.Join(m.Work, f.Workspace, "tree", job.Dir)
|
||||
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err != nil {
|
||||
return "", false, fmt.Errorf("%s at %s holds no go.mod in %s, so no composer can be built there", job.Fetch["repository"], job.Fetch["commit"], job.Dir)
|
||||
}
|
||||
into := filepath.Join(m.Work, f.Workspace, "composer", "mesh-compose")
|
||||
if err := os.MkdirAll(filepath.Dir(into), 0o700); err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
run, err := m.stepRunner(f.Workspace, false)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
env := append(m.goEnvFor(dir), "GOPROXY=off", "GOTOOLCHAIN=local", "CGO_ENABLED=0")
|
||||
r := run(ctx, env, "go", "-C", dir, "build", "-trimpath", "-o", into, job.Package)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
// The words Go says when the module holds no such package (a mesh-sdk from before the composer) are kept whole,
|
||||
// so the asker tells that apart from a fault.
|
||||
return "", false, fmt.Errorf("the composer %s does not build at %s: %s", job.Package, job.Fetch["commit"], compileError(r.Err+"\n"+r.Stderr+"\n"+r.Stdout))
|
||||
}
|
||||
if err := copyExecutable(into, exe); err != nil {
|
||||
return "", false, fmt.Errorf("the composer could not be kept: %w", err)
|
||||
}
|
||||
return exe, true, nil
|
||||
}
|
||||
|
||||
// copyExecutable copies a built program into place through a file beside it, so a half-written program is never kept.
|
||||
func copyExecutable(from, to string) error {
|
||||
if err := os.MkdirAll(filepath.Dir(to), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
src, err := os.Open(from)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer src.Close()
|
||||
tmp := to + ".new"
|
||||
dst, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o700)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(dst, src); err != nil {
|
||||
dst.Close()
|
||||
return err
|
||||
}
|
||||
if err := dst.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, to)
|
||||
}
|
||||
|
||||
// releaseSaid removes a workspace once its steps ended, saying a failure: the sweep removes it later.
|
||||
func (m *Module) releaseSaid(workspace string) {
|
||||
if err := m.spaces().Release(workspace); err != nil && m.Say != nil {
|
||||
m.Say("the compose ask's workspace %s: %v", workspace, err)
|
||||
}
|
||||
}
|
||||
|
||||
// runComposer runs the composer over req in a workspace of its own, shut off, and answers its verdict: the last line of
|
||||
// its standard output, a composer.Verdict of the version this module reads.
|
||||
func (m *Module) runComposer(ctx context.Context, exe string, req composer.Request) (json.RawMessage, error) {
|
||||
id := buildstep.NewID()
|
||||
done := m.spaces().Hold(id)
|
||||
defer m.releaseSaid(id)
|
||||
defer done()
|
||||
ws := filepath.Join(m.Work, id)
|
||||
if err := os.MkdirAll(ws, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
file := filepath.Join(ws, "request.json")
|
||||
if err := os.WriteFile(file, raw, 0o600); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
run := m.Run
|
||||
if !m.Inside {
|
||||
iso := m.isolation(id, false)
|
||||
iso.ReadOnly = append(iso.ReadOnly, exe)
|
||||
run = buildstep.Isolated(m.Run, iso)
|
||||
}
|
||||
r := run(ctx, nil, exe, "-request", file)
|
||||
line := strings.TrimSpace(lastLine(r.Stdout))
|
||||
var v composer.Verdict
|
||||
if r.Status != 0 || r.Err != "" || json.Unmarshal([]byte(line), &v) != nil || v.V != composer.Version {
|
||||
return nil, fmt.Errorf("the composer answered no verdict (status %d %s): %s", r.Status, r.Err, firstLine(r.Stderr+"\n"+r.Stdout))
|
||||
}
|
||||
return json.RawMessage(line), nil
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
)
|
||||
|
||||
// sdkPin is the mesh-sdk version this module's go.mod requires: the composer the test builds is that version's.
|
||||
func sdkPin(t *testing.T) string {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile("../../go.mod")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, l := range strings.Split(string(raw), "\n") {
|
||||
if f := strings.Fields(l); len(f) >= 2 && f[0] == "git.novox.be/novox/mesh-sdk/go" {
|
||||
return f[1]
|
||||
}
|
||||
}
|
||||
t.Fatal("go.mod requires no mesh-sdk")
|
||||
return ""
|
||||
}
|
||||
|
||||
// goEnvOf is one value of this machine's go env.
|
||||
func goEnvOf(t *testing.T, name string) string {
|
||||
t.Helper()
|
||||
out, err := exec.Command("go", "env", name).Output()
|
||||
if err != nil {
|
||||
t.Fatalf("go env %s: %v", name, err)
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
// 1622/AC1, AC3 against the real bus: a compose ask put on the Go build seat's check stream is pulled by the module
|
||||
// started whole on that bus, which fetches the commit (here a tree of a module requiring mesh-sdk at this module's pin,
|
||||
// as a reader repository is), builds mesh-sdk's composer there with nothing fetched, fills the request's manifests from
|
||||
// the indexer's state and runs the composer; the step key <node>.<ask id> passes with the composer's verdict. A second
|
||||
// ask of the same commit runs the kept composer: fetched and built once.
|
||||
func TestAComposeAskIsBuiltOnceAndAnsweredOnTheBus(t *testing.T) {
|
||||
nc, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
if err := buildstep.DeclareAskWorkers(ctx, js, delivery.AskGo, 2*time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = js.DeleteStream(context.Background(), "SEAT_NODE_BUILD_GO") })
|
||||
|
||||
states := newStates()
|
||||
m := moduleWith(states, "go1.26.0")
|
||||
m.Work, m.Cache, m.Inside = t.TempDir(), t.TempDir(), true
|
||||
m.Run = StepRunner(t.TempDir())
|
||||
m.GoModCache = goEnvOf(t, "GOMODCACHE") // the module cache this module's own build filled, read and not fetched
|
||||
run, gocache := m.Run, goEnvOf(t, "GOCACHE") // this machine's build cache, so the composer's packages compile once
|
||||
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
return run(ctx, append(env, "GOCACHE="+gocache), name, args...)
|
||||
}
|
||||
read, digests := indexerHolds(t, `{"module":"demo","version":"1"}`)
|
||||
m.ReadManifest = read
|
||||
|
||||
gosum, err := os.ReadFile("../../go.sum")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fetches := 0
|
||||
steps := fakeSteps(new([]string))
|
||||
for i := range steps {
|
||||
if steps[i].Name == "build_go_fetch" {
|
||||
steps[i].Run = func(ctx context.Context, args map[string]any) (any, error) {
|
||||
fetches++
|
||||
ws := buildstep.NewID()
|
||||
tree := filepath.Join(m.Work, ws, "tree")
|
||||
if err := os.MkdirAll(tree, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mod := "module reader\n\ngo 1.26.0\n\nrequire git.novox.be/novox/mesh-sdk/go " + sdkPin(t) + "\n"
|
||||
if err := os.WriteFile(filepath.Join(tree, "go.mod"), []byte(mod), 0o600); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"workspace": ws}, os.WriteFile(filepath.Join(tree, "go.sum"), gosum, 0o600)
|
||||
}
|
||||
}
|
||||
}
|
||||
p := m.Puller(steps, &buildstep.JetStreamAsks{JS: js, Language: delivery.AskGo})
|
||||
p.PullWait = 200 * time.Millisecond
|
||||
go p.Pull(ctx)
|
||||
|
||||
ask := func() delivery.Step {
|
||||
a := composeAskOf(t, digests[0])
|
||||
a.Compose.Fetch["repository"], a.Compose.Dir = "novox/mesh-controller", delivery.RootPath
|
||||
raw, err := delivery.EncodeStepAsk(a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
subject, _ := delivery.AskSubject(delivery.AskGo, delivery.AskCheck)
|
||||
if _, err := js.Publish(ctx, subject, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.AskStepKey("ace", a.ID)
|
||||
for deadline := time.Now().Add(90 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
states.mu.Lock()
|
||||
v := states.steps[key]
|
||||
states.mu.Unlock()
|
||||
if len(v) > 0 && v[len(v)-1].State != delivery.StepRunning {
|
||||
return v[len(v)-1]
|
||||
}
|
||||
}
|
||||
t.Fatalf("the compose ask %s never ended", a.ID)
|
||||
return delivery.Step{}
|
||||
}
|
||||
for n, wantBuilt := range []bool{true, false} {
|
||||
step := ask()
|
||||
if step.State != delivery.StepPassed {
|
||||
t.Fatalf("ask %d: %s: %s", n+1, step.State, step.Reason)
|
||||
}
|
||||
var ans delivery.ComposeAnswer
|
||||
var v composer.Verdict
|
||||
if err := json.Unmarshal(step.Result, &ans); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal(ans.Verdict, &v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ans.Built != wantBuilt || !strings.HasPrefix(ans.Key, "composer-") || len(v.Cases) != 1 || !v.Cases[0].Manifests["demo"].Accepted ||
|
||||
!strings.Contains(v.SDK, sdkPin(t)) {
|
||||
t.Fatalf("ask %d: answer built %v key %s, verdict %+v; want built %v and demo accepted by mesh-sdk %s", n+1, ans.Built, ans.Key, v, wantBuilt, sdkPin(t))
|
||||
}
|
||||
}
|
||||
if fetches != 1 {
|
||||
t.Errorf("two asks of one commit fetched %d times, want once", fetches)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const composeCommit = "4444444444444444444444444444444444444444"
|
||||
|
||||
// indexerHolds is the delivery indexer's manifests state holding each body under its digest, as the indexer writes it.
|
||||
func indexerHolds(t *testing.T, bodies ...string) (func(string) (json.RawMessage, error), []string) {
|
||||
t.Helper()
|
||||
kept, digests := map[string]json.RawMessage{}, []string{}
|
||||
for _, b := range bodies {
|
||||
d := delivery.ManifestDigest([]byte(b))
|
||||
key, _ := delivery.ManifestsKey(d)
|
||||
raw, err := delivery.EncodeManifest(delivery.Manifest{Module: "demo", JSON: b})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept[key], digests = raw, append(digests, d)
|
||||
}
|
||||
return func(key string) (json.RawMessage, error) { return kept[key], nil }, digests
|
||||
}
|
||||
|
||||
func composeAskOf(t *testing.T, digest string) delivery.StepAsk {
|
||||
t.Helper()
|
||||
req, _ := json.Marshal(composer.Request{V: composer.Version, Cases: []composer.Case{{Name: "change",
|
||||
Manifests: map[string]composer.Manifest{"demo": {Digest: digest}}}}})
|
||||
return delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: "mesh-delivery-assigner",
|
||||
At: time.Now(), Compose: &delivery.ComposeJob{Fetch: map[string]any{"repository": "novox/mesh-sdk", "commit": composeCommit},
|
||||
Dir: "go", Package: composer.Package, Request: req}}
|
||||
}
|
||||
|
||||
// composeModule is a module with work and cache directories of its own, running its commands as they are (Inside: a
|
||||
// unit test has no bubblewrap to shut them off in).
|
||||
func composeModule(t *testing.T) *Module {
|
||||
t.Helper()
|
||||
m := moduleWith(newStates(), "go1.26.0")
|
||||
m.Work, m.Cache, m.Inside, m.Run = t.TempDir(), t.TempDir(), true, StepRunner(t.TempDir())
|
||||
return m
|
||||
}
|
||||
|
||||
// 1622/AC3: a composer kept under the ask's key is run again, nothing fetched or built: the answer says it was not
|
||||
// built, names the key, and carries the composer's verdict whole; the request it read had its manifests filled from the
|
||||
// indexer's state.
|
||||
func TestAKeptComposerIsRunWithTheManifestsFilled(t *testing.T) {
|
||||
m := composeModule(t)
|
||||
read, digests := indexerHolds(t, `{"module":"demo","version":"1"}`)
|
||||
m.ReadManifest = read
|
||||
a := composeAskOf(t, digests[0])
|
||||
key := delivery.ComposerKey("novox/mesh-sdk", composeCommit, "go", composer.Package, delivery.ToolchainUsed{Version: "go1.26.0", Recipe: m.Recipe.Version()})
|
||||
exe := m.composerPath(key)
|
||||
if err := os.MkdirAll(filepath.Dir(exe), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A composer that answers which manifests its request held, filled or not, as a verdict.
|
||||
script := "#!/bin/sh\n" +
|
||||
`if grep -q '"digest"' "$2"; then echo '{"v":1,"sdk":"stand-in","cases":[{"name":"unfilled","manifests":{}}]}'; ` +
|
||||
`else echo '{"v":1,"sdk":"stand-in","cases":[{"name":"change","manifests":{"demo":{"accepted":true}}}]}'; fi` + "\n"
|
||||
if err := os.WriteFile(exe, []byte(script), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fetched := 0
|
||||
fetch := func(context.Context, map[string]any) (any, error) { fetched++; return nil, nil }
|
||||
out, err := m.compose(context.Background(), fetch, a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ans := out.(delivery.ComposeAnswer)
|
||||
var v composer.Verdict
|
||||
if err := json.Unmarshal(ans.Verdict, &v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ans.Built || ans.Key != key || fetched != 0 || len(v.Cases) != 1 || v.Cases[0].Name != "change" || !v.Cases[0].Manifests["demo"].Accepted {
|
||||
t.Fatalf("answer = %+v (fetched %d), verdict %+v", ans, fetched, v)
|
||||
}
|
||||
if entries, _ := os.ReadDir(m.Work); len(entries) != 0 {
|
||||
t.Errorf("the run's workspace was left: %v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
// 1622: a manifest the indexer does not keep fails the ask, naming its digest; a fetched commit with no go.mod where
|
||||
// the ask builds fails it naming the directory, and its workspace is released.
|
||||
func TestAComposeAskFailsLoudly(t *testing.T) {
|
||||
m := composeModule(t)
|
||||
m.ReadManifest, _ = indexerHolds(t)
|
||||
a := composeAskOf(t, "feedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeed")
|
||||
key := delivery.ComposerKey("novox/mesh-sdk", composeCommit, "go", composer.Package, delivery.ToolchainUsed{Version: "go1.26.0", Recipe: m.Recipe.Version()})
|
||||
os.MkdirAll(filepath.Dir(m.composerPath(key)), 0o700)
|
||||
os.WriteFile(m.composerPath(key), []byte("#!/bin/sh\nexit 0\n"), 0o700)
|
||||
if _, err := m.compose(context.Background(), nil, a); err == nil || !strings.Contains(err.Error(), "feedfeed") {
|
||||
t.Errorf("an unkept manifest: %v", err)
|
||||
}
|
||||
m2 := composeModule(t)
|
||||
fetch := func(context.Context, map[string]any) (any, error) {
|
||||
if err := os.MkdirAll(filepath.Join(m2.Work, "ws0000000000a", "tree", "go"), 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"workspace": "ws0000000000a"}, nil
|
||||
}
|
||||
if _, err := m2.compose(context.Background(), fetch, composeAskOf(t, "x")); err == nil || !strings.Contains(err.Error(), "no go.mod in go") {
|
||||
t.Errorf("a commit with no go.mod: %v", err)
|
||||
}
|
||||
if entries, _ := os.ReadDir(m2.Work); len(entries) != 0 {
|
||||
t.Errorf("the fetched workspace was left: %v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
// 1622: the module reads the delivery indexer's manifests, which a compose ask names by digest.
|
||||
func TestTheModuleReadsTheIndexersManifests(t *testing.T) {
|
||||
if m := readManifest(t); !slices.Contains(m.Reads, ManifestsState) {
|
||||
t.Errorf("reads = %v, lacks %s", m.Reads, ManifestsState)
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
@@ -24,12 +25,20 @@ import (
|
||||
// 2: no Go version in the text; the toolchain enters a fingerprint as the agreed version (ADR 0477 §3). 3: the siblings'
|
||||
// commits (the lead's decision L5) and the integration level's tags integration,isolation (the checker's agreement).
|
||||
// 4: the siblings' commits enter only the digest of a package whose tests reach a file naming MESH_CHECK_BESIDE (ADR 0536 §1).
|
||||
const DigestVersion = "4"
|
||||
// 5: a catalogue module's non-Go files enter the digest of its packages (issue 1935).
|
||||
const DigestVersion = "5"
|
||||
|
||||
// besideVariable is the name a file bears when it reads where the siblings lie; a package whose tests reach such a file
|
||||
// depends on the siblings' commits.
|
||||
const besideVariable = "MESH_CHECK_BESIDE"
|
||||
|
||||
// ReadsOutsideMarker is the name of a file in a package's directory that says its tests read files outside it (a
|
||||
// repository's documents, a sibling directory): such a package is never kept (issue 1935). The digest hashes only what
|
||||
// the Go toolchain lists, so it cannot name what a test opens elsewhere; a marker file, beside the Go files where a
|
||||
// reader of the package sees it, is the plain way to say so until the checker keeps the files a test opened (the Go
|
||||
// test log) with each verdict.
|
||||
const ReadsOutsideMarker = "reads-outside"
|
||||
|
||||
// integrationTags are the build tags of the integration level, its run and its digest alike: integration, and isolation
|
||||
// for the bubblewrap proofs (mesh-sdk's and the build modules'), which a build node can run.
|
||||
var integrationTags = []string{"integration", "isolation"}
|
||||
@@ -89,6 +98,9 @@ type DigestResult struct {
|
||||
// only when they are the siblings it gave the fetch.
|
||||
Beside []string `json:"beside,omitempty"`
|
||||
WithoutTests []string `json:"without_tests,omitempty"`
|
||||
// ReadsOutside names the packages that carry the marker: each has a digest that no kept verdict can have (a fresh
|
||||
// random one), so the check runs it every time.
|
||||
ReadsOutside []string `json:"reads_outside,omitempty"`
|
||||
// JudgedBy is the program that ran the digest (ADR 0535 §5): "installed", or "candidate <merge commit>" under the
|
||||
// runner workspace.
|
||||
JudgedBy string `json:"judged_by,omitempty"`
|
||||
@@ -199,6 +211,12 @@ func (m *Module) digestHere(ctx context.Context, workspace, level string) (Diges
|
||||
}
|
||||
}
|
||||
sort.Strings(res.WithoutTests)
|
||||
sort.Strings(res.ReadsOutside)
|
||||
for _, p := range res.ReadsOutside {
|
||||
if m.Say != nil {
|
||||
m.Say("the %s level of %s always runs: its directory carries %s, so its tests read files outside it that no digest names", level, p, ReadsOutsideMarker)
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
@@ -256,6 +274,17 @@ func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, he
|
||||
if testMain == nil {
|
||||
return fmt.Errorf("go list gave no test binary for %s", path)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root.Dir, ReadsOutsideMarker)); err == nil {
|
||||
// The digest of a package that reads outside itself is one nothing can have kept: the key is a fresh random
|
||||
// hex, so the ask finds no kept verdict and runs it, and what the checker writes under it is never read again.
|
||||
var nonce [32]byte
|
||||
if _, err := rand.Read(nonce[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
res.Packages[path] = hex.EncodeToString(nonce[:])
|
||||
res.ReadsOutside = append(res.ReadsOutside, path)
|
||||
continue
|
||||
}
|
||||
d, err := packageDigest(root, testMain, pkgs, tree, header, beside)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", path, err)
|
||||
@@ -303,6 +332,15 @@ func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree s
|
||||
}
|
||||
}
|
||||
}
|
||||
// A catalogue module's tests read its manifest, its README and its files from the package's directory (../../module.json):
|
||||
// every non-Go file of the module's directory is an input (issue 1935). The module is the nearest ancestor with a module.json.
|
||||
if modDir := enclosingModule(tree, root.Dir); modDir != "" {
|
||||
more, err := moduleFileLines(tree, root, modDir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
lines = append(lines, more...)
|
||||
}
|
||||
data := filepath.Join(root.Dir, "testdata")
|
||||
if _, err := os.Stat(data); err == nil {
|
||||
err := filepath.WalkDir(data, func(path string, e fs.DirEntry, err error) error {
|
||||
@@ -328,6 +366,41 @@ func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree s
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// enclosingModule is the nearest ancestor of dir, inside tree, that holds a module.json, or "" when there is none.
|
||||
func enclosingModule(tree, dir string) string {
|
||||
tree = filepath.Clean(tree)
|
||||
for d := filepath.Clean(dir); d != tree && strings.HasPrefix(d, tree+string(filepath.Separator)); d = filepath.Dir(d) {
|
||||
if st, err := os.Stat(filepath.Join(d, "module.json")); err == nil && !st.IsDir() {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// moduleFileLines are the content lines of every file of a module's directory that is not Go source (Go source is the
|
||||
// packages' own, listed by go list), skipping .git and node_modules.
|
||||
func moduleFileLines(tree string, p *goPackage, modDir string) ([]string, error) {
|
||||
var lines []string
|
||||
err := filepath.WalkDir(modDir, func(path string, e fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if e.IsDir() {
|
||||
if n := e.Name(); n == ".git" || n == "node_modules" {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if strings.HasSuffix(path, ".go") {
|
||||
return nil
|
||||
}
|
||||
l, err := contentLine(tree, p, path)
|
||||
lines = append(lines, l)
|
||||
return err
|
||||
})
|
||||
return lines, err
|
||||
}
|
||||
|
||||
func fileLine(tree string, p *goPackage, name string) (string, error) {
|
||||
return contentLine(tree, p, filepath.Join(p.Dir, name))
|
||||
}
|
||||
|
||||
@@ -189,3 +189,71 @@ func TestAModuleOutsideTheTreeCountsByVersionAndChecksum(t *testing.T) {
|
||||
t.Errorf("a new version left the digest as it was")
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1935: a package whose directory carries the reads-outside marker is never kept: its digest is one no kept verdict
|
||||
// can have (it differs on every call, though nothing changed), and the answer names it. A package without the marker
|
||||
// keeps its digest.
|
||||
func TestAPackageWithTheReadsOutsideMarkerIsNeverKept(t *testing.T) {
|
||||
t.Parallel()
|
||||
m, tree := digestFixture(t)
|
||||
plain := digests(t, m, "unit")
|
||||
if len(plain.ReadsOutside) != 0 {
|
||||
t.Fatalf("a package carries the marker in the fixture: %v", plain.ReadsOutside)
|
||||
}
|
||||
edit(t, tree, "c/"+ReadsOutsideMarker, "the tests read ../docs\n")
|
||||
first, second := digests(t, m, "unit"), digests(t, m, "unit")
|
||||
c := "fixture/digest/c"
|
||||
if len(first.Packages[c]) != 64 || first.Packages[c] == second.Packages[c] || first.Packages[c] == plain.Packages[c] {
|
||||
t.Errorf("c carries the marker and its digest is stable: %s, %s, before %s", first.Packages[c], second.Packages[c], plain.Packages[c])
|
||||
}
|
||||
if strings.Join(first.ReadsOutside, ",") != c {
|
||||
t.Errorf("the answer does not name the package that reads outside: %v", first.ReadsOutside)
|
||||
}
|
||||
a := "fixture/digest/a"
|
||||
if first.Packages[a] != plain.Packages[a] || first.Packages[a] != second.Packages[a] {
|
||||
t.Errorf("a has no marker and its digest moved: %s, %s, before %s", first.Packages[a], second.Packages[a], plain.Packages[a])
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1935: a catalogue module's packages read ../../module.json and the module's files: a change to them changes the
|
||||
// digest of the module's package, and a change to another module's files does not.
|
||||
func TestACatalogueModulesFilesChangeTheDigestOfItsPackagesOnly(t *testing.T) {
|
||||
t.Parallel()
|
||||
m, tree := digestFixture(t)
|
||||
for _, mod := range []string{"mod1", "mod2"} {
|
||||
dir := filepath.Join(tree, mod, "cmd", "p")
|
||||
if err := os.MkdirAll(filepath.Join(tree, mod, "files"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
edit(t, tree, mod+"/module.json", `{"module":"`+mod+`"}`)
|
||||
edit(t, tree, mod+"/README.md", "# "+mod+"\n")
|
||||
edit(t, tree, mod+"/files/f.env", "A=1\n")
|
||||
edit(t, tree, mod+"/cmd/p/p.go", "package p\n")
|
||||
edit(t, tree, mod+"/cmd/p/p_test.go", "package p\n\nimport (\n\t\"os\"\n\t\"testing\"\n)\n\nfunc TestReads(t *testing.T) {\n\tif _, err := os.ReadFile(\"../../module.json\"); err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n")
|
||||
}
|
||||
p1, p2 := "fixture/digest/mod1/cmd/p", "fixture/digest/mod2/cmd/p"
|
||||
before := digests(t, m, "unit")
|
||||
if len(before.Packages[p1]) != 64 || len(before.Packages[p2]) != 64 {
|
||||
t.Fatalf("no digest for the module packages: %+v", before.Packages)
|
||||
}
|
||||
if before.Packages[p1] == before.Packages[p2] {
|
||||
t.Errorf("two modules, one digest")
|
||||
}
|
||||
for _, file := range []string{"mod1/module.json", "mod1/README.md", "mod1/files/f.env"} {
|
||||
edit(t, tree, file, "changed "+file+"\n")
|
||||
after := digests(t, m, "unit")
|
||||
if after.Packages[p1] == before.Packages[p1] {
|
||||
t.Errorf("a change to %s did not change the module's digest", file)
|
||||
}
|
||||
if after.Packages[p2] != before.Packages[p2] {
|
||||
t.Errorf("a change to %s changed the other module's digest", file)
|
||||
}
|
||||
before = after
|
||||
}
|
||||
if got := digests(t, m, "unit"); got.Packages["fixture/digest/c"] == "" {
|
||||
t.Errorf("a package outside any module lost its digest")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -288,10 +288,14 @@ func inTestdata(path string) bool {
|
||||
// git, through the account's rewrite of the forge's address to ssh), and nothing that asks a question. For a directory
|
||||
// that vendors, see goEnvFor.
|
||||
func (m *Module) goEnv() []string {
|
||||
modcache := filepath.Join(m.Cache, "gomod")
|
||||
if m.GoModCache != "" {
|
||||
modcache = m.GoModCache
|
||||
}
|
||||
return []string{
|
||||
"GOTOOLCHAIN=local",
|
||||
"GOFLAGS=-mod=mod",
|
||||
"GOMODCACHE=" + filepath.Join(m.Cache, "gomod"),
|
||||
"GOMODCACHE=" + modcache,
|
||||
"GOCACHE=" + filepath.Join(m.Cache, "gobuild"),
|
||||
"GOPRIVATE=git.novox.be",
|
||||
"GIT_TERMINAL_PROMPT=0",
|
||||
|
||||
@@ -54,6 +54,7 @@ func main() {
|
||||
m.RegistryFile = m.Home + "/registry"
|
||||
m.Settings = m.Home + "/settings.json"
|
||||
m.ReadKept = readOf(stdio.State(KeptState)).Get
|
||||
m.ReadManifest = readOf(stdio.State(ManifestsState)).Get
|
||||
m.Probe = ProbeGo(m.Home, m.Cache)
|
||||
m.Bubblewrap = ProbeBubblewrap
|
||||
// Outside a launch by the runtime — a build, a check — there is no node and no state to write.
|
||||
|
||||
@@ -82,6 +82,10 @@ type Module struct {
|
||||
Settings string
|
||||
Capacity func() int
|
||||
ReadKept func(key string) (json.RawMessage, error)
|
||||
// ReadManifest reads a key of the delivery indexer's manifests state (ManifestsState), which a compose ask's request
|
||||
// names its manifests by (compose.go); GoModCache, when set, is the Go module cache in place of Cache's (a test).
|
||||
ReadManifest func(key string) (json.RawMessage, error)
|
||||
GoModCache string
|
||||
|
||||
spacesOnce sync.Once
|
||||
workspaces *buildstep.Workspaces
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
These tests read other catalogue modules' files, outside this module (issue 1935): never kept.
|
||||
@@ -3,7 +3,7 @@ module meshbuildgo
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024112544-bc0c25e51052
|
||||
github.com/distribution/distribution/v3 v3.1.2
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93 h1:FAiFY2KBKEoZSH49OZTt7/zRBIQxyl5g/KhYrpggbN4=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024112544-bc0c25e51052 h1:iPXa9g9QtywV2DHdtTFF4jHae5aKgCv+lCYdilRbp5k=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024112544-bc0c25e51052/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
|
||||
github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
|
||||
@@ -20,7 +20,8 @@
|
||||
}
|
||||
],
|
||||
"reads": [
|
||||
"mesh-delivery-checker.kept"
|
||||
"mesh-delivery-checker.kept",
|
||||
"mesh-delivery-indexer.manifests"
|
||||
],
|
||||
"config": {
|
||||
"capacity": {
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
These tests read other catalogue modules' files, outside this module (issue 1935): never kept.
|
||||
@@ -0,0 +1 @@
|
||||
These tests read other catalogue modules' files, outside this module (issue 1935): never kept.
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read other catalogue modules' files, outside this module (issue 1935): never kept.
|
||||
@@ -0,0 +1 @@
|
||||
These tests read other catalogue modules' files, outside this module (issue 1935): never kept.
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read other catalogue modules' files, outside this module (issue 1935): never kept.
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
Reference in New Issue
Block a user