From 9c5b7e14e1016f6ddcd51d31d0cfe0c5278044be Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:18:03 +0200 Subject: [PATCH] mesh-build-go: fold the siblings' commits into every digest and echo them, and test and digest the integration level under integration,isolation (lead decision L5, the checker's agreement; issue 1309) The fetch writes beside.json naming the siblings it placed; the digest adds a line per sibling to every package's text and answers them as beside, so a core repository's verdict is kept only for the siblings it was judged with. DigestVersion 3, once for both changes: the integration level's tags are integration,isolation, so the bubblewrap proofs run in every check on a build node. --- modules/mesh-build-go/README.md | 12 ++- .../cmd/mesh-build-go/beside_digest_test.go | 87 +++++++++++++++++++ .../mesh-build-go/cmd/mesh-build-go/digest.go | 28 ++++-- .../cmd/mesh-build-go/integration.go | 5 +- .../mesh-build-go/integration_level_test.go | 2 +- .../cmd/mesh-build-go/preview.go | 39 +++++++++ 6 files changed, 160 insertions(+), 13 deletions(-) create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go diff --git a/modules/mesh-build-go/README.md b/modules/mesh-build-go/README.md index 4ffe7ea8..666840a2 100644 --- a/modules/mesh-build-go/README.md +++ b/modules/mesh-build-go/README.md @@ -65,14 +65,16 @@ integration-tagged test runs the real go on the fixtures. ## The integration level -`build_go_test` with `level: integration` runs `go test -tags integration` against a store and a bus started for the -run alone (novox/hq ADR 0470). The store is a postgres of the installed package: `initdb` and `pg_ctl` into +`build_go_test` with `level: integration` runs `go test -tags integration,isolation` (isolation: the bubblewrap proofs, +which a build node can run) against a store and a bus started for the run alone (novox/hq ADR 0470). The store is a postgres of the installed package: `initdb` and `pg_ctl` into `//run-/pg`, as the module's account, listening on a unix socket there and on no TCP address, with durability off; the tests get `MESH_TEST_POSTGRES` (the socket directory percent-encoded in the URL, because the controller's tests cut the URL at its last slash). The bus is a nats-server from the Go library at the version this module's `go.mod` names, run by this same binary started as `mesh-build-go bus ` (`bus.go`: JetStream on, a loopback port of its own, its address on the first line, stopped when its input closes); the tests get `MESH_TEST_NATS`. -Both run inside the step, are stopped, and the run's directory is removed whatever the tests did. +Both run inside the step, are stopped, and the run's directory is removed whatever the tests did. The store, the bus and +the tests run together in one bubblewrap (ADR 0448, issue 1309): this binary again as `mesh-build-go level `, which +prints the level's result as one line of JSON, so the tests reach the level's own store and bus and nothing of the machine. Setup (compiling the tests, the store, the bus) is judged within 30 s and the tests' running time (the wall time of `go test`) within 60 s, apart; a level over either says `over_bound` and names the bound, with the slowest tests for the running @@ -87,7 +89,9 @@ is said until a source exists. one digest of everything those tests read (ADR 0445 §5): the files, by content and under their path in the tree, of the package and every package its test binary reaches (`go list -test -deps`) that lies in the module's own tree; the version and checksum of every module outside it; the package's own `testdata/` files; and the level, the build tags and, at the -integration level, the postgres major and the nats-server version. The Go version is not in it: the toolchain enters a +integration level, the postgres major and the nats-server version; and, when the fetch placed siblings beside the +workspace (`beside`), each one's repository and commit, which the answer echoes as `beside` (the checker keeps a verdict only +for the siblings it was judged with). The text's form is `DigestVersion` 3. The Go version is not in it: the toolchain enters a fingerprint once, as the version the build nodes agree on (ADR 0477 §3), and the standard library's packages are that version's. A package with no tests at the level has no digest and is named in `without_tests`. A directory that vendors is listed and tested from its vendor directory (`-mod=vendor`). diff --git a/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go new file mode 100644 index 00000000..9f14bd9c --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go @@ -0,0 +1,87 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The lead's decision L5 (the checker keeps a verdict of a core repository): the fetch writes the siblings it placed +// beside the workspace, and the digest folds each one's repository and commit into every package's digest and echoes +// them, so a verdict is kept only for the siblings it was judged with. +func TestTheSiblingsCommitsAreInEveryDigestAndEchoed(t *testing.T) { + t.Parallel() + m, _ := digestFixture(t) + alone := digests(t, m, "unit") + if len(alone.Beside) != 0 { + t.Errorf("a workspace without siblings echoes %v", alone.Beside) + } + write := func(placed []Sibling) { + raw, _ := json.Marshal(placed) + if err := os.WriteFile(filepath.Join(m.Work, "ws1", besideRecord), raw, 0o600); err != nil { + t.Fatal(err) + } + } + other := strings.Repeat("b", 40) + write([]Sibling{{Repository: "novox/mesh-sdk", Commit: commitSHA, Dir: "mesh-sdk"}, {Repository: "novox/mesh-catalog", Commit: commitSHA, Dir: "mesh-catalog"}}) + with := digests(t, m, "unit") + want := []string{"novox/mesh-catalog " + commitSHA, "novox/mesh-sdk " + commitSHA} + if !slices.Equal(with.Beside, want) { + t.Errorf("echoed %v, want %v", with.Beside, want) + } + write([]Sibling{{Repository: "novox/mesh-sdk", Commit: other, Dir: "mesh-sdk"}, {Repository: "novox/mesh-catalog", Commit: commitSHA, Dir: "mesh-catalog"}}) + moved := digests(t, m, "unit") + for p, d := range alone.Packages { + if with.Packages[p] == d { + t.Errorf("%s: the siblings left the digest as it was without them", p) + } + if moved.Packages[p] == with.Packages[p] { + t.Errorf("%s: a sibling's new commit left the digest as it was", p) + } + } +} + +// The fetch writes the siblings it placed, which the digest reads. +func TestTheFetchRecordsTheSiblingsItPlaced(t *testing.T) { + f := &fakeStep{gomods: "go.mod\n"} + m, states := fetchModule(t, f) + args := map[string]any{"repository": "novox/mesh-controller", "commit": commitSHA, "beside": []any{ + map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA}}} + if _, err := fetchTool(t, m)(args); err != nil { + t.Fatal(err) + } + st := states.waitEnd(t) + var res FetchResult + json.Unmarshal(st.Result, &res) + if st.State != delivery.StepPassed { + t.Fatalf("the step ended %s: %s", st.State, st.Reason) + } + raw, err := os.ReadFile(filepath.Join(m.Work, res.Workspace, besideRecord)) + var placed []Sibling + if err != nil || json.Unmarshal(raw, &placed) != nil || len(placed) != 1 || placed[0].Repository != "novox/mesh-sdk" || placed[0].Commit != commitSHA { + t.Errorf("the record is %s (%v)", raw, err) + } +} + +// The checker's agreement: the integration level runs and digests under the tags integration and isolation, so the +// bubblewrap proofs tagged isolation run in every check on a build node. +func TestTheIntegrationLevelDigestsUnderBothTags(t *testing.T) { + t.Parallel() + m, _ := digestFixture(t) + var seen []shutOff + m.Run = throughBubblewrap(t, m.Run, &seen) + digests(t, m, LevelIntegration) + for _, s := range seen { + if !strings.Contains(s.command, " -tags integration,isolation ") { + t.Errorf("the integration digest listed %q", s.command) + } + } + if DigestVersion != "3" { + t.Errorf("the digest's form changed in kind and is still version %s", DigestVersion) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest.go b/modules/mesh-build-go/cmd/mesh-build-go/digest.go index 14fd432e..55412464 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/digest.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest.go @@ -20,7 +20,13 @@ import ( ) // DigestVersion is the form of the digest's text: changed whenever what the digest hashes changes in kind. -const DigestVersion = "2" // 2: no Go version in the text; the toolchain enters a fingerprint as the agreed version (ADR 0477 §3) +// 2: no Go version in the text; the toolchain enters a fingerprint as the agreed version (ADR 0477 §3). 3: the siblings' +// commits (the lead's decision L5) and the integration level's tags integration,isolation (the checker's agreement). +const DigestVersion = "3" + +// integrationTags are the build tags of the integration level, its run and its digest alike: integration, and isolation +// for the bubblewrap proofs (mesh-sdk's and the build modules'), which a build node can run. +var integrationTags = []string{"integration", "isolation"} // goPackage is the part of `go list -json` a digest reads. type goPackage struct { @@ -71,9 +77,12 @@ func (p *goPackage) hasTests() bool { return len(p.TestGoFiles)+len(p.XTestGoFil // DigestResult is what a digest step answers: one digest per package that has tests at the level, and the packages // that have none. type DigestResult struct { - Level string `json:"level"` - Packages map[string]string `json:"packages"` - WithoutTests []string `json:"without_tests,omitempty"` + Level string `json:"level"` + Packages map[string]string `json:"packages"` + // Beside echoes the siblings folded into every digest, " " sorted: the checker keeps a verdict + // only when they are the siblings it gave the fetch. + Beside []string `json:"beside,omitempty"` + WithoutTests []string `json:"without_tests,omitempty"` } func (m *Module) digestTool() StepTool { @@ -113,18 +122,25 @@ func (m *Module) digest(ctx context.Context, args map[string]any) (any, error) { if _, err := m.checkPostgres(ctx); err != nil { return nil, err } - tags = []string{"integration"} + tags = integrationTags header = append(header, "postgres "+strconv.Itoa(m.Recipe.PostgresMajor), "nats-server "+m.Recipe.NatsServer) default: return nil, fmt.Errorf("the level %q is neither unit nor integration", level) } header = append(header, "tags "+strings.Join(tags, ",")) + beside, err := m.besideLines(workspace) + if err != nil { + return nil, err + } + for _, b := range beside { + header = append(header, "beside "+b) + } dirs, err := m.goModDirs(ctx, tree) if err != nil { return nil, err } - res := DigestResult{Level: level, Packages: map[string]string{}} + res := DigestResult{Level: level, Packages: map[string]string{}, Beside: beside} for _, d := range dirs { pkgs, err := m.listPackages(ctx, workspace, tree, d, tags) if err != nil { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/integration.go b/modules/mesh-build-go/cmd/mesh-build-go/integration.go index 5cc496b0..03de39ab 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/integration.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/integration.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "strconv" + "strings" "time" "git.novox.be/novox/mesh-sdk/go/buildstep" @@ -75,13 +76,13 @@ func (m *Module) integrationHere(ctx context.Context, workspace, tree string, pa // Compiling counts as setup: the tests are built once, here, with the addresses a TestMain may read, and the run // below finds them built. Whether they compile is for the run to say; its failure is not read here. for _, d := range dirs { - m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), env...), "go", "-C", filepath.Join(tree, d), "test", "-tags", "integration", "-run", "^$", "-count=1", "./...") + m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), env...), "go", "-C", filepath.Join(tree, d), "test", "-tags", strings.Join(integrationTags, ","), "-run", "^$", "-count=1", "./...") } setup := m.clock().Sub(began) return m.runLevel(ctx, tree, levelOpts{ workspace: workspace, run: pattern, - level: LevelIntegration, tags: []string{"integration"}, packages: packages, bounds: m.integrationBounds(), + level: LevelIntegration, tags: integrationTags, packages: packages, bounds: m.integrationBounds(), env: env, setup: setup, wall: true, store: "postgres " + strconv.Itoa(major), bus: "nats-server " + m.Recipe.NatsServer, diff --git a/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go b/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go index e0050cab..9d2aafa1 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go @@ -180,7 +180,7 @@ func TestTheStoreListensOnASocketUnderTheRunAndNowhereElse(t *testing.T) { test = c } } - if !strings.Contains(test.line, "-tags integration") { + if !strings.Contains(test.line, "-tags integration,isolation") { // the bubblewrap proofs run too (the checker's agreement) t.Errorf("the run does not ask for the tag: %s", test.line) } env := map[string]string{} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/preview.go b/modules/mesh-build-go/cmd/mesh-build-go/preview.go index 32fbd3af..87ae1066 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/preview.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/preview.go @@ -6,6 +6,7 @@ package main import ( "context" + "encoding/json" "fmt" "io/fs" "os" @@ -104,9 +105,47 @@ func (m *Module) placeSiblings(ctx context.Context, root string, siblings []sibl placed = append(placed, Sibling{Repository: s.repository, Commit: s.commit, Dir: name}) } sort.Slice(placed, func(i, j int) bool { return placed[i].Dir < placed[j].Dir }) + if len(placed) > 0 { + // What the digest folds in (the lead's decision L5): the siblings this workspace's tests read, by commit. + raw, err := json.Marshal(placed) + if err != nil { + return nil, err + } + if err := os.WriteFile(filepath.Join(root, besideRecord), raw, 0o600); err != nil { + return nil, err + } + } return placed, nil } +// besideRecord is the file in a workspace's root naming the siblings the fetch placed beside it. +const besideRecord = "beside.json" + +// besideLines are the siblings a workspace's fetch placed, as the digest folds them in and echoes them: " +// ", sorted; none when it placed none. +func (m *Module) besideLines(workspace string) ([]string, error) { + raw, err := os.ReadFile(filepath.Join(m.Work, workspace, besideRecord)) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, err + } + var placed []Sibling + if err := json.Unmarshal(raw, &placed); err != nil { + return nil, fmt.Errorf("the workspace's record of its siblings: %w", err) + } + var out []string + for _, s := range placed { + if !repositoryOK.MatchString(s.Repository) || !commitOK.MatchString(s.Commit) { + return nil, fmt.Errorf("the workspace's record of its siblings names %q at %q", s.Repository, s.Commit) + } + out = append(out, s.Repository+" "+s.Commit) + } + sort.Strings(out) + return out, nil +} + func firstSymlink(dir string) string { var found string filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {