From a00df90c477b492798464c0b6950718d0192079b Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 21:02:54 +0200 Subject: [PATCH 01/19] Keep a passing package's verdict once under its digest, so a second check can take it (issue 1287) --- modules/mesh-delivery-checker/go.mod | 22 ++++ modules/mesh-delivery-checker/go.sum | 29 +++++ .../internal/check/doc.go | 6 ++ .../internal/check/fakes_test.go | 62 +++++++++++ .../internal/check/kept.go | 90 ++++++++++++++++ .../internal/check/kept_test.go | 102 ++++++++++++++++++ .../internal/check/results.go | 95 ++++++++++++++++ .../internal/check/store.go | 19 ++++ 8 files changed, 425 insertions(+) create mode 100644 modules/mesh-delivery-checker/go.mod create mode 100644 modules/mesh-delivery-checker/go.sum create mode 100644 modules/mesh-delivery-checker/internal/check/doc.go create mode 100644 modules/mesh-delivery-checker/internal/check/fakes_test.go create mode 100644 modules/mesh-delivery-checker/internal/check/kept.go create mode 100644 modules/mesh-delivery-checker/internal/check/kept_test.go create mode 100644 modules/mesh-delivery-checker/internal/check/results.go create mode 100644 modules/mesh-delivery-checker/internal/check/store.go diff --git a/modules/mesh-delivery-checker/go.mod b/modules/mesh-delivery-checker/go.mod new file mode 100644 index 0000000..5035651 --- /dev/null +++ b/modules/mesh-delivery-checker/go.mod @@ -0,0 +1,22 @@ +module mesh-delivery-checker + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5 + github.com/nats-io/nats-server/v2 v2.11.17 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-delivery-checker/go.sum b/modules/mesh-delivery-checker/go.sum new file mode 100644 index 0000000..6f440f0 --- /dev/null +++ b/modules/mesh-delivery-checker/go.sum @@ -0,0 +1,29 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865 h1:VLir9NZbyi1LdH1YwWDIVRuL+lDC+AYlm6FZPlvmlm4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5 h1:1BiPKRPTYpgex8rSHWT9az0BteOafGWRDzd/Ba6GAcA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-delivery-checker/internal/check/doc.go b/modules/mesh-delivery-checker/internal/check/doc.go new file mode 100644 index 0000000..781347e --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/doc.go @@ -0,0 +1,6 @@ +// Package check is the checker's logic (novox/hq ADR 0484, ADR 0453 clause 4, ADR 0447, ADR 0444), apart from the bus: the +// claim of a verdict key, the judging of a pull request's head merged onto its trunk through the build modules' fetch, +// digest and test steps, the kept verdicts, the line of the status and the act the forge bridge performs. The bus is +// reached through the small interfaces here, so each part is tested with fakes. The checker runs dark: nothing reads its +// answers to merge or deliver (ADR 0453 clause 4). +package check diff --git a/modules/mesh-delivery-checker/internal/check/fakes_test.go b/modules/mesh-delivery-checker/internal/check/fakes_test.go new file mode 100644 index 0000000..2965b72 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/fakes_test.go @@ -0,0 +1,62 @@ +package check + +import ( + "context" + "sort" + "sync" +) + +// memKV is a bucket in memory with the revisions a real one gives. +type memKV struct { + mu sync.Mutex + vals map[string][]byte + revs map[string]uint64 + next uint64 + // creates counts the creates that succeeded, by key. + creates map[string]int +} + +func newMemKV() *memKV { + return &memKV{vals: map[string][]byte{}, revs: map[string]uint64{}, creates: map[string]int{}} +} + +func (m *memKV) Get(_ context.Context, key string) ([]byte, uint64, bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + v, ok := m.vals[key] + return v, m.revs[key], ok, nil +} + +func (m *memKV) Create(_ context.Context, key string, value []byte) (uint64, error) { + m.mu.Lock() + defer m.mu.Unlock() + if _, ok := m.vals[key]; ok { + return 0, ErrConflict + } + m.next++ + m.vals[key], m.revs[key] = append([]byte(nil), value...), m.next + m.creates[key]++ + return m.next, nil +} + +func (m *memKV) Update(_ context.Context, key string, value []byte, revision uint64) (uint64, error) { + m.mu.Lock() + defer m.mu.Unlock() + if _, ok := m.vals[key]; !ok || m.revs[key] != revision { + return 0, ErrConflict + } + m.next++ + m.vals[key], m.revs[key] = append([]byte(nil), value...), m.next + return m.next, nil +} + +func (m *memKV) Keys(context.Context) ([]string, error) { + m.mu.Lock() + defer m.mu.Unlock() + keys := make([]string, 0, len(m.vals)) + for k := range m.vals { + keys = append(keys, k) + } + sort.Strings(keys) + return keys, nil +} diff --git a/modules/mesh-delivery-checker/internal/check/kept.go b/modules/mesh-delivery-checker/internal/check/kept.go new file mode 100644 index 0000000..25d92ad --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/kept.go @@ -0,0 +1,90 @@ +package check + +import ( + "context" + "errors" + "fmt" + "sort" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Kept is the checker's bucket of kept verdicts (ADR 0444, ADR 0484 clause 13): a passing verdict per package and level, +// under the digest the language's build module named for it. A key is written once, by a create that fails when it +// exists, from the result the checker read from the test step and never from anything the code under test wrote; nothing +// changes it afterwards. +type Kept struct{ Store KV } + +// Lookup is the kept verdict of a level under a digest, or none. +func (k Kept) Lookup(ctx context.Context, level, digest string) (delivery.Kept, bool, error) { + key, err := delivery.KeptKey(level, digest) + if err != nil { + return delivery.Kept{}, false, err + } + value, _, found, err := k.Store.Get(ctx, key) + if err != nil || !found { + return delivery.Kept{}, false, err + } + v, err := delivery.DecodeKept(value) + if err != nil { + return delivery.Kept{}, false, fmt.Errorf("%s holds what is not a kept verdict: %w", key, err) + } + return v, true, nil +} + +// Keep writes v under level and digest by a create. It answers false, and writes nothing, when the key exists: the first +// value stands. +func (k Kept) Keep(ctx context.Context, level, digest string, v delivery.Kept) (bool, error) { + key, err := delivery.KeptKey(level, digest) + if err != nil { + return false, err + } + v.Level = level + data, err := delivery.EncodeKept(v) + if err != nil { + return false, err + } + if _, err := k.Store.Create(ctx, key, data); err != nil { + if errors.Is(err, ErrConflict) { + return false, nil + } + return false, fmt.Errorf("%s: %w", key, err) + } + return true, nil +} + +// KeepPasses keeps, for the packages of a test result that passed with tests, a verdict under the digest the build module +// named for the package at the level. A package that failed, has no tests or has no digest is never kept (ADR 0444 clause +// 1): it runs every check. It answers the packages it kept now. +func (k Kept) KeepPasses(ctx context.Context, repository, level, toolchain string, res Tested, digests map[string]string, at time.Time) ([]string, error) { + var kept []string + for _, p := range res.Packages { + digest, ok := digests[p.Package] + if !ok || !p.Passed || p.NoTests || p.Tests == 0 { + continue + } + if failedIn(res, p.Package) { + continue + } + made, err := k.Keep(ctx, level, digest, delivery.Kept{State: delivery.VerdictPass, Repository: repository, Package: p.Package, + Level: level, Tests: p.Tests, Took: int64(p.Elapsed * 1000), Toolchain: toolchain, At: at}) + if err != nil { + return kept, err + } + if made { + kept = append(kept, p.Package) + } + } + sort.Strings(kept) + return kept, nil +} + +func failedIn(res Tested, pkg string) bool { + for _, f := range res.Failed { + if f.Package == pkg { + return true + } + } + return false +} diff --git a/modules/mesh-delivery-checker/internal/check/kept_test.go b/modules/mesh-delivery-checker/internal/check/kept_test.go new file mode 100644 index 0000000..c2ff33b --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/kept_test.go @@ -0,0 +1,102 @@ +package check + +import ( + "context" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +var at0 = time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC) + +const ( + digA = "aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11" + digB = "bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22" +) + +func keptValue(pkg string, tests int) delivery.Kept { + return delivery.Kept{State: delivery.VerdictPass, Repository: "novox/mesh-delivery", Package: pkg, Tests: tests, Took: 120, Toolchain: "go1.26.0", At: at0} +} + +func TestAPassIsKeptByACreateUnderItsLevelAndDigest(t *testing.T) { // 1287/AC1 + ctx, st := context.Background(), newMemKV() + k := Kept{Store: st} + made, err := k.Keep(ctx, delivery.LevelUnit, digA, keptValue("m/a", 3)) + if err != nil || !made { + t.Fatalf("Keep = %v, %v, want made", made, err) + } + key, _ := delivery.KeptKey(delivery.LevelUnit, digA) + if st.creates[key] != 1 { + t.Fatalf("the key %s was created %d times, want once", key, st.creates[key]) + } + got, found, err := k.Lookup(ctx, delivery.LevelUnit, digA) + if err != nil || !found || got.Package != "m/a" || got.Tests != 3 || got.Level != delivery.LevelUnit || got.Toolchain != "go1.26.0" { + t.Fatalf("Lookup = %+v, %v, %v", got, found, err) + } +} + +func TestASecondKeepUnderTheSameDigestIsRefusedAndTheFirstStands(t *testing.T) { // 1287/AC2 + ctx, st := context.Background(), newMemKV() + k := Kept{Store: st} + if made, err := k.Keep(ctx, delivery.LevelUnit, digA, keptValue("m/a", 3)); err != nil || !made { + t.Fatal(made, err) + } + made, err := k.Keep(ctx, delivery.LevelUnit, digA, keptValue("m/other", 9)) + if err != nil || made { + t.Fatalf("the second Keep = %v, %v, want refused without an error", made, err) + } + got, _, _ := k.Lookup(ctx, delivery.LevelUnit, digA) + if got.Package != "m/a" || got.Tests != 3 { + t.Errorf("the first value did not stand: %+v", got) + } +} + +func TestOnlyPassingPackagesWithADigestAreKept(t *testing.T) { // 1287/AC3, AC4 + ctx, st := context.Background(), newMemKV() + k := Kept{Store: st} + res := Tested{Level: "unit", Passed: false, Packages: []GoPackage{ + {Package: "m/a", Passed: true, Tests: 4, Elapsed: 0.2}, + {Package: "m/b", Passed: false, Tests: 2}, + {Package: "m/c", Passed: true, Tests: 1}, // no digest named + {Package: "m/d", Passed: true, NoTests: true}, // no tests + }, Failed: []Failure{{Package: "m/b", Test: "TestB"}}} + kept, err := k.KeepPasses(ctx, "novox/mesh-delivery", "unit", "go1.26.0", res, map[string]string{"m/a": digA, "m/b": digB, "m/d": digB}, at0) + if err != nil { + t.Fatal(err) + } + if strings.Join(kept, ",") != "m/a" { + t.Fatalf("kept %v, want only m/a", kept) + } + if keys, _ := st.Keys(ctx); len(keys) != 1 { + t.Errorf("the bucket holds %v, want one key", keys) + } +} + +func TestAKeptValueOfAnotherDigestIsNotRead(t *testing.T) { // 1287/AC7 + ctx, st := context.Background(), newMemKV() + k := Kept{Store: st} + if _, err := k.Keep(ctx, delivery.LevelUnit, digA, keptValue("m/a", 3)); err != nil { + t.Fatal(err) + } + // A toolchain change changes the digest, so the new digest finds nothing and the old key is not deleted. + if _, found, _ := k.Lookup(ctx, delivery.LevelUnit, digB); found { + t.Error("a digest that was not kept was found") + } + if _, found, _ := k.Lookup(ctx, delivery.LevelIntegration, digA); found { + t.Error("a kept unit verdict was found at the integration level") + } + if keys, _ := st.Keys(ctx); len(keys) != 1 { + t.Errorf("the old key was removed: %v", keys) + } +} + +func TestAKeptVerdictThatIsNotAPassIsRefused(t *testing.T) { + k := Kept{Store: newMemKV()} + v := keptValue("m/a", 1) + v.State = delivery.VerdictFail + if _, err := k.Keep(context.Background(), delivery.LevelUnit, digA, v); err == nil { + t.Error("a failure was kept") + } +} diff --git a/modules/mesh-delivery-checker/internal/check/results.go b/modules/mesh-delivery-checker/internal/check/results.go new file mode 100644 index 0000000..5583592 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/results.go @@ -0,0 +1,95 @@ +package check + +import ( + "encoding/json" + "strings" +) + +// The shapes of what the build modules' steps answer, as their code writes them (mesh-catalog modules/mesh-build-go +// cmd/mesh-build-go levels.go TestResult, digest.go DigestResult, fetch.go FetchResult; modules/mesh-build-node +// cmd/mesh-build-node check.go TestResult, fetch.go FetchResult) with the fields the fetch step adds for a merge (ADR 0484 +// clause 3). Only the fields the checker reads are named; the rest is ignored. + +// MergeTree is what a fetch that merged says of one module directory: the tree sha, the digest of its manifest and the +// hashes of its lock files by name. A bare string is the tree alone. +type MergeTree struct { + Tree string `json:"tree"` + Manifest string `json:"manifest,omitempty"` + Locks map[string]string `json:"locks,omitempty"` +} + +// UnmarshalJSON takes a string as the tree alone, or an object. +func (t *MergeTree) UnmarshalJSON(data []byte) error { + if s := strings.TrimSpace(string(data)); strings.HasPrefix(s, `"`) { + return json.Unmarshal(data, &t.Tree) + } + type plain MergeTree + return json.Unmarshal(data, (*plain)(t)) +} + +// Fetched is the answer of a build module's fetch step. +type Fetched struct { + Workspace string `json:"workspace"` + Commit string `json:"commit"` + Modules []string `json:"modules"` + // What a fetch with a base adds: the trunk commit, the merge commit, the files in conflict (a conflict is an answer), + // the paths the merge changes and the trees of the module directories. + Base string `json:"base,omitempty"` + Merge string `json:"merge,omitempty"` + Conflicts []string `json:"conflicts,omitempty"` + Changed []string `json:"changed,omitempty"` + Trees map[string]MergeTree `json:"trees,omitempty"` +} + +// GoPackage is one package of a Go test result. +type GoPackage struct { + Dir string `json:"dir"` + Package string `json:"package"` + Passed bool `json:"passed"` + Elapsed float64 `json:"elapsed"` + Tests int `json:"tests"` + NoTests bool `json:"no_tests,omitempty"` +} + +// Failure is a test that failed, or a package that failed with no test to name. +type Failure struct { + Package string `json:"package"` + Test string `json:"test,omitempty"` + Output string `json:"output,omitempty"` +} + +// TestTime is one test and the seconds it took. +type TestTime struct { + Package string `json:"package"` + Test string `json:"test"` + Elapsed float64 `json:"elapsed"` +} + +// Tested is the answer of a test step of either build module (the Go one names packages and counts tests; the Node one +// names the package directory as Module and counts none). +type Tested struct { + Level string `json:"level"` + Module string `json:"module,omitempty"` + Passed bool `json:"passed"` + Why string `json:"why,omitempty"` + NoTests bool `json:"no_tests,omitempty"` + Tests int `json:"tests"` + Skipped int `json:"skipped"` + Packages []GoPackage `json:"packages,omitempty"` + Elapsed float64 `json:"elapsed"` + Setup float64 `json:"setup,omitempty"` + Bound float64 `json:"bound"` + SetupBound float64 `json:"setup_bound,omitempty"` + Slowest []TestTime `json:"slowest,omitempty"` + Failed []Failure `json:"failed,omitempty"` + OverBound bool `json:"over_bound"` + OverWhich string `json:"over_which,omitempty"` +} + +// Digested is the answer of the Go build module's digest step: one digest per package that has tests at the level, by import +// path. +type Digested struct { + Level string `json:"level"` + Packages map[string]string `json:"packages"` + WithoutTests []string `json:"without_tests,omitempty"` +} diff --git a/modules/mesh-delivery-checker/internal/check/store.go b/modules/mesh-delivery-checker/internal/check/store.go new file mode 100644 index 0000000..4df618e --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/store.go @@ -0,0 +1,19 @@ +package check + +import ( + "context" + "errors" +) + +// ErrConflict is what a KV answers when a Create finds the key or an Update finds another revision than the one given. +var ErrConflict = errors.New("the key exists or changed") + +// KV is one of the checker's buckets: read, create and compare-and-set, each write answering the key's new revision. Only +// the checker's credential writes them (ADR 0444 clause 5, ADR 0484 clause 2). +type KV interface { + Get(ctx context.Context, key string) (value []byte, revision uint64, found bool, err error) + Create(ctx context.Context, key string, value []byte) (uint64, error) + Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) + // Keys are every key of the bucket. + Keys(ctx context.Context) ([]string, error) +} From 27072a01b5d4e1c04e3ddcf1d349a361eb0bc75d Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 21:07:14 +0200 Subject: [PATCH 02/19] Add the checker module: it watches the candidates, claims a verdict once, owes the status mesh/check and ends a check past its bound (issue 1678) --- .../cmd/mesh-delivery-checker/bus.go | 159 ++++++ .../cmd/mesh-delivery-checker/judge.go | 35 ++ .../cmd/mesh-delivery-checker/main.go | 314 ++++++++++++ .../cmd/mesh-delivery-checker/settings.go | 29 ++ .../mesh-delivery-checker/settings_test.go | 68 +++ .../cmd/mesh-delivery-checker/toolchains.go | 104 ++++ .../mesh-delivery-checker/files/settings.json | 3 + .../internal/check/case.go | 135 ++++++ .../internal/check/checker.go | 451 ++++++++++++++++++ .../internal/check/checker_test.go | 367 ++++++++++++++ .../internal/check/line.go | 19 + .../mesh-delivery-checker/manifest_test.go | 197 ++++++++ modules/mesh-delivery-checker/module.json | 113 +++++ 13 files changed, 1994 insertions(+) create mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker/bus.go create mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go create mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go create mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker/settings.go create mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker/settings_test.go create mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go create mode 100644 modules/mesh-delivery-checker/files/settings.json create mode 100644 modules/mesh-delivery-checker/internal/check/case.go create mode 100644 modules/mesh-delivery-checker/internal/check/checker.go create mode 100644 modules/mesh-delivery-checker/internal/check/checker_test.go create mode 100644 modules/mesh-delivery-checker/internal/check/line.go create mode 100644 modules/mesh-delivery-checker/manifest_test.go create mode 100644 modules/mesh-delivery-checker/module.json diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/bus.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/bus.go new file mode 100644 index 0000000..e3489c3 --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/bus.go @@ -0,0 +1,159 @@ +package main + +import ( + "context" + "errors" + "fmt" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-checker/internal/check" +) + +// bucketOpener is the part of the bus the watches use. +type bucketOpener interface { + KeyValue(ctx context.Context, bucket string) (jetstream.KeyValue, error) +} + +// kvStore is the builds bucket as the builder's store: create and compare-and-set. +type kvStore struct{ kv jetstream.KeyValue } + +func conflict(err error) error { + if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + return check.ErrConflict + } + return err +} + +func (s kvStore) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) { + e, err := s.kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, 0, false, nil + } + if err != nil { + return nil, 0, false, err + } + return e.Value(), e.Revision(), true, nil +} + +func (s kvStore) Create(ctx context.Context, key string, value []byte) (uint64, error) { + rev, err := s.kv.Create(ctx, key, value) + return rev, conflict(err) +} + +func (s kvStore) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) { + rev, err := s.kv.Update(ctx, key, value, revision) + return rev, conflict(err) +} + +func (s kvStore) Keys(ctx context.Context) ([]string, error) { + keys, err := s.kv.Keys(ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return keys, err +} + +func opOf(e jetstream.KeyValueEntry) bool { return e.Operation() == jetstream.KeyValuePut } + +// settle reads the entries a watcher holds now, up to its mark of the last values, and returns the watcher for the +// changes that follow. +func settle(ctx context.Context, kv jetstream.KeyValue, each func(jetstream.KeyValueEntry)) (jetstream.KeyWatcher, error) { + w, err := kv.WatchAll(ctx) + if err != nil { + return nil, err + } + for { + select { + case <-ctx.Done(): + w.Stop() + return nil, ctx.Err() + case e, ok := <-w.Updates(): + if !ok { + return nil, errors.New("the watch ended") + } + if e == nil { + return w, nil + } + each(e) + } + } +} + +// follow hands each change of a watcher to each until ctx ends; a watch that ends is an error, and the process manager +// starts the program again. +func follow(ctx context.Context, w jetstream.KeyWatcher, each func(jetstream.KeyValueEntry)) error { + defer w.Stop() + for { + select { + case <-ctx.Done(): + return nil + case e, ok := <-w.Updates(): + if !ok { + return errors.New("a watch ended") + } + if e != nil { + each(e) + } + } + } +} + +// stepsWaiter waits for a build module's step by watching its key in the module's steps bucket (ADR 0469 clause 2): the +// watch hands over the value the key holds now, so a step that ended before the watch began is read at once, and every +// change after. No timer: the caller bounds the wait with its context. +type stepsWaiter struct{ js bucketOpener } + +func (w stepsWaiter) Wait(ctx context.Context, module, key string) (delivery.Step, error) { + kv, err := w.js.KeyValue(ctx, delivery.StepsBucket(module)) + if err != nil { + return delivery.Step{}, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err) + } + watch, err := kv.Watch(ctx, key) + if err != nil { + return delivery.Step{}, err + } + defer watch.Stop() + for { + select { + case <-ctx.Done(): + return delivery.Step{}, ctx.Err() + case e, ok := <-watch.Updates(): + if !ok { + return delivery.Step{}, errors.New("the watch of the step ended") + } + if e == nil || !opOf(e) { + continue + } + step, err := delivery.DecodeStep(e.Value()) + if err != nil { + return delivery.Step{}, fmt.Errorf("the step %s is not a step: %w", key, err) + } + if step.State != delivery.StepRunning { + return step, nil + } + } + } +} + +// manifestReader reads manifests. of the indexer and parses the manifest it holds. +type manifestReader struct{ kv jetstream.KeyValue } + +func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue.Manifest, error) { + key, err := delivery.ManifestsKey(digest) + if err != nil { + return catalogue.Manifest{}, err + } + e, err := m.kv.Get(ctx, key) + if err != nil { + return catalogue.Manifest{}, fmt.Errorf("manifests.%s: %w", key, err) + } + kept, err := delivery.DecodeManifest(e.Value()) + if err != nil { + return catalogue.Manifest{}, err + } + return catalogue.ParseManifest([]byte(kept.JSON)) +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go new file mode 100644 index 0000000..4b4a008 --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go @@ -0,0 +1,35 @@ +package main + +import ( + "context" + "errors" + "log" + + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" + + "mesh-delivery-checker/internal/check" +) + +// judgeDeps are the parts the judge of a check is made of. +type judgeDeps struct { + checker *check.Checker + runner buildrun.Runner + tools *buildrun.ToolchainSet + presence *buildrun.PresenceSet + have *buildrun.HaveSet + places *buildrun.Places + trunks *check.Trunks + kept check.KV + previews check.KV + logf func(format string, args ...any) +} + +var _ = log.Printf + +type unbuilt struct{} + +func (unbuilt) Judge(context.Context, *check.Case) (check.Outcome, error) { + return check.Outcome{}, errors.New("the judge of a check is not built yet") +} + +func newJudge(judgeDeps) check.Judge { return unbuilt{} } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go new file mode 100644 index 0000000..6faacfa --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -0,0 +1,314 @@ +// Command mesh-delivery-checker is delivery's checker (novox/hq ADR 0453 clause 4, ADR 0447, ADR 0484, issues 1678 to 1686): for +// each candidate the indexer writes it judges every pull request's head merged onto its trunk, through the build modules' +// fetch, digest and test steps, keeps the passing packages' verdicts, writes the verdict, and owes the forge the status +// `mesh/check`, which the forge bridge sets. It runs dark: nothing reads its answers to merge or deliver (ADR 0453 clause 4). +// It serves no request subject and holds no timer: every reaction is a watch or a bus event (ADR 0437 §8). +package main + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-checker/internal/check" +) + +const ( + // defaultStepBound is how long the checker waits for one step of a build module (a fetch downloads the Go modules, a + // test level runs a repository's tests). + defaultStepBound = 30 * time.Minute + // defaultLink is the base of the address a status points at; the verdict key follows. + defaultLink = "mesh-delivery-view/checks" +) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if err := run(ctx, os.Getenv, os.Stderr); err != nil { + fmt.Fprintln(os.Stderr, "mesh-delivery-checker:", err) + os.Exit(1) + } +} + +// credential is the broker file the controller issues to the module: JSON of the bus address, the certificate +// fingerprint to pin, the principal and its password (as the indexer reads it; issue 1242 moves this into the SDK). +type credential struct { + URL string `json:"url"` + Fingerprint string `json:"fingerprint"` + Node string `json:"node"` + Module string `json:"module"` + User string `json:"user"` + Password string `json:"password"` +} + +func readCredential(path string) (credential, error) { + raw, err := os.ReadFile(path) + if err != nil { + return credential{}, err + } + var c credential + if err := json.Unmarshal(raw, &c); err != nil { + return credential{}, fmt.Errorf("the broker credential is not one: %w", err) + } + if c.URL == "" || c.User == "" { + return credential{}, errors.New("the broker credential names no bus address or user") + } + return c, nil +} + +// pinned accepts exactly the certificate with this SHA-256 and no other. +func pinned(want string) *tls.Config { + return &tls.Config{ + InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter + MinVersion: tls.VersionTLS12, + VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + if len(rawCerts) == 0 { + return errors.New("the bus presented no certificate") + } + sum := sha256.Sum256(rawCerts[0]) + if got := "sha256:" + hex.EncodeToString(sum[:]); got != want { + return fmt.Errorf("the bus presented a certificate this mesh does not know (%.23s…)", got) + } + return nil + }, + } +} + +// durationEnv reads a duration setting from the environment, def when it is not set. +func durationEnv(getenv func(string) string, name string, def time.Duration) (time.Duration, error) { + v := strings.TrimSpace(getenv(name)) + if v == "" { + return def, nil + } + d, err := time.ParseDuration(v) + if err != nil || d <= 0 { + return 0, fmt.Errorf("%s: %q is not a duration of more than 0", name, v) + } + return d, nil +} + +// run is the program: it returns when ctx ends or the bus cannot be used. +func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error { + logger := log.New(logOut, "mesh-delivery-checker: ", log.LstdFlags) + path := strings.TrimSpace(getenv("MESH_BROKER_FILE")) + if path == "" { + return errors.New("MESH_BROKER_FILE is not set, so there is no credential to hear the bus with") + } + cred, err := readCredential(path) + if err != nil { + return err + } + opts := []nats.Option{ + nats.Name(cred.Node + "." + cred.Module), + nats.UserInfo(cred.User, cred.Password), + nats.CustomInboxPrefix("_INBOX." + cred.User), + nats.MaxReconnects(-1), + nats.RetryOnFailedConnect(true), + } + if strings.TrimSpace(cred.Fingerprint) != "" { + opts = append(opts, nats.Secure(pinned(cred.Fingerprint))) + } + conn, err := nats.Connect(cred.URL, opts...) + if err != nil { + return fmt.Errorf("connecting to the bus: %w", err) + } + defer conn.Close() + js, err := jetstream.New(conn) + if err != nil { + return err + } + bucket := func(name string) (jetstream.KeyValue, error) { + kv, err := js.KeyValue(ctx, name) + if err != nil { + return nil, fmt.Errorf("the bucket %s: %w", name, err) + } + return kv, nil + } + buckets := map[string]jetstream.KeyValue{} + for _, name := range []string{delivery.CheckerVerdictsBucket, delivery.CheckerKeptBucket, delivery.CheckerOwedBucket, delivery.PreviewsBucket, + delivery.CandidatesBucket, delivery.ModulesBucket, delivery.ManifestsBucket} { + if buckets[name], err = bucket(name); err != nil { + return err + } + } + // Calls to the build modules go out as this module's own account, naming the machine (mesh-sdk process.CallOn); the + // buckets are read and written on the connection above. + account, err := process.ReadAccount(path) + if err != nil { + return err + } + caller, err := process.Connect(account, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true)) + if err != nil { + return err + } + defer caller.Close() + + stepBound, err := durationEnv(getenv, "MESH_CHECKER_STEP_BOUND", defaultStepBound) + if err != nil { + return err + } + checkBound, err := durationEnv(getenv, "MESH_CHECKER_BOUND", check.DefaultBound) + if err != nil { + return err + } + capacity, err := capacityFrom(strings.TrimSpace(getenv("MESH_CHECKER_SETTINGS")), getenv("MESH_CHECKER_CAPACITY")) + if err != nil { + return fmt.Errorf("the capacity setting: %w", err) + } + link := strings.TrimSpace(getenv("MESH_CHECKER_LINK")) + if link == "" { + link = defaultLink + } + tools := &buildrun.ToolchainSet{} + presence := &buildrun.PresenceSet{} + have := &buildrun.HaveSet{} + trunks := &check.Trunks{} + places := &buildrun.Places{Capacity: capacity} + runner := buildrun.Runner{Caller: caller, Steps: stepsWaiter{js}, Bound: stepBound} + ch := &check.Checker{ + Verdicts: kvStore{buckets[delivery.CheckerVerdictsBucket]}, + Owed: kvStore{buckets[delivery.CheckerOwedBucket]}, + Trunks: trunks, + Bound: checkBound, + Link: link, + Logf: logger.Printf, + } + ch.Judge = newJudge(judgeDeps{ + checker: ch, runner: runner, tools: tools, presence: presence, have: have, places: places, trunks: trunks, + kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]}, + logf: logger.Printf, + }) + + // The toolchains first, so a check that starts from the first candidate finds the nodes. + watch := newToolchainWatch(ctx, js, tools, ch.NodesChanged, logger) + if err := watch.start(); err != nil { + return err + } + // A build module says each time it wrote its toolchains key: its stream, and the checker's durable consumer on it, are + // made by the controller. Where the stream is not there yet the checker says so and is started again with the build + // module. + for _, module := range []string{buildrun.GoBuildModule, buildrun.NodeBuildModule} { + stream := toolchainStreams[module] + cons, err := js.Consumer(ctx, stream, cred.Node+"_"+cred.Module) + if err != nil { + logger.Printf("the consumer %s_%s on %s does not exist yet, so %s's reports are not heard until the checker starts again: %v", cred.Node, cred.Module, stream, module, err) + continue + } + cc, err := cons.Consume(func(m jetstream.Msg) { watch.Reported(m) }, + jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) })) + if err != nil { + return err + } + defer cc.Stop() + } + errs := make(chan error, 6) + // The watchman's presence says which nodes are asleep. Where its bucket is not there yet no node is online, so nothing + // is checked; the checker says so and is started again with the watchman. + switch kv, err := js.KeyValue(ctx, delivery.PresenceBucket); { + case errors.Is(err, jetstream.ErrBucketNotFound): + logger.Printf("the bucket %s does not exist yet: no node counts as online until the checker starts again", delivery.PresenceBucket) + case err != nil: + return fmt.Errorf("the bucket %s: %w", delivery.PresenceBucket, err) + default: + pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) }) + if err != nil { + return err + } + go func() { + errs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) { + if presence.Put(e.Key(), e.Value(), !opOf(e)) { + ch.NodesChanged(ctx) // a node came online or went: a check that waited for one may go on + } + }) + }() + } + // The node-engine's have says whether a node's build module runs and is healthy: a node whose build module is not is not + // chosen. Where its bucket is not there yet no build module counts as healthy, so nothing is checked. + switch kv, err := js.KeyValue(ctx, delivery.HaveBucket); { + case errors.Is(err, jetstream.ErrBucketNotFound): + logger.Printf("the bucket %s does not exist yet: no build module counts as healthy until the checker starts again", delivery.HaveBucket) + case err != nil: + return fmt.Errorf("the bucket %s: %w", delivery.HaveBucket, err) + default: + hw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { have.Put(e.Key(), e.Value(), !opOf(e)) }) + if err != nil { + return err + } + go func() { + errs <- follow(ctx, hw, func(e jetstream.KeyValueEntry) { + if have.Put(e.Key(), e.Value(), !opOf(e)) { + ch.NodesChanged(ctx) + } + }) + }() + } + // The gate's source says the trunk head of each repository, which a pull request is merged onto. Where it is not there + // yet no trunk is known, so nothing is checked. + switch kv, err := js.KeyValue(ctx, delivery.SourceBucket); { + case errors.Is(err, jetstream.ErrBucketNotFound): + logger.Printf("the bucket %s does not exist yet: no trunk is known until the checker starts again", delivery.SourceBucket) + case err != nil: + return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err) + default: + sw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { + if e.Key() == delivery.SourceMainKey { + trunks.Put(e.Value(), !opOf(e)) + } + }) + if err != nil { + return err + } + go func() { + errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) { + if e.Key() == delivery.SourceMainKey && trunks.Put(e.Value(), !opOf(e)) { + ch.Reconsider(ctx) // the trunk moved: a verdict for the old trunk is judged again + } + }) + }() + } + each := func(e jetstream.KeyValueEntry) { + if opOf(e) { + ch.Candidate(ctx, e.Key(), e.Value()) + } else { + ch.Forget(e.Key()) + } + } + w, err := settle(ctx, buckets[delivery.CandidatesBucket], each) + if err != nil { + return err + } + // A key left running for heads no candidate holds is closed; the candidates are all known now. + if err := ch.Sweep(ctx); err != nil { + return err + } + logger.Printf("started on %s", cred.Node) + go func() { errs <- follow(ctx, w, each) }() + select { + case <-ctx.Done(): + ch.Wait() + return nil + case err := <-errs: + return err + case err := <-watch.errs: + return err + } +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/settings.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/settings.go new file mode 100644 index 0000000..d32d39f --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/settings.go @@ -0,0 +1,29 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" +) + +// capacityFrom is the build capacity of each node (ADR 0478 clause 3): the module's setting `capacity`, `=` pairs separated by commas, laid into the settings file the controller composes (issue 1661); empty is one step at a +// time on every node. With no settings file named, fallback is read instead, for a run outside the module's launch. +func capacityFrom(path, fallback string) (map[string]int, error) { + if path == "" { + return buildrun.ParseCapacity(fallback) + } + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("the settings file: %w", err) + } + var s struct { + Capacity string `json:"capacity"` + } + if err := json.Unmarshal(raw, &s); err != nil { + return nil, fmt.Errorf("the settings file %s is not JSON: %w", path, err) + } + return buildrun.ParseCapacity(s.Capacity) +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/settings_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/settings_test.go new file mode 100644 index 0000000..8543fe7 --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/settings_test.go @@ -0,0 +1,68 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" +) + +func settingsFile(t *testing.T, body string) string { + t.Helper() + p := filepath.Join(t.TempDir(), "settings.json") + if err := os.WriteFile(p, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + return p +} + +// ADR 0484 clause 12: the build capacity is the module's setting `capacity`, laid into the settings file: empty is one step at a time everywhere. +func TestCapacityIsReadFromTheSettingsFile(t *testing.T) { + for body, want := range map[string]map[string]int{ + `{"capacity": "shanks=3,ace=2"}`: {"shanks": 3, "ace": 2}, + `{"capacity": ""}`: {}, + `{}`: {}, + } { + got, err := capacityFrom(settingsFile(t, body), "") + if err != nil || len(got) != len(want) || got["shanks"] != want["shanks"] || got["ace"] != want["ace"] { + t.Errorf("%s: capacity %v, %v, want %v", body, got, err, want) + } + } + p := &buildrun.Places{Capacity: mustCapacity(t, settingsFile(t, `{"capacity": ""}`))} + if p.CapacityOf("shanks") != 1 || p.CapacityOf("ace") != 1 { + t.Errorf("an empty setting gives %d and %d, want 1 everywhere", p.CapacityOf("shanks"), p.CapacityOf("ace")) + } +} + +func mustCapacity(t *testing.T, path string) map[string]int { + t.Helper() + c, err := capacityFrom(path, "") + if err != nil { + t.Fatal(err) + } + return c +} + +func TestACapacitySettingThatIsNotOneIsRefusedNamingIt(t *testing.T) { + if _, err := capacityFrom(settingsFile(t, `{"capacity": "shanks=-1"}`), ""); err == nil { + t.Error("shanks=-1 accepted") + } + if c, err := capacityFrom(settingsFile(t, `{"capacity": "shanks=3,ace=2,g14=0,novox=0"}`), ""); err != nil || c["g14"] != 0 || len(c) != 4 { + t.Errorf("the starting value: %v, %v", c, err) + } + if _, err := capacityFrom(settingsFile(t, `not json`), ""); err == nil { + t.Error("a file that is not JSON accepted") + } + if _, err := capacityFrom(filepath.Join(t.TempDir(), "none"), ""); err == nil { + t.Error("a settings file that is not there accepted") + } +} + +// With no settings file named (a run outside the module's launch, as the tests do), the capacity is the environment's. +func TestWithNoSettingsFileTheCapacityIsTheEnvironments(t *testing.T) { + got, err := capacityFrom("", "ace=2") + if err != nil || got["ace"] != 2 { + t.Errorf("capacity %v, %v", got, err) + } +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go new file mode 100644 index 0000000..7b650b0 --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go @@ -0,0 +1,104 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log" + "strings" + "sync" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" + + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" +) + +// toolchainStreams are the stream each build module carries its toolchain.reported event on, as the bus names it +// (declared by the build modules, heard by this module through consumes). +var toolchainStreams = map[string]string{buildrun.GoBuildModule: "TOOLCHAINS_GO", buildrun.NodeBuildModule: "TOOLCHAINS_NODE"} + +// toolchainWatch keeps the build modules' toolchains in a buildrun.ToolchainSet and tells the builder when one changed. A +// build module's bucket is made when it is assigned, after the builder started, and the builder holds no timer (ADR 0437 +// §8): it watches the buckets that exist at start, and a build module says each time it wrote its toolchains key +// (toolchain.reported, handled by Reported); a bucket not watched yet is then watched. +type toolchainWatch struct { + ctx context.Context + js bucketOpener + set *buildrun.ToolchainSet + changed func(ctx context.Context) // a language's entries changed: the failed builds are tried again + logger *log.Logger + errs chan error + mu sync.Mutex + watched map[string]bool +} + +func newToolchainWatch(ctx context.Context, js bucketOpener, set *buildrun.ToolchainSet, changed func(context.Context), logger *log.Logger) *toolchainWatch { + return &toolchainWatch{ctx: ctx, js: js, set: set, changed: changed, logger: logger, errs: make(chan error, len(buildrun.LanguageOf)), watched: map[string]bool{}} +} + +// start watches the buckets that exist now; a missing one is said and waits for its build module's event. +func (w *toolchainWatch) start() error { + for module := range buildrun.LanguageOf { + if err := w.ensure(module); err != nil { + return err + } + } + return nil +} + +func (w *toolchainWatch) ensure(module string) error { + w.mu.Lock() + defer w.mu.Unlock() + if w.watched[module] { + return nil + } + name := delivery.ToolchainsBucket(module) + kv, err := w.js.KeyValue(w.ctx, name) + switch { + case errors.Is(err, jetstream.ErrBucketNotFound): + w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module) + return nil + case err != nil: + return fmt.Errorf("the bucket %s: %w", name, err) + } + watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.set.Put(module, e.Key(), e.Value(), !opOf(e)) }) + if err != nil { + return err + } + w.watched[module] = true + go func() { + w.errs <- follow(w.ctx, watcher, func(e jetstream.KeyValueEntry) { + if w.set.Put(module, e.Key(), e.Value(), !opOf(e)) { + w.changed(w.ctx) + } + }) + }() + w.changed(w.ctx) // what the settle read is new to the builder + return nil +} + +// delivered is the part of a JetStream message the watch uses. +type delivered interface { + Subject() string + Ack() error + Term() error +} + +// Reported takes one toolchain.reported event of a build module in: the module is read off the subject, +// mesh.stream..event.toolchain.reported, and its bucket is watched if it is not. A subject of no build module is +// terminated; a failure stays unacknowledged, and the server delivers it again. +func (w *toolchainWatch) Reported(m delivered) { + module, _, _ := strings.Cut(strings.TrimPrefix(m.Subject(), "mesh.stream."), ".event.") + if _, known := buildrun.LanguageOf[module]; !known || !strings.HasPrefix(m.Subject(), "mesh.stream.") { + w.logger.Printf("%s: not a build module's event, dropped", m.Subject()) + _ = m.Term() + return + } + if err := w.ensure(module); err != nil { + w.logger.Printf("%s: left for redelivery: %v", m.Subject(), err) + return + } + _ = m.Ack() +} diff --git a/modules/mesh-delivery-checker/files/settings.json b/modules/mesh-delivery-checker/files/settings.json new file mode 100644 index 0000000..1354ea4 --- /dev/null +++ b/modules/mesh-delivery-checker/files/settings.json @@ -0,0 +1,3 @@ +{ + "capacity": "" +} diff --git a/modules/mesh-delivery-checker/internal/check/case.go b/modules/mesh-delivery-checker/internal/check/case.go new file mode 100644 index 0000000..24cb19e --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/case.go @@ -0,0 +1,135 @@ +package check + +import ( + "fmt" + "maps" + "sort" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// HQRepository is the repository the dark checker writes no status for (ADR 0484 clause 8): hq builds no module and its +// check is a script no build module can run, so it is judged by its merge-check.sh until issue 1296 settles how. +const HQRepository = "novox/hq" + +// HQNote is what the read tools say of an hq pull request (ADR 0484 clause 8). +const HQNote = "hq: judged by merge-check.sh until issue 1296" + +// Pull is one pull request the checker judges: its head, the trunk head of its repository it is merged onto and the +// modules the indexer says it touches, with the fingerprints the indexer wrote for them. +type Pull struct { + Repository string + Number int + Head string + Trunk string + Modules []string + Fps map[string]delivery.CandidateModule +} + +// Case is one check: a candidate's pull requests that are judged (those of hq and of a repository whose trunk is not known +// are left out), the trunk head of each repository and the key of the verdict. +type Case struct { + // Key is the verdict key, `.`. + Key string + Group string + HeadsHash string + Pulls []Pull + // Trunk is the trunk head of each repository of Pulls. + Trunk map[string]string + // Left says why pull requests of the candidate are not judged (hq's, a repository with no known trunk); empty when all are. + Left []string + // Queued is when the checker took the case up, for the time of its queue. + Queued time.Time + // Progress says what the check is doing, so a check that ran past its bound names the step it was stuck at. + Progress *Progress +} + +// Progress is the step a check is at, written by the check and read when its bound passes. +type Progress struct { + mu sync.Mutex + stage, node string +} + +// Set records the step now running and the node it runs on, empty for none. +func (p *Progress) Set(stage, node string) { + p.mu.Lock() + p.stage, p.node = stage, node + p.mu.Unlock() +} + +// Now is the step recorded last and its node. +func (p *Progress) Now() (stage, node string) { + p.mu.Lock() + defer p.mu.Unlock() + return p.stage, p.node +} + +// CaseOf is the check of a candidate against the trunk heads (the gate's source.main): the pull requests that are judged. +// The checker judges pull requests into main only (ADR 0484 clause 11): the pull message carries no base, a merge into +// another branch is a closed message, so every candidate is into the trunk; a repository whose trunk head is not known has +// none to merge onto. It answers nil when no pull request is left to judge. +func CaseOf(c delivery.Candidate, trunk map[string]string) *Case { + key, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash) + if err != nil { + return nil + } + cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Trunk: map[string]string{}, Progress: &Progress{}} + for _, p := range c.Pulls { + switch head, known := trunk[p.Repository]; { + case p.Repository == HQRepository: + cs.Left = append(cs.Left, fmt.Sprintf("%s#%d: %s", p.Repository, p.Number, HQNote)) + case !known || head == "": + cs.Left = append(cs.Left, fmt.Sprintf("%s#%d: the trunk head of %s is not known", p.Repository, p.Number, p.Repository)) + default: + cs.Pulls = append(cs.Pulls, Pull{Repository: p.Repository, Number: p.Number, Head: p.Head.Head, Trunk: head, Modules: p.Modules, Fps: p.Fps}) + cs.Trunk[p.Repository] = head + } + } + if len(cs.Pulls) == 0 { + return nil + } + sort.SliceStable(cs.Pulls, func(i, j int) bool { + if cs.Pulls[i].Repository != cs.Pulls[j].Repository { + return cs.Pulls[i].Repository < cs.Pulls[j].Repository + } + return cs.Pulls[i].Number < cs.Pulls[j].Number + }) + return cs +} + +// sameTrunk says whether two verdicts' trunk heads are the same. +func sameTrunk(a, b map[string]string) bool { return maps.Equal(a, b) } + +// Trunks holds the trunk head of each repository, as the gate's source.main says (a watch puts it in). +type Trunks struct { + mu sync.Mutex + heads map[string]string +} + +// Put takes a source.main value in and answers whether the heads changed. +func (t *Trunks) Put(value []byte, deleted bool) bool { + var heads map[string]string + if !deleted { + s, err := delivery.DecodeSource(value) + if err != nil { + return false + } + heads = s.Heads + } + t.mu.Lock() + defer t.mu.Unlock() + if maps.Equal(t.heads, heads) { + return false + } + t.heads = maps.Clone(heads) + return true +} + +// Heads are the trunk heads now (a copy). +func (t *Trunks) Heads() map[string]string { + t.mu.Lock() + defer t.mu.Unlock() + return maps.Clone(t.heads) +} diff --git a/modules/mesh-delivery-checker/internal/check/checker.go b/modules/mesh-delivery-checker/internal/check/checker.go new file mode 100644 index 0000000..5706ae4 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/checker.go @@ -0,0 +1,451 @@ +package check + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// DefaultBound is how long a check may run before it is ended failed naming the step it was stuck at (ADR 0484, issue 1678). +const DefaultBound = 30 * time.Minute + +// NodesWords is in the Words of a failure that is the build nodes' (none qualifies, the toolchains disagree): such a verdict +// is judged again when the nodes change, by a watch and never a timer (ADR 0437 §8). +const NodesWords = "build node qualifies" + +// Outcome is what judging a case gives: the verdict (state, failure, levels, times, composition; the checker adds the +// group, the heads and the trunk) and, for the line, what the check knows of the builds of the modules it touches. +type Outcome struct { + Verdict delivery.Verdict + Builds []BuildNote +} + +// BuildNote is the state of a candidate's module build, by module: built, failed or waiting (ADR 0484 clause 7). +type BuildNote struct { + Module string + State string +} + +// Judge runs the check of a case: the merge, the levels. It writes what it is doing to c.Progress. It answers an error +// only when it could not judge at all; a failure of the change is a Verdict. +type Judge interface { + Judge(ctx context.Context, c *Case) (Outcome, error) +} + +// Checker judges the candidates the indexer writes, dark (ADR 0453 clause 4, ADR 0484): for each candidate it claims a +// verdict key, runs the check through its Judge, writes the verdict and, for each pull request judged, the owed status +// `mesh/check` the forge bridge performs. It holds no timer; a check starts from a candidate or from a change of the +// nodes, the trunk or the controller's plan, and ends when its judge does or its bound passes. +type Checker struct { + Verdicts KV + Owed KV + Judge Judge + Trunks interface{ Heads() map[string]string } + // Bound is how long one check may run; DefaultBound when 0. + Bound time.Duration + // Limit is how many checks run at once; two when 0. + Limit int + // Hold says whether a delivery builds, so no new check starts (ADR 0484 clause 12); nil never holds. + Hold func() bool + // Link is the base of the address a status points at; the verdict key follows. + Link string + Now func() time.Time + Logf func(format string, args ...any) + + once sync.Once + sem chan struct{} + wg sync.WaitGroup + mu sync.Mutex + known map[string]delivery.Candidate // by candidates key + inflight map[string]bool // by verdict key + again map[string]bool // by verdict key: asked again while it ran + tried map[string]bool // by verdict key: looked at since this process started + epoch uint64 // changes of the nodes seen since this process started + nodesFail map[string]uint64 // by verdict key: the epoch a failure of the nodes was written at +} + +func (ch *Checker) init() { + ch.once.Do(func() { + limit := ch.Limit + if limit <= 0 { + limit = 2 + } + ch.sem = make(chan struct{}, limit) + ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{} + ch.nodesFail = map[string]uint64{} + }) +} + +func (ch *Checker) logf(format string, args ...any) { + if ch.Logf != nil { + ch.Logf(format, args...) + } +} + +func (ch *Checker) now() time.Time { + if ch.Now != nil { + return ch.Now() + } + return time.Now() +} + +func (ch *Checker) bound() time.Duration { + if ch.Bound > 0 { + return ch.Bound + } + return DefaultBound +} + +// Wait returns when every check started so far has ended. +func (ch *Checker) Wait() { ch.wg.Wait() } + +// Candidate takes one value of the indexer's candidates bucket in (the key is the group) and checks it if it is to be +// checked. A value that is not a candidate is said and left. +func (ch *Checker) Candidate(ctx context.Context, key string, value []byte) { + ch.init() + c, err := delivery.DecodeCandidate(value) + if err != nil { + ch.logf("candidates.%s holds what is not a candidate, left: %v", key, err) + return + } + ch.mu.Lock() + ch.known[key] = c + ch.mu.Unlock() + ch.consider(ctx, key, c) +} + +// Forget takes a candidate out (its key was deleted). +func (ch *Checker) Forget(key string) { + ch.init() + ch.mu.Lock() + delete(ch.known, key) + ch.mu.Unlock() +} + +// NodesChanged is Reconsider after a change of the nodes: one came online or went, a build module became healthy or +// stopped being, or a toolchain moved. A check that failed because no node qualified is run again only after such a change. +func (ch *Checker) NodesChanged(ctx context.Context) { + ch.init() + ch.mu.Lock() + ch.epoch++ + ch.mu.Unlock() + ch.Reconsider(ctx) +} + +// Reconsider looks at every candidate again: the trunk moved, the nodes changed or a delivery ended. A candidate whose +// verdict stands for the trunk it was judged against is left. +func (ch *Checker) Reconsider(ctx context.Context) { + ch.init() + ch.mu.Lock() + known := make(map[string]delivery.Candidate, len(ch.known)) + for k, c := range ch.known { + known[k] = c + } + ch.mu.Unlock() + for k, c := range known { + ch.consider(ctx, k, c) + } +} + +func (ch *Checker) heads() map[string]string { + if ch.Trunks == nil { + return nil + } + return ch.Trunks.Heads() +} + +func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidate) { + cs := CaseOf(c, ch.heads()) + if cs == nil { + return // nothing to judge: hq's pull requests, or no trunk known + } + if ch.Hold != nil && ch.Hold() { + ch.logf("%s: not started, a delivery builds", cs.Key) + return // started when the plan bucket empties (Reconsider) + } + cs.Queued = ch.now() + ch.mu.Lock() + if ch.inflight[cs.Key] { + ch.again[cs.Key] = true // looked at once more when the check ends, so a change of the trunk is never lost + ch.mu.Unlock() + return + } + ch.inflight[cs.Key] = true + ch.mu.Unlock() + ch.wg.Add(1) + go func() { + defer ch.wg.Done() + ch.sem <- struct{}{} + defer func() { <-ch.sem }() + for { + ch.try(ctx, key, cs) + ch.mu.Lock() + more := ch.again[cs.Key] + delete(ch.again, cs.Key) + if !more || ctx.Err() != nil { + delete(ch.inflight, cs.Key) + ch.mu.Unlock() + return + } + ch.mu.Unlock() + if next := CaseOf(ch.candidate(key, c), ch.heads()); next != nil { + cs = next + } + } + }() +} + +func (ch *Checker) candidate(key string, fallback delivery.Candidate) delivery.Candidate { + ch.mu.Lock() + defer ch.mu.Unlock() + if c, ok := ch.known[key]; ok { + return c + } + return fallback +} + +// claimed is a verdict key the checker holds: the revision the next write is made at. +type claimed struct{ rev uint64 } + +// claim takes the check of the case, or answers nil when it is not this checker's to run now: +// - no key: it is created running; a create that finds the key made meanwhile gets nothing; +// - a key left running is taken over as it stands when this process looks at it for the first time (it was left by the +// checker that stopped; there is one instance, as there is one builder per node, ADR 0478 clause 6), and left otherwise; +// - a pass or a failure for the trunk heads the case has stands; one for other trunk heads (the trunk moved) is taken +// again by compare-and-set on the same key, and so is a failure of the nodes after the nodes changed. +func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64) (*claimed, error) { + running := delivery.Verdict{Group: cs.Group, Heads: cs.HeadsHash, Trunk: cs.Trunk, State: delivery.VerdictRunning} + data, err := delivery.EncodeVerdict(running) + if err != nil { + return nil, err + } + value, rev, found, err := ch.Verdicts.Get(ctx, cs.Key) + if err != nil { + return nil, fmt.Errorf("%s: %w", cs.Key, err) + } + if !found { + rev, err := ch.Verdicts.Create(ctx, cs.Key, data) + if errors.Is(err, ErrConflict) { + return nil, nil // another checker made it first + } + if err != nil { + return nil, fmt.Errorf("%s: %w", cs.Key, err) + } + return &claimed{rev}, nil + } + cur, err := delivery.DecodeVerdict(value) + if err != nil { + return nil, fmt.Errorf("%s holds what is not a verdict: %w", cs.Key, err) + } + retake := false + switch { + case cur.State == delivery.VerdictRunning: + if first { + return &claimed{rev}, nil + } + return nil, nil + case !sameTrunk(cur.Trunk, cs.Trunk): + retake = true + case cur.State == delivery.VerdictFail && IsNodesFailure(cur): + ch.mu.Lock() + at, ok := ch.nodesFail[cs.Key] + ch.mu.Unlock() + retake = !ok || epoch > at + } + if !retake { + return nil, nil + } + rev, err = ch.Verdicts.Update(ctx, cs.Key, data, rev) + if errors.Is(err, ErrConflict) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("%s: %w", cs.Key, err) + } + return &claimed{rev}, nil +} + +// IsNodesFailure says whether a verdict failed because no build node could take the check. +func IsNodesFailure(v delivery.Verdict) bool { + return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, NodesWords) +} + +// try claims the check, judges it within its bound and writes the verdict and the owed statuses. A check that ends +// because ctx did is left running: the checker takes it again when it starts. +func (ch *Checker) try(ctx context.Context, key string, cs *Case) { + ch.mu.Lock() + first := !ch.tried[cs.Key] + ch.tried[cs.Key] = true + epoch := ch.epoch + ch.mu.Unlock() + cl, err := ch.claim(ctx, cs, first, epoch) + if err != nil { + ch.logf("%s: %v", cs.Key, err) + return + } + if cl == nil { + return + } + began := ch.now() + ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, nil) + out := ch.judge(ctx, cs) + if ctx.Err() != nil { + return + } + v := out.Verdict + v.Group, v.Heads, v.Trunk = cs.Group, cs.HeadsHash, cs.Trunk + if v.Took.Queue == 0 && !cs.Queued.IsZero() && began.After(cs.Queued) { + v.Took.Queue = began.Sub(cs.Queued).Milliseconds() + } + data, err := delivery.EncodeVerdict(v) + if err != nil { + ch.logf("%s: the verdict cannot be written: %v", cs.Key, err) + v = delivery.Verdict{Group: cs.Group, Heads: cs.HeadsHash, Trunk: cs.Trunk, State: delivery.VerdictFail, + Failed: delivery.FailedFirst{Machine: "checker", Words: "the verdict could not be written: " + err.Error()}} + if data, err = delivery.EncodeVerdict(v); err != nil { + return + } + } + if _, err := ch.Verdicts.Update(ctx, cs.Key, data, cl.rev); err != nil { + ch.logf("%s: the verdict could not be written: %v", cs.Key, err) + return + } + if IsNodesFailure(v) { + ch.mu.Lock() + ch.nodesFail[cs.Key] = epoch + ch.mu.Unlock() + } + ch.writeOwed(ctx, cs, v, out.Builds) + ch.logf("%s: %s", cs.Key, v.State) +} + +// judge runs the case within the bound. A judge that fails to judge, and a check that ran past its bound, are verdicts of +// failure naming the machine and the step. +func (ch *Checker) judge(ctx context.Context, cs *Case) Outcome { + bound := ch.bound() + run, cancel := context.WithTimeoutCause(ctx, bound, errBound) + defer cancel() + out, err := ch.Judge.Judge(run, cs) + stage, node := cs.Progress.Now() + if node == "" { + node = "checker" + } + switch { + case ctx.Err() != nil: + return Outcome{} + case context.Cause(run) == errBound: + if stage == "" { + stage = "starting" + } + return Outcome{Verdict: failure(node, fmt.Sprintf("the check ran past its bound of %s, stuck at: %s", bound, stage))} + case err != nil: + return Outcome{Verdict: failure(node, err.Error())} + } + return out +} + +var errBound = errors.New("the check ran past its bound") + +func failure(machine, words string) delivery.Verdict { + return delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: machine, Words: words}} +} + +// writeOwed writes, for each pull request judged, the status the forge bridge performs: pending while the check runs, success +// or failure at its end (ADR 0484 clause 2). A key already holding the same value is not written again, so the bridge +// performs one act for each change; an older head's key is never touched. +func (ch *Checker) writeOwed(ctx context.Context, cs *Case, v delivery.Verdict, builds []BuildNote) { + for _, p := range cs.Pulls { + key, err := delivery.CheckerOwedKey(p.Repository, p.Number, p.Head) + if err != nil { + ch.logf("%s: %v", cs.Key, err) + continue + } + state := map[string]string{delivery.VerdictRunning: "pending", delivery.VerdictPass: "success", delivery.VerdictFail: "failure"}[v.State] + o := delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: state, + Target: strings.TrimSuffix(ch.Link, "/") + "/" + cs.Key, Text: Line(v, cs, p, builds)} + data, err := delivery.EncodeCheckerOwed(o) + if err != nil { + ch.logf("%s: the status of %s cannot be written: %v", cs.Key, key, err) + continue + } + if err := putOnce(ctx, ch.Owed, key, data, func(cur []byte) bool { + held, err := delivery.DecodeCheckerOwed(cur) + o.V = delivery.Version + return err == nil && delivery.SameOwed(held, o) + }); err != nil { + ch.logf("%s: the status of %s cannot be written: %v", cs.Key, key, err) + } + } +} + +// putOnce writes data under key by create, or by compare-and-set when the key exists, and writes nothing when same says the +// key holds the value already. +func putOnce(ctx context.Context, kv KV, key string, data []byte, same func(cur []byte) bool) error { + for try := 0; try < 4; try++ { + cur, rev, found, err := kv.Get(ctx, key) + if err != nil { + return err + } + if found && same(cur) { + return nil + } + if !found { + _, err = kv.Create(ctx, key, data) + } else { + _, err = kv.Update(ctx, key, data, rev) + } + if errors.Is(err, ErrConflict) { + continue + } + return err + } + return fmt.Errorf("%s changed under every one of 4 tries", key) +} + +// Sweep closes, at start, every verdict left running for a candidate that no longer holds those heads: nothing will check +// them again, and the key would stay running for ever (the builder's Sweep, issue 1660). Call it once the candidates have been +// read. A key of heads a candidate holds now is not touched: the check takes it over. +func (ch *Checker) Sweep(ctx context.Context) error { + ch.init() + keys, err := ch.Verdicts.Keys(ctx) + if err != nil { + return err + } + ch.mu.Lock() + current := map[string]bool{} + for _, c := range ch.known { + if k, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash); err == nil { + current[k] = true + } + } + ch.mu.Unlock() + for _, key := range keys { + if current[key] { + continue + } + value, rev, found, err := ch.Verdicts.Get(ctx, key) + if err != nil || !found { + continue + } + cur, err := delivery.DecodeVerdict(value) + if err != nil || cur.State != delivery.VerdictRunning { + continue + } + cur.State = delivery.VerdictFail + cur.Failed = delivery.FailedFirst{Machine: "checker", Words: "closed at start: left running, and no candidate holds these heads any more, so nothing will check them"} + data, err := delivery.EncodeVerdict(cur) + if err != nil { + return err + } + if _, err := ch.Verdicts.Update(ctx, key, data, rev); err != nil && !errors.Is(err, ErrConflict) { + return fmt.Errorf("%s: %w", key, err) + } + ch.logf("%s: closed, left running for heads no candidate holds", key) + } + return nil +} diff --git a/modules/mesh-delivery-checker/internal/check/checker_test.go b/modules/mesh-delivery-checker/internal/check/checker_test.go new file mode 100644 index 0000000..4a791f4 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/checker_test.go @@ -0,0 +1,367 @@ +package check + +import ( + "context" + "encoding/json" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +const ( + repoA = "novox/mesh-delivery" + repoSDK = "novox/mesh-sdk" + head1 = "1111111111111111111111111111111111111111" + head2 = "2222222222222222222222222222222222222222" + trunkA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + trunkA2 = "abababababababababababababababababababab" + trunkS = "5555555555555555555555555555555555555555" +) + +// candidateOf is a candidate of a group with the given pull requests, as the indexer writes one. +func candidateOf(t testing.TB, group string, pulls ...delivery.CandidatePull) []byte { + t.Helper() + heads := make([]delivery.Head, len(pulls)) + for i, p := range pulls { + heads[i] = p.Head + } + hash, err := delivery.HeadsHash(heads) + if err != nil { + t.Fatal(err) + } + c := delivery.Candidate{V: delivery.Version, Group: group, HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)} + data, err := delivery.EncodeCandidate(c) + if err != nil { + t.Fatal(err) + } + return data +} + +func pullOf(repo string, number int, head string) delivery.CandidatePull { + return delivery.CandidatePull{Head: delivery.Head{Repository: repo, Number: number, Head: head}, Modules: []string{}} +} + +// judgeFunc is a Judge written as a function; it counts its calls. +type judgeFunc struct { + mu sync.Mutex + calls int + f func(ctx context.Context, c *Case, call int) (Outcome, error) +} + +func (j *judgeFunc) Judge(ctx context.Context, c *Case) (Outcome, error) { + j.mu.Lock() + j.calls++ + n := j.calls + j.mu.Unlock() + return j.f(ctx, c, n) +} + +func (j *judgeFunc) count() int { j.mu.Lock(); defer j.mu.Unlock(); return j.calls } + +func passing() delivery.Verdict { + return delivery.Verdict{State: delivery.VerdictPass, Levels: map[string]delivery.LevelResult{ + delivery.LevelUnit: {Run: 2, Kept: 3}, delivery.LevelIntegration: {Run: 1}}} +} + +func passJudge() *judgeFunc { + return &judgeFunc{f: func(context.Context, *Case, int) (Outcome, error) { return Outcome{Verdict: passing()}, nil }} +} + +type rig struct { + ch *Checker + verdicts *memKV + owed *memKV + trunks *Trunks +} + +func newRig(t testing.TB, j Judge) *rig { + t.Helper() + r := &rig{verdicts: newMemKV(), owed: newMemKV(), trunks: &Trunks{}} + r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS}) + r.ch = &Checker{Verdicts: r.verdicts, Owed: r.owed, Judge: j, Trunks: r.trunks, Link: "https://view.invalid/checks", Now: time.Now} + return r +} + +func (r *rig) setTrunks(t testing.TB, heads map[string]string) bool { + t.Helper() + data, err := delivery.EncodeSource(delivery.Source{Group: "g", Heads: heads}) + if err != nil { + t.Fatal(err) + } + return r.trunks.Put(data, false) +} + +func (r *rig) verdict(t testing.TB, key string) delivery.Verdict { + t.Helper() + raw, _, found, _ := r.verdicts.Get(context.Background(), key) + if !found { + t.Fatalf("no verdict %s; the bucket holds %v", key, r.verdicts.vals) + } + v, err := delivery.DecodeVerdict(raw) + if err != nil { + t.Fatal(err) + } + return v +} + +func keyOf(t testing.TB, group string, value []byte) string { + t.Helper() + c, err := delivery.DecodeCandidate(value) + if err != nil { + t.Fatal(err) + } + k, err := delivery.CheckerVerdictKey(group, c.HeadsHash) + if err != nil { + t.Fatal(err) + } + return k +} + +func TestACandidateNotYetCheckedGetsItsVerdictKeyCreatedOnceAndIsJudged(t *testing.T) { // N6/AC1, 1678 + ctx, j := context.Background(), passJudge() + r := newRig(t, j) + cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1)) + r.ch.Candidate(ctx, "feat%2Fx", cand) + r.ch.Wait() + key := keyOf(t, "feat/x", cand) + if v := r.verdict(t, key); v.State != delivery.VerdictPass || v.Group != "feat/x" || v.Trunk[repoA] != trunkA { + t.Fatalf("verdict = %+v", v) + } + if r.verdicts.creates[key] != 1 || j.count() != 1 { + t.Errorf("creates = %d, judged = %d, want one each", r.verdicts.creates[key], j.count()) + } + // The same candidate heard again, with nothing changed, is not judged again. + r.ch.Candidate(ctx, "feat%2Fx", cand) + r.ch.Wait() + if j.count() != 1 { + t.Errorf("judged %d times after the same candidate again, want once", j.count()) + } +} + +// racyKV lets another instance create the key just before this one's create. +type racyKV struct { + *memKV + other []byte +} + +func (r racyKV) Create(ctx context.Context, key string, value []byte) (uint64, error) { + if r.other != nil { + _, _ = r.memKV.Create(ctx, key, r.other) + } + return r.memKV.Create(ctx, key, value) +} + +func TestASecondInstancesCreateIsRefusedAndItDoesNothing(t *testing.T) { // N6/AC1 + ctx, j := context.Background(), passJudge() + r := newRig(t, j) + cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1)) + key := keyOf(t, "feat/x", cand) + other, _ := delivery.EncodeVerdict(delivery.Verdict{Group: "feat/x", Heads: strings.SplitN(key, ".", 2)[1], State: delivery.VerdictRunning}) + r.ch.Verdicts = racyKV{r.verdicts, other} + r.ch.Candidate(ctx, "feat%2Fx", cand) + r.ch.Wait() + if j.count() != 0 { + t.Errorf("the instance that lost the create judged %d times, want none", j.count()) + } + if v := r.verdict(t, key); v.State != delivery.VerdictRunning { + t.Errorf("the other instance's key was changed: %+v", v) + } +} + +func TestARestartWithARunningKeyJudgesItAgain(t *testing.T) { // N6/AC2 + ctx, j := context.Background(), passJudge() + r := newRig(t, j) + cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1)) + key := keyOf(t, "feat/x", cand) + left, _ := delivery.EncodeVerdict(delivery.Verdict{Group: "feat/x", Heads: strings.SplitN(key, ".", 2)[1], Trunk: map[string]string{repoA: trunkA}, State: delivery.VerdictRunning}) + if _, err := r.verdicts.Create(ctx, key, left); err != nil { + t.Fatal(err) + } + r.ch.Candidate(ctx, "feat%2Fx", cand) + r.ch.Wait() + if j.count() != 1 || r.verdict(t, key).State != delivery.VerdictPass { + t.Fatalf("judged %d times, verdict %+v, want the left check judged once and passed", j.count(), r.verdict(t, key)) + } +} + +func TestACheckPastItsBoundFailsNamingTheStuckStep(t *testing.T) { // N6/AC3 + ctx := context.Background() + j := &judgeFunc{f: func(ctx context.Context, c *Case, _ int) (Outcome, error) { + c.Progress.Set("test the unit level of mesh-delivery", "shanks") + <-ctx.Done() + return Outcome{}, ctx.Err() + }} + r := newRig(t, j) + r.ch.Bound = 30 * time.Millisecond + cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1)) + r.ch.Candidate(ctx, "feat%2Fx", cand) + r.ch.Wait() + v := r.verdict(t, keyOf(t, "feat/x", cand)) + if v.State != delivery.VerdictFail || v.Failed.Machine != "shanks" || !strings.Contains(v.Failed.Words, "stuck at: test the unit level of mesh-delivery") { + t.Fatalf("verdict = %+v, want a failure naming shanks and the stuck step", v) + } +} + +func TestNoQualifyingNodeFailsWithTheReasonAndIsJudgedAgainWhenTheNodesChange(t *testing.T) { // N6/AC4 + ctx := context.Background() + j := &judgeFunc{f: func(_ context.Context, _ *Case, n int) (Outcome, error) { + if n == 1 { + return Outcome{Verdict: failure("shanks", "no go "+NodesWords+": none is online")}, nil + } + return Outcome{Verdict: passing()}, nil + }} + r := newRig(t, j) + cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1)) + key := keyOf(t, "feat/x", cand) + r.ch.Candidate(ctx, "feat%2Fx", cand) + r.ch.Wait() + if v := r.verdict(t, key); !IsNodesFailure(v) { + t.Fatalf("verdict = %+v, want a failure of the nodes", v) + } + r.ch.Reconsider(ctx) // nothing changed: not judged again + r.ch.Wait() + if j.count() != 1 { + t.Fatalf("judged %d times with the nodes unchanged, want once", j.count()) + } + r.ch.NodesChanged(ctx) + r.ch.Wait() + if j.count() != 2 || r.verdict(t, key).State != delivery.VerdictPass { + t.Fatalf("after the nodes changed: judged %d times, verdict %+v", j.count(), r.verdict(t, key)) + } +} + +func TestHqAndARepositoryWithNoTrunkGetNoVerdict(t *testing.T) { // N6/AC6, ADR 0484 clauses 8 and 11 + ctx, j := context.Background(), passJudge() + r := newRig(t, j) + r.ch.Candidate(ctx, "feat/hq", candidateOf(t, "feat/hq", pullOf(HQRepository, 3, head1))) + r.ch.Candidate(ctx, "feat/y", candidateOf(t, "feat/y", pullOf("novox/unknown", 4, head1))) + r.ch.Wait() + if j.count() != 0 || len(r.verdicts.vals) != 0 || len(r.owed.vals) != 0 { + t.Errorf("judged %d times, %d verdicts, %d statuses, want none", j.count(), len(r.verdicts.vals), len(r.owed.vals)) + } +} + +func TestAGroupWithHqJudgesItsOtherMembersAndWritesNoStatusForHq(t *testing.T) { // ADR 0484 clause 8 + ctx, j := context.Background(), passJudge() + r := newRig(t, j) + r.ch.Candidate(ctx, "feat/x", candidateOf(t, "feat/x", pullOf(HQRepository, 3, head1), pullOf(repoA, 7, head2))) + r.ch.Wait() + if j.count() != 1 { + t.Fatalf("judged %d times, want once", j.count()) + } + keys, _ := r.owed.Keys(ctx) + if len(keys) != 1 || !strings.HasPrefix(keys[0], "novox/mesh-delivery.7.") && !strings.Contains(keys[0], "mesh-delivery") { + t.Errorf("statuses %v, want one for mesh-delivery#7 only", keys) + } +} + +func TestATrunkMoveRewritesTheSameVerdictKeyAndTheHeadsStatus(t *testing.T) { // 1300/AC7 + ctx, j := context.Background(), passJudge() + r := newRig(t, j) + cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1)) + key := keyOf(t, "feat/x", cand) + r.ch.Candidate(ctx, "feat%2Fx", cand) + r.ch.Wait() + _, rev1, _, _ := r.verdicts.Get(ctx, key) + owedKey, _ := delivery.CheckerOwedKey(repoA, 7, head1) + _, orev1, _, _ := r.owed.Get(ctx, owedKey) + if !r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) { + t.Fatal("the trunks did not change") + } + r.ch.Reconsider(ctx) + r.ch.Wait() + _, rev2, _, _ := r.verdicts.Get(ctx, key) + if j.count() != 2 || rev2 <= rev1 || r.verdict(t, key).Trunk[repoA] != trunkA2 { + t.Fatalf("judged %d times, revisions %d then %d, trunk %v", j.count(), rev1, rev2, r.verdict(t, key).Trunk) + } + if r.verdicts.creates[key] != 1 { + t.Errorf("the key was created %d times, want the same key rewritten", r.verdicts.creates[key]) + } + if _, orev2, _, _ := r.owed.Get(ctx, owedKey); orev2 <= orev1 { + // the line is the same when nothing differs; the status is rewritten when the words change, here the trunk is in them + t.Logf("the status keeps revision %d (its words did not change)", orev2) + } + // A new head is another key, and the older head's status is not touched. + older, _, _, _ := r.owed.Get(ctx, owedKey) + cand2 := candidateOf(t, "feat/x", pullOf(repoA, 7, head2)) + r.ch.Candidate(ctx, "feat%2Fx", cand2) + r.ch.Wait() + if again, _, _, _ := r.owed.Get(ctx, owedKey); string(again) != string(older) { + t.Errorf("the older head's status was rewritten") + } + if n := len(r.owed.vals); n != 2 { + t.Errorf("%d statuses, want one for each head", n) + } +} + +func TestNoCheckStartsWhileADeliveryBuildsAndOneStartsWhenItEnds(t *testing.T) { // N8/AC1 + ctx, j := context.Background(), passJudge() + r := newRig(t, j) + held := true + r.ch.Hold = func() bool { return held } + r.ch.Candidate(ctx, "feat/x", candidateOf(t, "feat/x", pullOf(repoA, 7, head1))) + r.ch.Wait() + if j.count() != 0 { + t.Fatalf("judged %d times while held", j.count()) + } + held = false + r.ch.Reconsider(ctx) + r.ch.Wait() + if j.count() != 1 { + t.Fatalf("judged %d times after the hold ended, want once", j.count()) + } +} + +func TestTheStatusIsPendingWhileTheCheckRunsAndEndsSuccessOrFailure(t *testing.T) { // 1300/AC5 (unit part) + ctx := context.Background() + var seen []string + var r *rig + j := &judgeFunc{f: func(ctx context.Context, c *Case, _ int) (Outcome, error) { + key, _ := delivery.CheckerOwedKey(repoA, 7, head1) + raw, _, _, _ := r.owed.Get(ctx, key) + o, _ := delivery.DecodeCheckerOwed(raw) + seen = append(seen, o.State) + return Outcome{Verdict: delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Test: "m/a.TestX"}}}, nil + }} + r = newRig(t, j) + r.ch.Candidate(ctx, "feat/x", candidateOf(t, "feat/x", pullOf(repoA, 7, head1))) + r.ch.Wait() + key, _ := delivery.CheckerOwedKey(repoA, 7, head1) + raw, _, _, _ := r.owed.Get(ctx, key) + o, err := delivery.DecodeCheckerOwed(raw) + if err != nil { + t.Fatal(err) + } + if len(seen) != 1 || seen[0] != "pending" || o.State != "failure" || o.Context != delivery.StatusContextCheck || !strings.HasPrefix(o.Target, "https://view.invalid/checks/") { + t.Errorf("while running %v, at the end %+v", seen, o) + } +} + +func TestSweepClosesARunningKeyNoCandidateHoldsAndLeavesOneItDoes(t *testing.T) { + ctx, j := context.Background(), passJudge() + r := newRig(t, j) + cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1)) + live := keyOf(t, "feat/x", cand) + dead, _ := delivery.CheckerVerdictKey("feat/gone", "0123456789ab") + for _, k := range []string{live, dead} { + group, heads, _ := delivery.ParseCheckerVerdictKey(k) + raw, _ := delivery.EncodeVerdict(delivery.Verdict{Group: group, Heads: heads, State: delivery.VerdictRunning}) + _, _ = r.verdicts.Create(ctx, k, raw) + } + r.ch.init() + var c delivery.Candidate + _ = json.Unmarshal(cand, &c) + r.ch.known["feat%2Fx"] = c + if err := r.ch.Sweep(ctx); err != nil { + t.Fatal(err) + } + if v := r.verdict(t, dead); v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "closed at start") { + t.Errorf("the dead key = %+v", v) + } + if v := r.verdict(t, live); v.State != delivery.VerdictRunning { + t.Errorf("the live key = %+v, want it left for the check to take over", v) + } +} diff --git a/modules/mesh-delivery-checker/internal/check/line.go b/modules/mesh-delivery-checker/internal/check/line.go new file mode 100644 index 0000000..d2aaced --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/line.go @@ -0,0 +1,19 @@ +package check + +import ( + "fmt" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Line is the words of a status for one pull request of a case (ADR 0447 clause 3, ADR 0484 clause 7). +func Line(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote) string { + return fmt.Sprintf("%s: %s", v.State, short12(p.Head)) +} + +func short12(s string) string { + if len(s) > 12 { + return s[:12] + } + return s +} diff --git a/modules/mesh-delivery-checker/manifest_test.go b/modules/mesh-delivery-checker/manifest_test.go new file mode 100644 index 0000000..892ecbe --- /dev/null +++ b/modules/mesh-delivery-checker/manifest_test.go @@ -0,0 +1,197 @@ +package manifest + +import ( + "bytes" + "encoding/json" + "io/fs" + "os" + "path/filepath" + "reflect" + "slices" + "strings" + "testing" +) + +// The controller's manifest reader is mesh-controller internal/catalogue (Manifest, manifest.go: Consumes :521, +// Invokes :590, Streams :613, Reads :659; StreamDeclaration, streams.go), an internal package of another repository. +// These tests decode the manifest strictly into the fields those types name (DisallowUnknownFields) and hold the +// rules of issues 1678 and 1683 (ADR 0453 §4, ADR 0484). The merge gate's module check proves the controller takes the manifest. + +type state struct { + Name string `json:"name"` + History int `json:"history,omitempty"` +} + +type artifact struct { + Name string `json:"name"` + Kind string `json:"kind"` + Language string `json:"language"` + System string `json:"system"` + From string `json:"from"` + Binary string `json:"binary"` +} + +type manifest struct { + Module string `json:"module"` + Version string `json:"version"` + Slug string `json:"slug"` + Consumes []string `json:"consumes"` + State []state `json:"state"` + Reads []string `json:"reads"` + Invokes []string `json:"invokes"` + Config map[string]struct { + Type string `json:"type"` + Description string `json:"description"` + Pattern string `json:"pattern"` + Default string `json:"default"` + } `json:"config"` + OwnSecrets map[string]string `json:"own-secrets"` + SecretsOwner string `json:"secrets-owner"` + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []artifact `json:"artifacts"` + } `json:"build"` +} + +func load(t *testing.T) (manifest, map[string]json.RawMessage) { + t.Helper() + raw, err := os.ReadFile("module.json") + if err != nil { + t.Fatal(err) + } + var m manifest + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&m); err != nil { + t.Fatalf("the controller's reader refuses a key this manifest has: %v", err) + } + var top map[string]json.RawMessage + if err := json.Unmarshal(raw, &top); err != nil { + t.Fatal(err) + } + return m, top +} + +func sorted(s []string) []string { + s = slices.Clone(s) + slices.Sort(s) + return s +} + +func TestManifestDeclarations(t *testing.T) { // 1678/AC1 + m, top := load(t) + if m.Module != "mesh-delivery-checker" { + t.Errorf("module = %q", m.Module) + } + // delivery.CheckerKeptHistory, CheckerOwedHistory, PreviewsHistory, CheckerVerdictsHistory; the names follow the SDK's buckets. + if want := []state{{Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) { + t.Errorf("state = %+v, want %+v", m.State, want) + } + if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", + "mesh-controller.plans-building", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", + "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) { + t.Errorf("reads = %v, want %v", m.Reads, want) + } + if want := []string{"mesh-build-go.build_go_digest", "mesh-build-go.build_go_fetch", "mesh-build-go.build_go_test", "mesh-build-node.build_node_fetch", "mesh-build-node.build_node_test"}; !reflect.DeepEqual(sorted(m.Invokes), want) { + t.Errorf("invokes = %v, want only the build modules' fetch, digest and test (no run step before issue 1309, ADR 0484 clause 5)", m.Invokes) + } + // The checker is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it starts. + if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + t.Errorf("consumes = %v, want %v", m.Consumes, want) + } + if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { + t.Errorf("own-secrets = %v, want exactly broker", m.OwnSecrets) + } + if m.SecretsOwner != "mesh-delivery-checker" { + t.Errorf("secrets-owner = %q", m.SecretsOwner) + } + for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats", "bus"} { + if _, ok := top[k]; ok { + t.Errorf("the manifest declares %q: the checker has none", k) + } + } +} + +func TestManifestProcess(t *testing.T) { // 1678/AC1 + m, _ := load(t) + var procs []map[string]any + for _, r := range m.Resources { + if r["type"] == "process" { + procs = append(procs, r) + } + if r["type"] == "tool" || r["type"] == "seat" { + t.Errorf("resource %v: the checker serves no tool and holds no seat", r) + } + } + if len(procs) != 1 { + t.Fatalf("%d process resources, want one", len(procs)) + } + p := procs[0] + if p["artifact"] != "program" || !reflect.DeepEqual(p["run"], []any{"./mesh-delivery-checker"}) { + t.Errorf("process = %v", p) + } + env, _ := p["env"].(map[string]any) + // The capacity of each node is the module's setting `capacity` (ADR 0484 clause 12, as the builder's, issue 1661), laid into the + // settings file the program reads; empty is one step at a time everywhere. The environment holds no capacity. + if len(env) != 2 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_CHECKER_SETTINGS"] != "${dir:state}/settings.json" { + t.Errorf("env = %v", p["env"]) + } + if !reflect.DeepEqual(p["restart-on"], []any{"settings"}) { + t.Errorf("restart-on = %v, want the settings file, so a changed capacity reaches the running program", p["restart-on"]) + } + c, ok := m.Config["capacity"] + if !ok || len(m.Config) != 1 || c.Type != "string" || c.Default != "" || c.Description == "" { + t.Errorf("config = %+v, want one string setting, capacity, empty (one step at a time on every node) to start", m.Config) + } + var settings map[string]any + for _, r := range m.Resources { + if r["id"] == "settings" { + settings = r + } + } + if settings == nil || settings["type"] != "file" || settings["merge"] != "json" || !reflect.DeepEqual(settings["takes-config"], []any{"capacity"}) || settings["path"] != "${dir:state}/settings.json" { + t.Errorf("settings resource = %v", settings) + } + if raw, err := os.ReadFile("files/settings.json"); err != nil || !strings.Contains(string(raw), `"capacity": ""`) { + t.Errorf("files/settings.json = %q, %v, want the setting's starting value", raw, err) + } + if _, bus := p["bus"]; bus { + t.Errorf("the process declares a bus: it runs on its own broker credential") + } + if len(m.Build.Artifacts) != 2 || m.Build.Artifacts[1].Name != "files" || m.Build.Artifacts[1].Kind != "archive" || m.Build.Artifacts[1].From != "files" { + t.Fatalf("artifacts = %+v, want the program and the files", m.Build.Artifacts) + } + a := m.Build.Artifacts[0] + if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-checker" || !strings.HasSuffix(a.From, a.Binary) || a.Kind != "bundle" || a.Language != "go" { + t.Errorf("artifact = %+v", a) + } +} + +func TestServesNoRequestSubjectAndHoldsNoTicker(t *testing.T) { // 1678/AC6 (ADR 0437 §8: no timer, a wait is a context's bound) + forbidden := []string{"Subscribe(", "QueueSubscribe", "ChanSubscribe", "time.NewTicker", "time.Tick", "time.NewTimer", "time.AfterFunc", "time.After(", "time.Sleep"} + var files int + for _, root := range []string{"cmd", "internal"} { + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return err + } + files++ + src, err := os.ReadFile(path) + if err != nil { + return err + } + for _, f := range forbidden { + if strings.Contains(string(src), f) { + t.Errorf("%s holds %s: the checker serves no request subject and holds no timer", path, f) + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } + } + if files < 1 { + t.Fatalf("read %d source files, want the program's", files) + } +} diff --git a/modules/mesh-delivery-checker/module.json b/modules/mesh-delivery-checker/module.json new file mode 100644 index 0000000..8044314 --- /dev/null +++ b/modules/mesh-delivery-checker/module.json @@ -0,0 +1,113 @@ +{ + "module": "mesh-delivery-checker", + "version": "1", + "slug": "checker", + "consumes": [ + "mesh-build-go.toolchain.reported", + "mesh-build-node.toolchain.reported" + ], + "state": [ + { "name": "kept", "history": 1 }, + { "name": "owed", "history": 5 }, + { "name": "previews", "history": 5 }, + { "name": "verdicts", "history": 5 } + ], + "reads": [ + "mesh-build-go.steps", + "mesh-build-go.toolchains", + "mesh-build-node.steps", + "mesh-build-node.toolchains", + "mesh-controller.plans-building", + "mesh-delivery-builder.builds", + "mesh-delivery-gate.source", + "mesh-delivery-indexer.candidates", + "mesh-delivery-indexer.manifests", + "mesh-delivery-indexer.modules", + "mesh-delivery-watchman.presence", + "node-engine.have" + ], + "invokes": [ + "mesh-build-go.build_go_digest", + "mesh-build-go.build_go_fetch", + "mesh-build-go.build_go_test", + "mesh-build-node.build_node_fetch", + "mesh-build-node.build_node_test" + ], + "config": { + "capacity": { + "type": "string", + "description": "How many build steps the checker runs at once on each node, as node=steps pairs separated by commas, such as shanks=2,ace=1 (ADR 0484 clause 12); a node set to 0 never checks, a node not named runs one at a time, and empty is one everywhere. The builder's capacity is the builder's own.", + "pattern": "|[a-z0-9_-]+=[0-9]+(,[a-z0-9_-]+=[0-9]+)*", + "default": "" + } + }, + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-delivery-checker", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-delivery-checker", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-delivery-checker" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-delivery-checker", + "place": ".", + "unknown": "program" + }, + { + "id": "settings", + "trusted": false, + "type": "file", + "path": "${dir:state}/settings.json", + "mode": "0600", + "owner": "mesh-delivery-checker", + "takes-config": ["capacity"], + "merge": "json", + "template": "files/settings.json" + }, + { + "id": "checker", + "type": "process", + "name": "mesh-delivery-checker", + "artifact": "program", + "run": [ + "./mesh-delivery-checker" + ], + "env": { + "MESH_BROKER_FILE": "${dir:state}/broker", + "MESH_CHECKER_SETTINGS": "${dir:state}/settings.json" + }, + "restart-on": [ + "settings" + ], + "user": "mesh-delivery-checker", + "health": { + "kind": "unit" + } + } + ], + "build": { + "artifacts": [ + { + "name": "program", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-delivery-checker", + "binary": "mesh-delivery-checker" + }, + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } +} From cfbd5b6d2d4616e70d60f0214e193c1c897a8838 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 21:11:43 +0200 Subject: [PATCH 03/19] Judge a head merged onto its trunk through the build modules: run only packages without a kept verdict, the change's own tests first, stop at the first failure (issues 1679, 1288, 1289) --- .../cmd/mesh-delivery-checker/judge.go | 96 +++- .../cmd/mesh-delivery-checker/judge_test.go | 72 +++ .../cmd/mesh-delivery-checker/main.go | 15 +- .../internal/check/beside.go | 62 +++ .../internal/check/checker.go | 2 + .../internal/check/examiner_test.go | 456 ++++++++++++++++++ .../internal/check/judge.go | 455 +++++++++++++++++ .../internal/check/testdata/captured/README | 9 + .../check/testdata/captured/go-digest.json | 11 + .../testdata/captured/go-test-empty.json | 21 + .../testdata/captured/go-test-failing.json | 45 ++ .../check/testdata/captured/go-test-slow.json | 30 ++ 12 files changed, 1261 insertions(+), 13 deletions(-) create mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go create mode 100644 modules/mesh-delivery-checker/internal/check/beside.go create mode 100644 modules/mesh-delivery-checker/internal/check/examiner_test.go create mode 100644 modules/mesh-delivery-checker/internal/check/judge.go create mode 100644 modules/mesh-delivery-checker/internal/check/testdata/captured/README create mode 100644 modules/mesh-delivery-checker/internal/check/testdata/captured/go-digest.json create mode 100644 modules/mesh-delivery-checker/internal/check/testdata/captured/go-test-empty.json create mode 100644 modules/mesh-delivery-checker/internal/check/testdata/captured/go-test-failing.json create mode 100644 modules/mesh-delivery-checker/internal/check/testdata/captured/go-test-slow.json diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go index 4b4a008..b80e7b0 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go @@ -2,9 +2,11 @@ package main import ( "context" - "errors" - "log" + "sort" + "sync" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "mesh-delivery-checker/internal/check" @@ -12,7 +14,6 @@ import ( // judgeDeps are the parts the judge of a check is made of. type judgeDeps struct { - checker *check.Checker runner buildrun.Runner tools *buildrun.ToolchainSet presence *buildrun.PresenceSet @@ -21,15 +22,90 @@ type judgeDeps struct { trunks *check.Trunks kept check.KV previews check.KV + repos *repoModules logf func(format string, args ...any) } -var _ = log.Printf - -type unbuilt struct{} - -func (unbuilt) Judge(context.Context, *check.Case) (check.Outcome, error) { - return check.Outcome{}, errors.New("the judge of a check is not built yet") +func newJudge(d judgeDeps) check.Judge { + return &check.Examiner{Steps: d.runner, Toolchains: d.tools, Presence: d.presence, Health: d.have, Places: d.places, + Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Trunks: d.trunks, Logf: d.logf} } -func newJudge(judgeDeps) check.Judge { return unbuilt{} } +// manifestKV reads a manifest by its digest. +type manifestKV interface { + Get(ctx context.Context, key string) (value []byte, revision uint64, found bool, err error) +} + +// repoModules knows which build modules judge a repository, from the indexer's modules and their manifests: the Go build +// module always (a repository's go.mod files are its own), the Node one when a module of the repository has a TypeScript +// artifact (ADR 0484 clause 10). Safe for use from the watch and from a check at once. +type repoModules struct { + manifests manifestKV + logf func(format string, args ...any) + + mu sync.Mutex + langs map[string]map[string]bool // by manifest digest: the languages of its artifacts + repoOf map[string]string // by module: its repository + typed map[string]string // by module: its manifest digest +} + +// Put takes a change of the indexer's modules bucket in: a module's repository and the languages its manifest names. +func (r *repoModules) Put(ctx context.Context, key string, value []byte, deleted bool) { + r.mu.Lock() + if r.repoOf == nil { + r.repoOf, r.typed, r.langs = map[string]string{}, map[string]string{}, map[string]map[string]bool{} + } + if deleted { + delete(r.repoOf, key) + delete(r.typed, key) + r.mu.Unlock() + return + } + r.mu.Unlock() + m, err := delivery.DecodeModule(value) + if err != nil { + return + } + r.mu.Lock() + r.repoOf[key] = m.Repository + r.typed[key] = m.Manifest + _, known := r.langs[m.Manifest] + r.mu.Unlock() + if m.Manifest == "" || known { + return + } + set := map[string]bool{} + if mk, err := delivery.ManifestsKey(m.Manifest); err == nil { + if raw, _, found, err := r.manifests.Get(ctx, mk); err == nil && found { + if kept, err := delivery.DecodeManifest(raw); err == nil { + if man, err := catalogue.ParseManifest([]byte(kept.JSON)); err == nil && man.Build != nil { + for _, a := range man.Build.Artifacts { + if a.Language != "" { + set[a.Language] = true + } + } + } else if err != nil && r.logf != nil { + r.logf("the manifest %s cannot be read, so its module counts as no TypeScript: %v", m.Manifest, err) + } + } + } + } + r.mu.Lock() + r.langs[m.Manifest] = set + r.mu.Unlock() +} + +// BuildModules are the build modules that judge a repository, sorted. +func (r *repoModules) BuildModules(repository string) []string { + out := []string{buildrun.GoBuildModule} + r.mu.Lock() + defer r.mu.Unlock() + for module, repo := range r.repoOf { + if repo == repository && r.langs[r.typed[module]]["typescript"] { + out = append(out, buildrun.NodeBuildModule) + break + } + } + sort.Strings(out) + return out +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go new file mode 100644 index 0000000..ba3c1c0 --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go @@ -0,0 +1,72 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" +) + +type memManifests map[string][]byte + +func (m memManifests) Get(_ context.Context, key string) ([]byte, uint64, bool, error) { + v, ok := m[key] + return v, 1, ok, nil +} + +func putModule(t *testing.T, r *repoModules, name, repository, manifest string) { + t.Helper() + data, err := delivery.EncodeModule(delivery.Module{Repository: repository, Commit: strings.Repeat("a", 40), FP: "src1:" + strings.Repeat("a", 64), Manifest: delivery.ManifestDigest([]byte(manifest))}) + if err != nil { + t.Fatal(err) + } + r.Put(context.Background(), name, data, false) +} + +func TestARepositoryWithATypeScriptModuleIsJudgedByTheNodeBuildModuleToo(t *testing.T) { // 1292/AC2 + own, err := os.ReadFile("../../module.json") // a manifest with a Go bundle: this module's own + if err != nil { + t.Fatal(err) + } + goManifest := string(own) + // A TypeScript bundle names neither a system nor a directory it is built from (the catalogue's parser says so). + var doc map[string]any + if err := json.Unmarshal(own, &doc); err != nil { + t.Fatal(err) + } + artifacts := doc["build"].(map[string]any)["artifacts"].([]any) + program := artifacts[0].(map[string]any) + program["language"] = "typescript" + delete(program, "system") + delete(program, "from") + delete(program, "binary") + tsRaw, err := json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + tsManifest := string(tsRaw) + store := memManifests{} + for _, m := range []string{goManifest, tsManifest} { + data, err := delivery.EncodeManifest(delivery.Manifest{Module: "demo", JSON: m}) + if err != nil { + t.Fatal(err) + } + store[delivery.ManifestDigest([]byte(m))] = data + } + r := &repoModules{manifests: store} + putModule(t, r, "mesh-delivery-checker", "novox/mesh-delivery", goManifest) + putModule(t, r, "mesh-ui", "novox/mesh-catalog", tsManifest) + if got := r.BuildModules("novox/mesh-delivery"); strings.Join(got, ",") != buildrun.GoBuildModule { + t.Errorf("mesh-delivery: %v, want the Go build module alone", got) + } + if got := r.BuildModules("novox/mesh-catalog"); strings.Join(got, ",") != buildrun.GoBuildModule+","+buildrun.NodeBuildModule { + t.Errorf("mesh-catalog: %v, want both", got) + } + if got := r.BuildModules("novox/mesh-sdk"); strings.Join(got, ",") != buildrun.GoBuildModule { + t.Errorf("a repository with no module: %v, want the Go build module", got) + } +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index 6faacfa..9fc4703 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -192,10 +192,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Link: link, Logf: logger.Printf, } + repos := &repoModules{manifests: kvStore{buckets[delivery.ManifestsBucket]}, logf: logger.Printf} ch.Judge = newJudge(judgeDeps{ - checker: ch, runner: runner, tools: tools, presence: presence, have: have, places: places, trunks: trunks, + runner: runner, tools: tools, presence: presence, have: have, places: places, trunks: trunks, kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]}, - logf: logger.Printf, + repos: repos, logf: logger.Printf, }) // The toolchains first, so a check that starts from the first candidate finds the nodes. @@ -220,7 +221,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } defer cc.Stop() } - errs := make(chan error, 6) + errs := make(chan error, 8) // The watchman's presence says which nodes are asleep. Where its bucket is not there yet no node is online, so nothing // is checked; the checker says so and is started again with the watchman. switch kv, err := js.KeyValue(ctx, delivery.PresenceBucket); { @@ -285,6 +286,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro }) }() } + // The modules say which repository holds a TypeScript module, so the Node build module judges it too. + mw, err := settle(ctx, buckets[delivery.ModulesBucket], func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) }) + if err != nil { + return err + } + go func() { + errs <- follow(ctx, mw, func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) }) + }() each := func(e jetstream.KeyValueEntry) { if opOf(e) { ch.Candidate(ctx, e.Key(), e.Value()) diff --git a/modules/mesh-delivery-checker/internal/check/beside.go b/modules/mesh-delivery-checker/internal/check/beside.go new file mode 100644 index 0000000..f18ae1a --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/beside.go @@ -0,0 +1,62 @@ +package check + +import "strconv" + +// BesideRef is a sibling repository a repository's tests read, placed beside the fetched tree as a plain directory (ADR +// 0484 clause 4). Ref names the pull request's head when the commit is only there. +type BesideRef struct { + Repository string `json:"repository"` + Commit string `json:"commit"` + Ref string `json:"ref,omitempty"` +} + +// besideRules are the repositories each core repository's tests read, as the tests state it (mesh-sdk go/internal/beside +// beside.go): the catalogue at its trunk, the node-engine, the SDK as the controller pins it, and the delivery's pinned +// forge dependencies (mesh-catalog checks/deps). The checker chooses which sibling at which commit; the build module only +// places what it is told. +var besideRules = map[string][]string{ + "novox/mesh-controller": {"novox/mesh-catalog", "novox/mesh-host", "novox/mesh-sdk"}, + "novox/mesh-host": {"novox/mesh-catalog", "novox/mesh-sdk"}, + "novox/mesh-sdk": {"novox/mesh-catalog", "novox/mesh-host"}, + "novox/mesh-catalog": {"novox/mesh-sdk", "novox/mesh-controller"}, + "novox/mesh-delivery": {"novox/mesh-sdk", "novox/mesh-catalog"}, +} + +// Beside are the siblings a pull request's tests read: each at its trunk head, and at the head of the group's pull request +// when the group holds one of that repository (a group's members are checked together, ADR 0482 clause 2) — except the +// catalogue, which the tests read at its trunk even when the group holds a catalogue pull request (beside.go). +func (j *Examiner) Beside(c *Case, p Pull) []BesideRef { + var heads map[string]string + if j.Trunks != nil { + heads = j.Trunks.Heads() + } + var out []BesideRef + for _, repo := range besideRules[p.Repository] { + if member := c.member(repo); member != nil && repo != "novox/mesh-catalog" { + out = append(out, BesideRef{Repository: repo, Commit: member.Head, Ref: refOf(member.Number)}) + continue + } + if head := heads[repo]; head != "" { + out = append(out, BesideRef{Repository: repo, Commit: head}) + } + } + return out +} + +func (c *Case) member(repository string) *Pull { + for i := range c.Pulls { + if c.Pulls[i].Repository == repository { + return &c.Pulls[i] + } + } + return nil +} + +func refOf(number int) string { + if number <= 0 { + return "" + } + return "refs/pull/" + itoa(number) + "/head" +} + +func itoa(n int) string { return strconv.Itoa(n) } diff --git a/modules/mesh-delivery-checker/internal/check/checker.go b/modules/mesh-delivery-checker/internal/check/checker.go index 5706ae4..9084403 100644 --- a/modules/mesh-delivery-checker/internal/check/checker.go +++ b/modules/mesh-delivery-checker/internal/check/checker.go @@ -23,6 +23,8 @@ const NodesWords = "build node qualifies" type Outcome struct { Verdict delivery.Verdict Builds []BuildNote + // Slowest names the slowest tests of a level that went over its bound ("unit pkg.Test 1.2s"). + Slowest []string } // BuildNote is the state of a candidate's module build, by module: built, failed or waiting (ADR 0484 clause 7). diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go new file mode 100644 index 0000000..446ee9c --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -0,0 +1,456 @@ +package check + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" +) + +// captured reads a result the build module's own code marshalled (testdata/captured/README). +func captured[T any](t testing.TB, name string) T { + t.Helper() + raw, err := os.ReadFile(filepath.Join("testdata", "captured", name+".json")) + if err != nil { + t.Fatal(err) + } + var v T + if err := json.Unmarshal(raw, &v); err != nil { + t.Fatal(err) + } + return v +} + +type stepCall struct { + module, node, tool string + args map[string]any +} + +// fakeSteps stands for the build modules: a handler for each tool answers what the module's code answers. +type fakeSteps struct { + mu sync.Mutex + calls []stepCall + handlers map[string]func(c stepCall) (any, error) +} + +func (f *fakeSteps) Run(_ context.Context, module, node, tool string, args map[string]any) (json.RawMessage, error) { + c := stepCall{module, node, tool, args} + f.mu.Lock() + f.calls = append(f.calls, c) + h := f.handlers[tool] + f.mu.Unlock() + if h == nil { + return nil, fmt.Errorf("the fake build module has no tool %s", tool) + } + v, err := h(c) + if err != nil { + return nil, err + } + return json.Marshal(v) +} + +func (f *fakeSteps) tool(name string) []stepCall { + f.mu.Lock() + defer f.mu.Unlock() + var out []stepCall + for _, c := range f.calls { + if strings.HasSuffix(c.tool, "_"+name) { + out = append(out, c) + } + } + return out +} + +type entries []delivery.NodeToolchain + +func (e entries) Entries(language string) []delivery.NodeToolchain { + var out []delivery.NodeToolchain + for _, x := range e { + if x.Language == language { + out = append(out, x) + } + } + return out +} + +type langs map[string][]string + +func (l langs) BuildModules(repository string) []string { + if m, ok := l[repository]; ok { + return m + } + return []string{buildrun.GoBuildModule} +} + +func goEntries(nodes ...string) entries { + var e entries + for _, n := range nodes { + e = append(e, delivery.NodeToolchain{Node: n, Toolchain: delivery.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "7"}}) + e = append(e, delivery.NodeToolchain{Node: n, Toolchain: delivery.Toolchain{Language: "typescript", Version: "node24", Recipe: "3"}}) + } + return e +} + +const digC = "cc33cc33cc33cc33cc33cc33cc33cc33cc33cc33cc33cc33cc33cc33cc33cc33" +const digD = "dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44" + +type bench struct { + ex *Examiner + steps *fakeSteps + kept *memKV + prev *memKV + fetch Fetched + digest map[string]Digested // by level + test func(c stepCall) (any, error) +} + +// newBench is an examiner over fake build modules that merge cleanly, digest the captured fixture's packages and pass +// the captured slow fixture's result at both levels, on shanks. +func newBench(t testing.TB) *bench { + b := &bench{steps: &fakeSteps{}, kept: newMemKV(), prev: newMemKV()} + b.fetch = Fetched{Workspace: "ws1", Commit: head1, Modules: []string{"."}, Base: trunkA, Merge: "9999999999999999999999999999999999999999", + Changed: []string{"README.md"}} + one := captured[Digested](t, "go-digest") + b.digest = map[string]Digested{ + delivery.LevelUnit: {Level: "unit", Packages: map[string]string{"fixture/slow": digA}}, + delivery.LevelIntegration: {Level: "integration", Packages: map[string]string{"fixture/slow": digB}}, + } + _ = one + slow := captured[Tested](t, "go-test-slow") + b.test = func(c stepCall) (any, error) { + r := slow + r.Level, _ = c.args["level"].(string) + return r, nil + } + b.steps.handlers = map[string]func(stepCall) (any, error){ + "build_go_fetch": func(stepCall) (any, error) { return b.fetch, nil }, + "build_node_fetch": func(stepCall) (any, error) { return b.fetch, nil }, + "build_go_digest": func(c stepCall) (any, error) { return b.digest[c.args["level"].(string)], nil }, + "build_go_test": func(c stepCall) (any, error) { return b.test(c) }, + "build_node_test": func(c stepCall) (any, error) { return b.test(c) }, + } + now := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC) + b.ex = &Examiner{Steps: b.steps, Toolchains: goEntries("shanks"), Places: &buildrun.Places{}, Kept: Kept{Store: b.kept}, Previews: b.prev, + Modules: langs{}, Trunks: &Trunks{}, Now: func() time.Time { now = now.Add(50 * time.Millisecond); return now }} + return b +} + +func caseOf(t testing.TB, pulls ...delivery.CandidatePull) *Case { + t.Helper() + raw := candidateOf(t, "feat/x", pulls...) + c, err := delivery.DecodeCandidate(raw) + if err != nil { + t.Fatal(err) + } + cs := CaseOf(c, map[string]string{repoA: trunkA, repoSDK: trunkS}) + if cs == nil { + t.Fatal("no case") + } + return cs +} + +func (b *bench) judge(t testing.TB, cs *Case) Outcome { + t.Helper() + out, err := b.ex.Judge(context.Background(), cs) + if err != nil { + t.Fatal(err) + } + return out +} + +func testedWith(t testing.TB, name string, edit func(*Tested)) Tested { + r := captured[Tested](t, name) + if edit != nil { + edit(&r) + } + return r +} + +func packagesOf(c stepCall) []string { + var out []string + if l, ok := c.args["packages"].([]string); ok { + out = l + } + return out +} + +func TestACleanMergeIsFetchedAtItsTrunkAndBothLevelsPass(t *testing.T) { // 1679/AC1 + b := newBench(t) + cs := caseOf(t, pullOf(repoA, 7, head1)) + out := b.judge(t, cs) + v := out.Verdict + if v.State != delivery.VerdictPass || v.Levels[delivery.LevelUnit].Run != 2 || v.Levels[delivery.LevelIntegration].Run != 2 { + t.Fatalf("verdict = %+v", v) + } + if v.Composition == nil || v.Composition.State != delivery.CompositionNotRun { + t.Errorf("composition = %+v, want not run", v.Composition) + } + if len(v.Machines) != 1 || v.Machines[0] != "shanks" { + t.Errorf("machines = %v", v.Machines) + } + fetch := b.steps.tool("fetch") + if len(fetch) != 1 || fetch[0].args["repository"] != repoA || fetch[0].args["commit"] != head1 || fetch[0].args["base"] != trunkA || fetch[0].args["ref"] != "refs/pull/7/head" { + t.Fatalf("fetch calls = %+v", fetch) + } + if v.Took.Clone <= 0 || v.Took.Unit <= 0 { + t.Errorf("took = %+v, want the clone and the unit level timed", v.Took) + } + // What passed is kept under each level's digest. + for _, k := range []string{"unit." + digA, "integration." + digB} { + if _, _, found, _ := b.kept.Get(context.Background(), k); !found { + t.Errorf("%s was not kept", k) + } + } +} + +func TestOnlyThePackageWithoutAKeptVerdictRuns(t *testing.T) { // 1288/AC1 + b := newBench(t) + b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"fixture/slow": digA, "fixture/other": digC}} + if _, err := (Kept{Store: b.kept}).Keep(context.Background(), "unit", digC, keptValue("fixture/other", 40)); err != nil { + t.Fatal(err) + } + out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + tests := b.steps.tool("test") + if len(tests) != 2 || strings.Join(packagesOf(tests[0]), ",") != "fixture/slow" || tests[0].args["level"] != "unit" { + t.Fatalf("test calls = %+v, want the unit level of fixture/slow alone, then the integration level", tests) + } + if u := out.Verdict.Levels[delivery.LevelUnit]; u.Run != 2 || u.Kept != 40 { + t.Errorf("unit = %+v, want 2 run and 40 kept", u) + } +} + +func TestAReAskRunsOnlyWhatFailedAndWhatChanged(t *testing.T) { // 1288/AC2, AC3 + b := newBench(t) + failing := captured[Tested](t, "go-test-failing") + b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"fixture/failing/bad": digA, "fixture/failing/good": digB}} + b.test = func(c stepCall) (any, error) { + if c.args["level"] == "unit" { + return failing, nil + } + return captured[Tested](t, "go-test-slow"), nil + } + cs := caseOf(t, pullOf(repoA, 7, head1)) + first := b.judge(t, cs).Verdict + if first.State != delivery.VerdictFail || first.Failed.Test != "unit: fixture/failing/bad.TestThreeIsFour" { + t.Fatalf("first = %+v", first) + } + if _, _, found, _ := b.kept.Get(context.Background(), "unit."+digB); !found { + t.Fatal("the package that passed in the failing run was not kept") + } + if _, _, found, _ := b.kept.Get(context.Background(), "unit."+digA); found { + t.Fatal("the package that failed was kept") + } + // Asked again with nothing changed: only the failed package runs; the good one is taken. + b.steps.calls = nil + b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + tests := b.steps.tool("test") + if len(tests) == 0 || strings.Join(packagesOf(tests[0]), ",") != "fixture/failing/bad" { + t.Fatalf("the re-ask ran %+v, want only fixture/failing/bad", tests) + } + // The trunk moved and one digest changed: that package runs, the unchanged one is taken. + b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"fixture/failing/bad": digA, "fixture/failing/good": digD}} + b.steps.calls = nil + b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + tests = b.steps.tool("test") + if got := strings.Join(packagesOf(tests[0]), ","); got != "fixture/failing/bad,fixture/failing/good" && got != "fixture/failing/good,fixture/failing/bad" { + t.Errorf("after a digest changed the run was %q, want the failed package and the changed one", got) + } +} + +func TestANodePackageIsNeverKeptAndRunsEveryCheck(t *testing.T) { // 1288/AC4, 1292/AC1 + b := newBench(t) + b.ex.Modules = langs{repoA: {buildrun.NodeBuildModule}} + b.fetch.Modules = []string{"modules/ui"} + b.test = func(c stepCall) (any, error) { + return Tested{Level: c.args["level"].(string), Module: "modules/ui", Passed: true, Elapsed: 0.4, Bound: 5}, nil + } + cs := caseOf(t, pullOf(repoA, 7, head1)) + for i := 0; i < 2; i++ { + v := b.judge(t, cs).Verdict + if v.State != delivery.VerdictPass || v.Levels[delivery.LevelUnit].Kept != 0 || v.Levels[delivery.LevelUnit].Run != 1 { + t.Fatalf("round %d: verdict = %+v", i, v) + } + } + if n := len(b.steps.tool("test")); n != 4 { + t.Errorf("%d node test calls in two checks, want 4 (both levels each time)", n) + } + if n := len(b.steps.tool("digest")); n != 0 { + t.Errorf("the node build module was asked for a digest %d times", n) + } + if keys, _ := b.kept.Keys(context.Background()); len(keys) != 0 { + t.Errorf("kept %v for a node package", keys) + } +} + +func TestTheChangesOwnFailingTestRunsFirstAndNothingElseRuns(t *testing.T) { // 1289/AC1, AC2 + b := newBench(t) + b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"} + b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}} + failing := captured[Tested](t, "go-test-failing") + b.test = func(c stepCall) (any, error) { + if got := packagesOf(c); len(got) == 1 && got[0] == "m/internal/b" { + return failing, nil + } + return captured[Tested](t, "go-test-slow"), nil + } + out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + if out.Verdict.State != delivery.VerdictFail || !strings.Contains(out.Verdict.Failed.Test, "TestThreeIsFour") { + t.Fatalf("verdict = %+v", out.Verdict) + } + tests := b.steps.tool("test") + if len(tests) != 1 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" { + t.Fatalf("test calls = %+v, want only m/internal/b, once", tests) + } + if n := len(b.steps.tool("digest")); n != 1 { + t.Errorf("%d digest calls, want the unit level's alone (the check stopped)", n) + } + if out.Verdict.Composition == nil || out.Verdict.Composition.State != delivery.CompositionNotRun { + t.Errorf("composition = %+v", out.Verdict.Composition) + } +} + +func TestTheChangesOwnPassingTestRunsFirstAndTheRestFollow(t *testing.T) { // 1289/AC3 + b := newBench(t) + b.fetch.Changed = []string{"modules/m/internal/b/b_test.go", "modules/m/internal/a/a.go"} + b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}} + out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + if out.Verdict.State != delivery.VerdictPass { + t.Fatalf("verdict = %+v", out.Verdict) + } + var unit []string + for _, c := range b.steps.tool("test") { + if c.args["level"] == "unit" { + unit = append(unit, strings.Join(packagesOf(c), ",")) + } + } + if strings.Join(unit, " | ") != "m/internal/b | m/internal/a" { + t.Errorf("the unit level ran %v, want the changed test's package first, then the rest", unit) + } +} + +func TestAHeadThatDoesNotMergeEndsFailedNamingTheFilesAndRunsNoTest(t *testing.T) { // 1289/AC4 + b := newBench(t) + b.fetch = Fetched{Workspace: "ws1", Commit: head1, Base: trunkA, Conflicts: []string{"go.mod", "internal/x.go"}} + out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + f := out.Verdict.Failed + if out.Verdict.State != delivery.VerdictFail || f.Machine != "shanks" || !strings.Contains(f.Words, "files in conflict: go.mod, internal/x.go") { + t.Fatalf("verdict = %+v", out.Verdict) + } + if n := len(b.steps.tool("digest")) + len(b.steps.tool("test")); n != 0 { + t.Errorf("%d digest and test steps ran after a conflict", n) + } +} + +func TestNoQualifyingNodeIsAFailureOfTheNodes(t *testing.T) { // N6/AC4 + b := newBench(t) + b.ex.Toolchains = entries{} + out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + if !IsNodesFailure(out.Verdict) { + t.Fatalf("verdict = %+v, want a failure of the nodes", out.Verdict) + } + b.ex.Toolchains = goEntries("shanks") + b.ex.Presence = offline{} + if out = b.judge(t, caseOf(t, pullOf(repoA, 7, head1))); !IsNodesFailure(out.Verdict) { + t.Fatalf("an offline node: verdict = %+v", out.Verdict) + } +} + +type offline struct{} + +func (offline) Online(string) bool { return false } + +func TestAStepThatFailsIsAVerdictNamingTheNode(t *testing.T) { + b := newBench(t) + b.steps.handlers["build_go_fetch"] = func(stepCall) (any, error) { return nil, fmt.Errorf("the step failed: ssh refused") } + out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + if f := out.Verdict.Failed; out.Verdict.State != delivery.VerdictFail || f.Machine != "shanks" || !strings.Contains(f.Words, "ssh refused") { + t.Fatalf("verdict = %+v", out.Verdict) + } +} + +func TestALevelWhoseDigestIsNotNamedRunsWholeAndKeepsNothing(t *testing.T) { // 1287/AC4 + b := newBench(t) + b.steps.handlers["build_go_digest"] = func(stepCall) (any, error) { return nil, fmt.Errorf("go list failed") } + out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + if out.Verdict.State != delivery.VerdictPass { + t.Fatalf("verdict = %+v", out.Verdict) + } + tests := b.steps.tool("test") + if len(tests) != 2 || tests[0].args["packages"] != nil { + t.Errorf("test calls = %+v, want each level once, whole", tests) + } + if keys, _ := b.kept.Keys(context.Background()); len(keys) != 0 { + t.Errorf("kept %v with no digest", keys) + } +} + +func TestALevelOverItsBoundIsNamedWithItsSlowestTestsAndKeepsItsVerdict(t *testing.T) { // 1300/AC3 + b := newBench(t) + b.test = func(c stepCall) (any, error) { + return testedWith(t, "go-test-slow", func(r *Tested) { + r.Level = c.args["level"].(string) + r.OverBound = true + r.OverWhich = "run" + r.Elapsed = 7 + }), nil + } + out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + v := out.Verdict + if v.State != delivery.VerdictPass || strings.Join(v.OverBound, ",") != "unit,integration" { + t.Fatalf("verdict = %+v", v) + } + if len(out.Slowest) == 0 || !strings.Contains(out.Slowest[0], "TestTakesAWhile") { + t.Errorf("slowest = %v", out.Slowest) + } +} + +func TestSkippedTestsAreCountedPerLevel(t *testing.T) { // 1300/AC6, 1251 + b := newBench(t) + b.test = func(c stepCall) (any, error) { + return testedWith(t, "go-test-slow", func(r *Tested) { r.Level = c.args["level"].(string); r.Skipped = 3 }), nil + } + v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict + if v.Levels[delivery.LevelUnit].Skipped != 3 || v.Levels[delivery.LevelIntegration].Skipped != 3 { + t.Errorf("levels = %+v", v.Levels) + } +} + +func TestAGroupIsOneVerdictAndEachMembersTestsReadTheOthersAtTheirHeads(t *testing.T) { // 1624/AC1 + b := newBench(t) + cs := caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2)) + v := b.judge(t, cs).Verdict + if v.State != delivery.VerdictPass { + t.Fatalf("verdict = %+v", v) + } + fetch := b.steps.tool("fetch") + if len(fetch) != 2 { + t.Fatalf("%d fetches, want one for each member", len(fetch)) + } + var forA []BesideRef + for _, f := range fetch { + if f.args["repository"] == repoA { + forA, _ = f.args["beside"].([]BesideRef) + } + } + found := false + for _, r := range forA { + if r.Repository == repoSDK && r.Commit == head2 && r.Ref == "refs/pull/12/head" { + found = true + } + } + if !found { + t.Errorf("mesh-delivery's tests were not given the SDK member at its head: %+v", forA) + } +} + +func TestAnExaminerIsAJudgeOfTheChecker(t *testing.T) { + var _ Judge = (*Examiner)(nil) +} diff --git a/modules/mesh-delivery-checker/internal/check/judge.go b/modules/mesh-delivery-checker/internal/check/judge.go new file mode 100644 index 0000000..a6ac48c --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/judge.go @@ -0,0 +1,455 @@ +package check + +import ( + "context" + "encoding/json" + "fmt" + "path" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" +) + +// Steps runs a build module's long step on a node and answers its result (buildrun.Runner). +type Steps interface { + Run(ctx context.Context, module, node, tool string, args map[string]any) (json.RawMessage, error) +} + +// ToolchainEntries are the nodes' toolchain entries of a language (buildrun.ToolchainSet). +type ToolchainEntries interface { + Entries(language string) []delivery.NodeToolchain +} + +// BuildModules says which build modules judge a repository: the Go one always, the Node one when the repository holds a +// TypeScript module. +type BuildModules interface { + BuildModules(repository string) []string +} + +// toolPrefix is the start of each build module's tool names. +var toolPrefix = map[string]string{buildrun.GoBuildModule: "build_go_", buildrun.NodeBuildModule: "build_node_"} + +// Examiner judges a case through the build modules (ADR 0484 clauses 3, 4, 10, 13): for each pull request it fetches the head +// merged onto its trunk head on a build node, then runs the unit level and the integration level, taking the kept verdicts +// of unchanged packages, running the change's own tests first and stopping at the first failure. +type Examiner struct { + Steps Steps + Toolchains ToolchainEntries + // Presence says which nodes are online; Health whether a node's build module is running and healthy. A node that is not + // is not chosen; nil means every node is. + Presence interface{ Online(node string) bool } + Health interface { + Healthy(node, module string) bool + } + // Places holds a step back while its node is at the checker's capacity (ADR 0484 clause 12). + Places *buildrun.Places + Kept Kept + // Previews is the previews bucket; nil writes none. + Previews KV + Modules BuildModules + // Trunks are the trunk heads of every repository, for the siblings a repository's tests read. + Trunks interface{ Heads() map[string]string } + Now func() time.Time + Logf func(format string, args ...any) +} + +func (j *Examiner) now() time.Time { + if j.Now != nil { + return j.Now() + } + return time.Now() +} + +func (j *Examiner) logf(format string, args ...any) { + if j.Logf != nil { + j.Logf(format, args...) + } +} + +// target is a pull request on one build module: the node chosen, the workspace its fetch made and what it said. +type target struct { + pull Pull + module string + node string + toolchain string + fetched Fetched + // priority are the Go packages (import paths) or the Node module directories whose test files the merge changes. + changed []string + plans map[string]*goPlan // by level +} + +// goPlan is what a Go target has to run at one level: the digests, the packages without a kept verdict and the tests taken. +type goPlan struct { + digests map[string]string + todo []string + // all says the digest step gave nothing to key a kept verdict by, so the level runs whole and keeps nothing. + all bool +} + +// acc collects a level's counts across targets. +type acc struct { + run, kept, skipped int + elapsed, setup float64 + over bool + slowest []TestTime +} + +// state is one judging. +type state struct { + j *Examiner + c *Case + targets []*target + levels map[string]*acc + nodes map[string]bool + clone time.Duration + slow []string +} + +// Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot +// be judged at all. +func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) { + st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}} + if v := st.fetchAll(ctx); v != nil { + return st.outcome(*v), nil + } + for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} { + for _, priority := range []bool{true, false} { + for _, t := range st.targets { + if v := st.runTarget(ctx, t, level, priority); v != nil { + return st.outcome(*v), nil + } + } + } + } + return st.outcome(delivery.Verdict{State: delivery.VerdictPass}), nil +} + +// outcome fills the verdict's levels, times, machines and composition from what ran. +func (st *state) outcome(v delivery.Verdict) Outcome { + if v.State == delivery.VerdictPass || len(v.Levels) == 0 { + v.Levels = map[string]delivery.LevelResult{} + for level, a := range st.levels { + v.Levels[level] = delivery.LevelResult{Run: a.run, Kept: a.kept, Skipped: a.skipped} + } + } + ms := func(s float64) int64 { return int64(s * 1000) } + u, i := st.levels[delivery.LevelUnit], st.levels[delivery.LevelIntegration] + v.Took = delivery.Took{Clone: st.clone.Milliseconds(), Unit: ms(u.elapsed), IntegrationSetup: ms(i.setup), IntegrationRun: ms(i.elapsed)} + for node := range st.nodes { + v.Machines = append(v.Machines, node) + } + sort.Strings(v.Machines) + for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} { + if st.levels[level].over { + v.OverBound = append(v.OverBound, level) + } + } + // The composition is the assigner's check (issue 1297): until it delivers a verdict says it was not run (ADR 0484 clause 6). + v.Composition = &delivery.Composition{State: delivery.CompositionNotRun} + var slow []string + for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} { + a := st.levels[level] + if !a.over { + continue + } + sort.SliceStable(a.slowest, func(x, y int) bool { return a.slowest[x].Elapsed > a.slowest[y].Elapsed }) + for n, s := range a.slowest { + if n == 3 { + break + } + slow = append(slow, fmt.Sprintf("%s %s %.1fs", level, path.Base(s.Package)+"."+s.Test, s.Elapsed)) + } + } + return Outcome{Verdict: v, Slowest: slow} +} + +// fail is a verdict of failure at a machine. +func fail(machine, words string) *delivery.Verdict { + v := failure(machine, words) + return &v +} + +// step runs one step of a build module on the node the target is on, holding a place there while it runs. +func (st *state) step(ctx context.Context, t *target, tool, label string, args map[string]any) (json.RawMessage, error) { + st.c.Progress.Set(label, t.node) + release, err := st.j.Places.Acquire(ctx, t.node) + if err != nil { + return nil, err + } + defer release() + return st.j.Steps.Run(ctx, t.module, t.node, toolPrefix[t.module]+tool, args) +} + +// choose picks the build node for a pull request's repository on a build module. +func (st *state) choose(bm string) (node, toolchain string, err error) { + language := buildrun.LanguageOf[bm] + qualifies := func(node string) bool { + return (st.j.Presence == nil || st.j.Presence.Online(node)) && (st.j.Health == nil || st.j.Health.Healthy(node, bm)) + } + nodes, tc, err := buildrun.ChooseNodes(language, st.j.Toolchains.Entries(language), st.j.Places, qualifies, 1) + if err != nil { + return "", "", err + } + return nodes[0], tc.Version, nil +} + +// fetchAll fetches every pull request's head merged onto its trunk on a build node of each build module the repository +// needs. A head that does not merge, or a step that fails, ends the check at once, running no test (ADR 0484 clause 13). +func (st *state) fetchAll(ctx context.Context) *delivery.Verdict { + began := st.j.now() + defer func() { st.clone = st.j.now().Sub(began) }() + for _, p := range st.c.Pulls { + for _, bm := range st.j.Modules.BuildModules(p.Repository) { + node, tc, err := st.choose(bm) + if err != nil { + return fail(bm, err.Error()) + } + t := &target{pull: p, module: bm, node: node, toolchain: tc, plans: map[string]*goPlan{}} + st.nodes[node] = true + args := map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number), "base": p.Trunk} + if bm == buildrun.GoBuildModule { + if beside := st.j.Beside(st.c, p); len(beside) > 0 { + args["beside"] = beside + } + } + raw, err := st.step(ctx, t, "fetch", fmt.Sprintf("fetch %s#%d on %s", p.Repository, p.Number, node), args) + if err != nil { + return fail(node, fmt.Sprintf("fetch of %s#%d: %v", p.Repository, p.Number, err)) + } + if err := json.Unmarshal(raw, &t.fetched); err != nil || t.fetched.Workspace == "" { + return fail(node, fmt.Sprintf("the fetch of %s#%d answered no workspace: %s", p.Repository, p.Number, raw)) + } + if len(t.fetched.Conflicts) > 0 { + return fail(node, fmt.Sprintf("%s#%d does not merge into the trunk: files in conflict: %s", p.Repository, p.Number, strings.Join(t.fetched.Conflicts, ", "))) + } + t.changed = t.fetched.Changed + st.targets = append(st.targets, t) + } + } + return nil +} + +// isTestFile says whether a repository-relative path is a test file of the languages the checker judges: Go's _test.go and +// the TypeScript module's .test. and .spec. files and test directories. +func isTestFile(p string) bool { + switch { + case strings.HasSuffix(p, "_test.go"): + return true + case strings.Contains(p, ".test.") || strings.Contains(p, ".spec."): + return true + } + for _, seg := range strings.Split(path.Dir(p), "/") { + if seg == "__tests__" || seg == "test" || seg == "tests" { + return true + } + } + return false +} + +// changesTestOf says whether the merge changes a test file of the Go package with the import path pkg: the directory of the +// file and the import path agree on their last segments, whichever holds the module's directory and path in front. +func changesTestOf(changed []string, pkg string) bool { + for _, f := range changed { + if !strings.HasSuffix(f, "_test.go") { + continue + } + dir := path.Dir(f) + if dir == pkg || strings.HasSuffix(pkg, "/"+dir) || strings.HasSuffix(dir, "/"+pkg) { + return true + } + } + return false +} + +// changesTestIn says whether the merge changes a test file inside the directory of a Node module. +func changesTestIn(changed []string, dir string) bool { + for _, f := range changed { + if !isTestFile(f) || strings.HasSuffix(f, "_test.go") { + continue + } + if dir == "." || dir == "" || strings.HasPrefix(f, strings.TrimSuffix(dir, "/")+"/") { + return true + } + } + return false +} + +// failedName names a failure: its package and test. +func failedName(f Failure) string { + if f.Test == "" { + return f.Package + } + return f.Package + "." + f.Test +} + +// firstLine is the first non-empty line of a failure's output, shortened. +func firstLine(s string) string { + for _, l := range strings.Split(s, "\n") { + if l = strings.TrimSpace(l); l != "" { + if len(l) > 120 { + l = l[:120] + } + return l + } + } + return "" +} + +// failure turns a level that did not pass into a verdict naming what failed first. +func levelFailure(t *target, level string, res Tested) *delivery.Verdict { + v := delivery.Verdict{State: delivery.VerdictFail} + switch { + case len(res.Failed) > 0: + f := res.Failed[0] + name := failedName(f) + if t.module == buildrun.NodeBuildModule && f.Package == "" { + name = res.Module + } + v.Failed = delivery.FailedFirst{Test: level + ": " + name, Words: firstLine(f.Output)} + default: + why := res.Why + if why == "" { + why = "the " + level + " level did not pass" + } + v.Failed = delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: why} + } + return &v +} + +// runTarget runs one level of a target: first the packages the change's own tests are in (priority), then the others. +func (st *state) runTarget(ctx context.Context, t *target, level string, priority bool) *delivery.Verdict { + if t.module == buildrun.NodeBuildModule { + return st.runNode(ctx, t, level, priority) + } + return st.runGo(ctx, t, level, priority) +} + +// A level's work is planned once for a target: the digest of each package, the kept verdicts taken, the packages left to run. +func (st *state) planGo(ctx context.Context, t *target, level string) *delivery.Verdict { + if t.plans[level] != nil { + return nil + } + plan := &goPlan{digests: map[string]string{}} + t.plans[level] = plan + raw, err := st.step(ctx, t, "digest", fmt.Sprintf("digest the %s level of %s#%d on %s", level, t.pull.Repository, t.pull.Number, t.node), + map[string]any{"workspace": t.fetched.Workspace, "level": level}) + var d Digested + if err == nil { + err = json.Unmarshal(raw, &d) + } + if ctx.Err() != nil { + return fail(t.node, ctx.Err().Error()) + } + if err != nil || len(d.Packages) == 0 { + // The build module named no digest (a level it cannot digest here): the level runs whole and keeps nothing. + if err != nil { + st.j.logf("%s#%d: no digest of the %s level, so it runs whole: %v", t.pull.Repository, t.pull.Number, level, err) + } + plan.all = true + return nil + } + plan.digests = d.Packages + a := st.levels[level] + pkgs := make([]string, 0, len(d.Packages)) + for p := range d.Packages { + pkgs = append(pkgs, p) + } + sort.Strings(pkgs) + for _, p := range pkgs { + k, found, err := st.j.Kept.Lookup(ctx, level, d.Packages[p]) + if err != nil { + st.j.logf("the kept verdict of %s at %s cannot be read, so it runs: %v", p, level, err) + } + if found { + a.kept += k.Tests + continue + } + plan.todo = append(plan.todo, p) + } + return nil +} + +func (st *state) runGo(ctx context.Context, t *target, level string, priority bool) *delivery.Verdict { + if v := st.planGo(ctx, t, level); v != nil { + return v + } + plan := t.plans[level] + var run []string + switch { + case plan.all: + if priority { + return nil // nothing to put first: the level runs whole after + } + default: + for _, p := range plan.todo { + if changesTestOf(t.changed, p) == priority { + run = append(run, p) + } + } + if len(run) == 0 { + return nil + } + } + args := map[string]any{"workspace": t.fetched.Workspace, "level": level} + if run != nil { + args["packages"] = run + } + raw, err := st.step(ctx, t, "test", fmt.Sprintf("test the %s level of %s#%d on %s", level, t.pull.Repository, t.pull.Number, t.node), args) + if err != nil { + return fail(t.node, fmt.Sprintf("the %s level of %s#%d: %v", level, t.pull.Repository, t.pull.Number, err)) + } + var res Tested + if err := json.Unmarshal(raw, &res); err != nil { + return fail(t.node, fmt.Sprintf("the %s level of %s#%d answered what is not a result: %v", level, t.pull.Repository, t.pull.Number, err)) + } + a := st.levels[level] + a.run += res.Tests + a.skipped += res.Skipped + a.elapsed += res.Elapsed + a.setup += res.Setup + a.over = a.over || res.OverBound + a.slowest = append(a.slowest, res.Slowest...) + if len(plan.digests) > 0 { + if _, err := st.j.Kept.KeepPasses(ctx, t.pull.Repository, level, t.toolchain, res, plan.digests, st.j.now()); err != nil { + st.j.logf("%s#%d: a passing package could not be kept: %v", t.pull.Repository, t.pull.Number, err) + } + } + if !res.Passed { + return levelFailure(t, level, res) + } + return nil +} + +func (st *state) runNode(ctx context.Context, t *target, level string, priority bool) *delivery.Verdict { + for _, dir := range t.fetched.Modules { + if changesTestIn(t.changed, dir) != priority { + continue + } + raw, err := st.step(ctx, t, "test", fmt.Sprintf("test the %s level of %s#%d (%s) on %s", level, t.pull.Repository, t.pull.Number, dir, t.node), + map[string]any{"workspace": t.fetched.Workspace, "module": dir, "level": level}) + if err != nil { + return fail(t.node, fmt.Sprintf("the %s level of %s#%d (%s): %v", level, t.pull.Repository, t.pull.Number, dir, err)) + } + var res Tested + if err := json.Unmarshal(raw, &res); err != nil { + return fail(t.node, fmt.Sprintf("the %s level of %s#%d (%s) answered what is not a result: %v", level, t.pull.Repository, t.pull.Number, dir, err)) + } + if res.Module == "" { + res.Module = dir + } + a := st.levels[level] + if !res.NoTests { + a.run++ // a Node package is counted whole: its build module counts no tests (ADR 0484 clause 10) + } + a.elapsed += res.Elapsed + a.setup += res.Setup + a.over = a.over || res.OverBound + if !res.Passed { + return levelFailure(t, level, res) + } + } + return nil +} diff --git a/modules/mesh-delivery-checker/internal/check/testdata/captured/README b/modules/mesh-delivery-checker/internal/check/testdata/captured/README new file mode 100644 index 0000000..e39f215 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/testdata/captured/README @@ -0,0 +1,9 @@ +go-test-failing.json, go-test-slow.json, go-test-empty.json and go-digest.json are what mesh-catalog's mesh-build-go answers +(TestResult and DigestResult, cmd/mesh-build-go levels.go and digest.go), marshalled by the module's own code: a throwaway +test beside the module's test_test.go and digest_test.go (workspaceFor + runUnit on the fixtures testdata/captured/failing, +slow and empty, and digestFixture + digests on testdata/repos/digest) at mesh-catalog origin/main cdbdb474, captured on +2026-10-21 for issue 1679 and not kept there. The fixtures' package names are the build module's own (fixture/failing/bad...). + +The fields a fetch with a base adds (base, merge, conflicts, changed, trees) are the contract the lead relayed to the +mesh-catalog agent (ADR 0484 clause 3, 2026-10-21); mesh-catalog's fetch step is not on main yet, so the fetch fixtures in +the tests are written from that contract and are to be replaced by a capture once its fetch_test can marshal one. diff --git a/modules/mesh-delivery-checker/internal/check/testdata/captured/go-digest.json b/modules/mesh-delivery-checker/internal/check/testdata/captured/go-digest.json new file mode 100644 index 0000000..50f848b --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/testdata/captured/go-digest.json @@ -0,0 +1,11 @@ +{ + "level": "unit", + "packages": { + "fixture/digest/a": "c288afe6252c38d9e7286ecb5f28b3c961780871fcc253670dcc3151c11c1df2", + "fixture/digest/c": "11abf5f32f0f12caafd76c565e4c2c2f3bf2e3ce6c2c7fcb6c2264a06db40087" + }, + "without_tests": [ + "fixture/digest/b", + "fixture/digest/d" + ] +} diff --git a/modules/mesh-delivery-checker/internal/check/testdata/captured/go-test-empty.json b/modules/mesh-delivery-checker/internal/check/testdata/captured/go-test-empty.json new file mode 100644 index 0000000..daee148 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/testdata/captured/go-test-empty.json @@ -0,0 +1,21 @@ +{ + "level": "unit", + "passed": true, + "why": "the repository has no tests at the unit level", + "no_tests": true, + "tests": 0, + "skipped": 0, + "packages": [ + { + "dir": ".", + "package": "fixture/empty", + "passed": true, + "elapsed": 0, + "tests": 0, + "no_tests": true + } + ], + "elapsed": 0, + "bound": 5, + "over_bound": false +} diff --git a/modules/mesh-delivery-checker/internal/check/testdata/captured/go-test-failing.json b/modules/mesh-delivery-checker/internal/check/testdata/captured/go-test-failing.json new file mode 100644 index 0000000..2f6d90b --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/testdata/captured/go-test-failing.json @@ -0,0 +1,45 @@ +{ + "level": "unit", + "passed": false, + "why": "1 failed: fixture/failing/bad TestThreeIsFour", + "tests": 2, + "skipped": 0, + "packages": [ + { + "dir": ".", + "package": "fixture/failing/bad", + "passed": false, + "elapsed": 0.003, + "tests": 1 + }, + { + "dir": ".", + "package": "fixture/failing/good", + "passed": true, + "elapsed": 0.002, + "tests": 1 + } + ], + "elapsed": 0.003, + "bound": 5, + "slowest": [ + { + "package": "fixture/failing/bad", + "test": "TestThreeIsFour", + "elapsed": 0 + }, + { + "package": "fixture/failing/good", + "test": "TestTwo", + "elapsed": 0 + } + ], + "failed": [ + { + "package": "fixture/failing/bad", + "test": "TestThreeIsFour", + "output": " bad_test.go:7: Three() = 3, want 4\n--- FAIL: TestThreeIsFour (0.00s)" + } + ], + "over_bound": false +} diff --git a/modules/mesh-delivery-checker/internal/check/testdata/captured/go-test-slow.json b/modules/mesh-delivery-checker/internal/check/testdata/captured/go-test-slow.json new file mode 100644 index 0000000..aa6e575 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/testdata/captured/go-test-slow.json @@ -0,0 +1,30 @@ +{ + "level": "unit", + "passed": true, + "tests": 2, + "skipped": 0, + "packages": [ + { + "dir": ".", + "package": "fixture/slow", + "passed": true, + "elapsed": 0.065, + "tests": 2 + } + ], + "elapsed": 0.065, + "bound": 5, + "slowest": [ + { + "package": "fixture/slow", + "test": "TestTakesAWhile", + "elapsed": 0.06 + }, + { + "package": "fixture/slow", + "test": "TestQuick", + "elapsed": 0 + } + ], + "over_bound": false +} From e17534383716974e201d1f833a531e86a07ec278 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 21:12:48 +0200 Subject: [PATCH 04/19] Write the merge preview of each head the checker fetched, for the indexer's fingerprints (issue 1680) --- .../internal/check/examiner_test.go | 50 +++++++++++++ .../internal/check/judge.go | 3 + .../internal/check/preview.go | 70 +++++++++++++++++++ 3 files changed, 123 insertions(+) create mode 100644 modules/mesh-delivery-checker/internal/check/preview.go diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go index 446ee9c..c7aa9a1 100644 --- a/modules/mesh-delivery-checker/internal/check/examiner_test.go +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -454,3 +454,53 @@ func TestAGroupIsOneVerdictAndEachMembersTestsReadTheOthersAtTheirHeads(t *testi func TestAnExaminerIsAJudgeOfTheChecker(t *testing.T) { var _ Judge = (*Examiner)(nil) } + +const ( + manA = "0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a" +) + +func TestACleanMergeWritesItsPreviewWithTheTreesManifestsAndLocks(t *testing.T) { // 1445/AC1 + b := newBench(t) + b.fetch.Trees = map[string]MergeTree{ + "modules/avahi": {Tree: "7777777777777777777777777777777777777777", Manifest: manA, Locks: map[string]string{"go.sum": "ff00"}}, + "modules/bare": {Tree: "8888888888888888888888888888888888888888"}, // no manifest digest: left out + } + b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + key, _ := delivery.PreviewKey(repoA, head1, trunkA) + raw, _, found, _ := b.prev.Get(context.Background(), key) + if !found { + t.Fatalf("no preview %s; the bucket holds %v", key, b.prev.vals) + } + pv, err := delivery.DecodePreview(raw) + if err != nil { + t.Fatal(err) + } + m := pv.Modules["modules/avahi"] + if pv.Merge != b.fetch.Merge || pv.Trunk != trunkA || pv.Head != head1 || len(pv.Modules) != 1 || m.Tree != "7777777777777777777777777777777777777777" || m.Manifest != manA || m.Locks["go.sum"] != "ff00" { + t.Errorf("preview = %+v", pv) + } +} + +func TestAHeadThatDoesNotMergeWritesNoPreview(t *testing.T) { // 1445/AC5 + b := newBench(t) + b.fetch = Fetched{Workspace: "ws1", Commit: head1, Base: trunkA, Conflicts: []string{"go.mod"}} + b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + if len(b.prev.vals) != 0 { + t.Errorf("previews %v written for a head that does not merge", b.prev.vals) + } +} + +func TestAPreviewIsWrittenOnceAndAnExistingKeyStands(t *testing.T) { + b := newBench(t) + b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + b.fetch.Merge = "5656565656565656565656565656565656565656" + b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + key, _ := delivery.PreviewKey(repoA, head1, trunkA) + if b.prev.creates[key] != 1 { + t.Errorf("the preview was created %d times, want once", b.prev.creates[key]) + } + raw, _, _, _ := b.prev.Get(context.Background(), key) + if pv, _ := delivery.DecodePreview(raw); pv.Merge == b.fetch.Merge { + t.Error("the second merge commit replaced the first") + } +} diff --git a/modules/mesh-delivery-checker/internal/check/judge.go b/modules/mesh-delivery-checker/internal/check/judge.go index a6ac48c..6f90d74 100644 --- a/modules/mesh-delivery-checker/internal/check/judge.go +++ b/modules/mesh-delivery-checker/internal/check/judge.go @@ -202,6 +202,7 @@ func (st *state) fetchAll(ctx context.Context) *delivery.Verdict { began := st.j.now() defer func() { st.clone = st.j.now().Sub(began) }() for _, p := range st.c.Pulls { + var merged []Fetched for _, bm := range st.j.Modules.BuildModules(p.Repository) { node, tc, err := st.choose(bm) if err != nil { @@ -226,8 +227,10 @@ func (st *state) fetchAll(ctx context.Context) *delivery.Verdict { return fail(node, fmt.Sprintf("%s#%d does not merge into the trunk: files in conflict: %s", p.Repository, p.Number, strings.Join(t.fetched.Conflicts, ", "))) } t.changed = t.fetched.Changed + merged = append(merged, t.fetched) st.targets = append(st.targets, t) } + st.j.writePreview(ctx, p, merged) } return nil } diff --git a/modules/mesh-delivery-checker/internal/check/preview.go b/modules/mesh-delivery-checker/internal/check/preview.go new file mode 100644 index 0000000..6884a1e --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/preview.go @@ -0,0 +1,70 @@ +package check + +import ( + "context" + "errors" + "fmt" + "regexp" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +var digest64 = regexp.MustCompile(`^[0-9a-f]{64}$`) + +// PreviewOf is what the fetch steps said of a pull request's head merged onto its trunk head, as the value of a previews +// key (ADR 0484 clause 3): the merge commit and, for each module directory the fetches named with a manifest, its tree, the +// digest of its manifest and the hashes of its lock files. The fetches of the repository's build modules merge the same +// trees; the first one's merge commit is kept. A module directory with no tree or no manifest digest is left out, so the +// indexer gives its module no fingerprint and says why. It answers false when no fetch merged. +func PreviewOf(p Pull, fetched []Fetched) (delivery.Preview, bool) { + var merge string + mods := map[string]delivery.PreviewModule{} + for _, f := range fetched { + if f.Merge == "" { + continue + } + if merge == "" { + merge = f.Merge + } + for dir, t := range f.Trees { + if t.Tree == "" || !digest64.MatchString(t.Manifest) { + continue + } + mods[dir] = delivery.PreviewModule{Tree: t.Tree, Manifest: t.Manifest, Locks: t.Locks} + } + } + if merge == "" { + return delivery.Preview{}, false + } + pv := delivery.Preview{Repository: p.Repository, Head: p.Head, Trunk: p.Trunk, Merge: merge} + if len(mods) > 0 { + pv.Modules = mods + } + return pv, true +} + +// writePreview writes the preview of a pull request by a create: one writer per key, and the same head on the same trunk +// head merges to the same trees, so a key that exists stands. A head that did not merge has no preview. A failure is said +// and does not fail the check: the preview is for the indexer's fingerprints, not for the verdict. +func (j *Examiner) writePreview(ctx context.Context, p Pull, fetched []Fetched) { + if j.Previews == nil { + return + } + pv, ok := PreviewOf(p, fetched) + if !ok { + return + } + key, err := delivery.PreviewKey(p.Repository, p.Head, p.Trunk) + if err == nil { + var data []byte + if data, err = delivery.EncodePreview(pv); err == nil { + _, err = j.Previews.Create(ctx, key, data) + if errors.Is(err, ErrConflict) { + return + } + } + } + if err != nil { + j.logf("%s#%d: the preview cannot be written: %v", p.Repository, p.Number, fmt.Errorf("%w", err)) + } +} From 4242fb5b0bc4017c0c5ebdefeedaefaad84e1fba Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 21:17:09 +0200 Subject: [PATCH 05/19] The indexer gives a candidate's touched module the fingerprint its merge preview gives it, and wakes on the checker's preview.written without a restart; pin mesh-sdk to the R4c-5a head (issues 1445, 1681) --- modules/mesh-delivery-builder/go.mod | 2 +- modules/mesh-delivery-builder/go.sum | 4 +- .../cmd/mesh-delivery-checker/judge.go | 3 +- .../cmd/mesh-delivery-checker/main.go | 10 + .../internal/check/examiner_test.go | 12 + .../internal/check/judge.go | 2 + .../internal/check/preview.go | 3 + .../mesh-delivery-checker/manifest_test.go | 12 +- modules/mesh-delivery-checker/module.json | 3 + modules/mesh-delivery-forge-bridge/go.mod | 2 +- modules/mesh-delivery-forge-bridge/go.sum | 4 +- .../mesh-delivery-indexer/integration_test.go | 89 +++++++ .../cmd/mesh-delivery-indexer/main.go | 19 ++ .../cmd/mesh-delivery-indexer/previews.go | 88 +++++++ modules/mesh-delivery-indexer/go.mod | 2 +- modules/mesh-delivery-indexer/go.sum | 4 +- .../internal/index/candidates.go | 4 +- .../internal/index/previews.go | 247 ++++++++++++++++++ .../internal/index/previews_test.go | 181 +++++++++++++ .../internal/index/source.go | 5 +- .../internal/index/toolchains.go | 8 + .../mesh-delivery-indexer/manifest_test.go | 4 +- modules/mesh-delivery-indexer/module.json | 2 + 23 files changed, 694 insertions(+), 16 deletions(-) create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go create mode 100644 modules/mesh-delivery-indexer/internal/index/previews.go create mode 100644 modules/mesh-delivery-indexer/internal/index/previews_test.go diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index 4431597..b8b9bd8 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-builder go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-builder/go.sum b/modules/mesh-delivery-builder/go.sum index 639aa71..7c963a0 100644 --- a/modules/mesh-delivery-builder/go.sum +++ b/modules/mesh-delivery-builder/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865 h1:VLir9NZbyi1LdH1YwWDIVRuL+lDC+AYlm6FZPlvmlm4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5 h1:1BiPKRPTYpgex8rSHWT9az0BteOafGWRDzd/Ba6GAcA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go index b80e7b0..0d55bda 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go @@ -23,12 +23,13 @@ type judgeDeps struct { kept check.KV previews check.KV repos *repoModules + announce func(ctx context.Context, key string) logf func(format string, args ...any) } func newJudge(d judgeDeps) check.Judge { return &check.Examiner{Steps: d.runner, Toolchains: d.tools, Presence: d.presence, Health: d.have, Places: d.places, - Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Trunks: d.trunks, Logf: d.logf} + Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Announce: d.announce, Trunks: d.trunks, Logf: d.logf} } // manifestKV reads a manifest by its digest. diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index 9fc4703..537680e 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -32,6 +32,10 @@ import ( "mesh-delivery-checker/internal/check" ) +// previewWrittenSubject is the subject of the checker's preview.written event, on its stream previews (the bus names the +// stream PREVIEWS); the indexer's watch of the previews bucket is woken by it. +const previewWrittenSubject = "mesh.stream." + delivery.CheckerModule + ".event.preview.written" + const ( // defaultStepBound is how long the checker waits for one step of a build module (a fetch downloads the Go modules, a // test level runs a repository's tests). @@ -197,6 +201,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro runner: runner, tools: tools, presence: presence, have: have, places: places, trunks: trunks, kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]}, repos: repos, logf: logger.Printf, + announce: func(ctx context.Context, key string) { + body, _ := json.Marshal(map[string]string{"key": key}) + if _, err := js.Publish(ctx, previewWrittenSubject, body); err != nil { + logger.Printf("previews.%s was written and the indexer could not be told: %v", key, err) + } + }, }) // The toolchains first, so a check that starts from the first candidate finds the nodes. diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go index c7aa9a1..270a504 100644 --- a/modules/mesh-delivery-checker/internal/check/examiner_test.go +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -504,3 +504,15 @@ func TestAPreviewIsWrittenOnceAndAnExistingKeyStands(t *testing.T) { t.Error("the second merge commit replaced the first") } } + +func TestTheIndexerIsToldOfAPreviewOnceWhenItIsCreated(t *testing.T) { // 1445/AC4 (the checker's half) + b := newBench(t) + var told []string + b.ex.Announce = func(_ context.Context, key string) { told = append(told, key) } + b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + key, _ := delivery.PreviewKey(repoA, head1, trunkA) + if len(told) != 1 || told[0] != key { + t.Errorf("told %v, want once of %s", told, key) + } +} diff --git a/modules/mesh-delivery-checker/internal/check/judge.go b/modules/mesh-delivery-checker/internal/check/judge.go index 6f90d74..c29508d 100644 --- a/modules/mesh-delivery-checker/internal/check/judge.go +++ b/modules/mesh-delivery-checker/internal/check/judge.go @@ -50,6 +50,8 @@ type Examiner struct { // Previews is the previews bucket; nil writes none. Previews KV Modules BuildModules + // Announce says a preview was written under the key (the checker's preview.written event); nil says nothing. + Announce func(ctx context.Context, key string) // Trunks are the trunk heads of every repository, for the siblings a repository's tests read. Trunks interface{ Heads() map[string]string } Now func() time.Time diff --git a/modules/mesh-delivery-checker/internal/check/preview.go b/modules/mesh-delivery-checker/internal/check/preview.go index 6884a1e..e44a9d5 100644 --- a/modules/mesh-delivery-checker/internal/check/preview.go +++ b/modules/mesh-delivery-checker/internal/check/preview.go @@ -62,6 +62,9 @@ func (j *Examiner) writePreview(ctx context.Context, p Pull, fetched []Fetched) if errors.Is(err, ErrConflict) { return } + if err == nil && j.Announce != nil { + j.Announce(ctx, key) + } } } if err != nil { diff --git a/modules/mesh-delivery-checker/manifest_test.go b/modules/mesh-delivery-checker/manifest_test.go index 892ecbe..1517d59 100644 --- a/modules/mesh-delivery-checker/manifest_test.go +++ b/modules/mesh-delivery-checker/manifest_test.go @@ -31,7 +31,13 @@ type artifact struct { Binary string `json:"binary"` } +type stream struct { + Name string `json:"name"` + Events []string `json:"events"` +} + type manifest struct { + Streams []stream `json:"streams"` Module string `json:"module"` Version string `json:"version"` Slug string `json:"slug"` @@ -105,7 +111,11 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 if m.SecretsOwner != "mesh-delivery-checker" { t.Errorf("secrets-owner = %q", m.SecretsOwner) } - for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats", "bus"} { + // The checker says each time it wrote a preview, so the indexer's watch finds the bucket (ADR 0484 clause 3). + if want := []stream{{Name: "previews", Events: []string{"preview.**"}}}; !reflect.DeepEqual(m.Streams, want) { + t.Errorf("streams = %+v, want %+v", m.Streams, want) + } + for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats", "bus"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the checker has none", k) } diff --git a/modules/mesh-delivery-checker/module.json b/modules/mesh-delivery-checker/module.json index 8044314..eef0cc1 100644 --- a/modules/mesh-delivery-checker/module.json +++ b/modules/mesh-delivery-checker/module.json @@ -6,6 +6,9 @@ "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported" ], + "streams": [ + { "name": "previews", "events": ["preview.**"] } + ], "state": [ { "name": "kept", "history": 1 }, { "name": "owed", "history": 5 }, diff --git a/modules/mesh-delivery-forge-bridge/go.mod b/modules/mesh-delivery-forge-bridge/go.mod index 25936a8..d073e5f 100644 --- a/modules/mesh-delivery-forge-bridge/go.mod +++ b/modules/mesh-delivery-forge-bridge/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-forge-bridge/go.sum b/modules/mesh-delivery-forge-bridge/go.sum index 0e67d23..7c963a0 100644 --- a/modules/mesh-delivery-forge-bridge/go.sum +++ b/modules/mesh-delivery-forge-bridge/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960 h1:FCId3IuRScG03X7Gx9vfKfGNZH9jWnfx9RChfJ3eWfo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5 h1:1BiPKRPTYpgex8rSHWT9az0BteOafGWRDzd/Ba6GAcA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index 11d9623..a885d9d 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -351,3 +351,92 @@ func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *tes t.Errorf("none2 = %q within 5 s of the toolchain key, want the builder's fingerprint not yet known", got) } } + +// TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBucket: the indexer starts with the stream PREVIEWS +// and its durable consumer (made by the controller from the checker's declaration and the indexer's consumes) and no bucket +// mesh-delivery-checker_previews; a pull request of mesh-catalog touching avahi makes the candidate with the reason +// 'no merge preview is computed'; the checker then makes its bucket, writes the preview of that head onto the trunk head +// the source names, and says so on its stream; the candidate's avahi gets, within 5 s and with no restart, the reason +// computed from the preview (the toolchain is not reported here, so 'toolchain not yet reported'). +func TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBucket(t *testing.T) { // 1445/AC4 + manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json") + if err != nil { + t.Fatal(err) + } + const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381" + const inputs = `{"Module":"avahi","Tree":"oldtree","Bases":null,"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}` + const trunk, head = "5555555555555555555555555555555555555555", "0123456789abcdef0123456789abcdef01234567" + js, ctx, candidates, _ := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) { + rec := `{"id":"build-1","repository":"novox/mesh-catalog","ref":"main","module":"avahi","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/avahi","at":"2026-10-19T09:38:00Z","source-inputs":` + inputs + `}` + if _, err := builds.Put(ctx, delivery.BuildKey("avahi", "build-1"), []byte(rec)); err != nil { + t.Fatal(err) + } + if _, err := builds.Put(ctx, delivery.NewestKey("avahi"), []byte("build-1")); err != nil { + t.Fatal(err) + } + }, func(ctx context.Context, js jetstream.JetStream) { + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: PreviewsStream, Subjects: []string{previewWrittenSubject}}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateOrUpdateConsumer(ctx, PreviewsStream, jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-indexer", AckPolicy: jetstream.AckExplicitPolicy}); err != nil { + t.Fatal(err) + } + source, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.SourceBucket, History: 4}) + if err != nil { + t.Fatal(err) + } + data, err := delivery.EncodeSource(delivery.Source{Group: "g", Heads: map[string]string{"novox/mesh-catalog": trunk}}) + if err != nil { + t.Fatal(err) + } + if _, err := source.Put(ctx, delivery.SourceMainKey, data); err != nil { + t.Fatal(err) + } + }) + pull, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, Head: head, Action: delivery.PullUpdated, + Dirs: []string{"modules/avahi"}, DirsKnown: true, Manifests: map[string]string{"modules/avahi": string(manifest)}}) + if err != nil { + t.Fatal(err) + } + if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.pull", pull); err != nil { + t.Fatal(err) + } + none2 := func(want string) string { + var got string + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + if e, err := candidates.Get(ctx, "feat/x"); err == nil { + if c, err := delivery.DecodeCandidate(e.Value()); err == nil && len(c.Pulls) == 1 { + got = c.Pulls[0].Fps["avahi"].None2 + if got == want { + break + } + } + } + } + return got + } + if got := none2("no merge preview is computed"); got != "no merge preview is computed" { + t.Fatalf("none2 = %q within 5 s of the pull request, want 'no merge preview is computed'", got) + } + // The checker's bucket is made only now, as when the checker is assigned after the indexer started. + kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.PreviewsBucket, History: delivery.PreviewsHistory}) + if err != nil { + t.Fatal(err) + } + pv := delivery.Preview{Repository: "novox/mesh-catalog", Head: head, Trunk: trunk, Merge: "6666666666666666666666666666666666666666", + Modules: map[string]delivery.PreviewModule{"modules/avahi": {Tree: "newtree", Manifest: delivery.ManifestDigest(manifest), Locks: map[string]string{"go.sum": goSum}}}} + key, _ := delivery.PreviewKey(pv.Repository, pv.Head, pv.Trunk) + data, err := delivery.EncodePreview(pv) + if err != nil { + t.Fatal(err) + } + if _, err := kv.Create(ctx, key, data); err != nil { + t.Fatal(err) + } + if _, err := js.Publish(ctx, previewWrittenSubject, []byte(`{"key":"`+key+`"}`)); err != nil { + t.Fatal(err) + } + if got := none2(delivery.ToolchainNoReport); got != delivery.ToolchainNoReport { + t.Errorf("none2 = %q within 5 s of the preview, want %q (computed from the preview)", got, delivery.ToolchainNoReport) + } +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 846879f..54df569 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -309,6 +309,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := tools.start(); err != nil { return err } + // The checker's merge previews give the candidates their fingerprints: the bucket is watched where it exists, and + // otherwise when the checker says it wrote a preview. + previews := newPreviewWatch(ctx, js, ix, logger) + if err := previews.ensure(); err != nil { + return err + } // Only now, with the toolchains read, are the builds heard to compute from (a build before it would write every Go // module as having no toolchain reported, and back); the builds heard until now are computed once. started.Store(ix) @@ -332,6 +338,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } defer tc.Stop() } + switch pcons, err := js.Consumer(ctx, PreviewsStream, cred.Node+"_"+cred.Module); { + case err != nil: + logger.Printf("the consumer %s_%s on %s does not exist yet, so the checker's previews are not heard until the indexer starts again: %v", cred.Node, cred.Module, PreviewsStream, err) + default: + pc, err := pcons.Consume(func(m jetstream.Msg) { previews.Written(m) }, + jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) })) + if err != nil { + return err + } + defer pc.Stop() + } cons, err := js.Consumer(ctx, forgeStream, cred.Node+"_"+cred.Module) if err != nil { return fmt.Errorf("the consumer %s_%s on %s: %w", cred.Node, cred.Module, forgeStream, err) @@ -351,5 +368,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err case err := <-tools.errs: return err + case err := <-previews.errs: + return err } } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go new file mode 100644 index 0000000..fcf6a31 --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go @@ -0,0 +1,88 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log" + "strings" + "sync" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-indexer/internal/index" +) + +// PreviewsStream is the stream the checker carries its preview.written event on, as the bus names it (declared by the +// checker as the stream previews, heard by this module through consumes; novox/hq ADR 0484 clause 3). +const PreviewsStream = "PREVIEWS" + +// previewWrittenSubject is the subject of the checker's preview.written event. +const previewWrittenSubject = "mesh.stream." + delivery.CheckerModule + ".event.preview.written" + +// previewWatch reads the merge previews the checker writes (mesh-delivery-checker_previews) and follows them: every change +// writes the candidates' fingerprints again (index.Indexer.ApplyPreview). The checker's bucket is made when it is assigned, +// after the indexer started, and the indexer holds no timer (ADR 0437 §8): it starts with the bucket absent, and the +// checker says each time it wrote a preview (preview.written, handled by Written); the bucket is then watched and settled +// as the others, with no restart. +type previewWatch struct { + ctx context.Context + js bucketOpener + ix *index.Indexer + logger *log.Logger + errs chan error + mu sync.Mutex + watched bool +} + +func newPreviewWatch(ctx context.Context, js bucketOpener, ix *index.Indexer, logger *log.Logger) *previewWatch { + return &previewWatch{ctx: ctx, js: js, ix: ix, logger: logger, errs: make(chan error, 1)} +} + +// ensure watches the previews bucket if it is not watched and exists, settling what it holds, then writes the candidates' +// fingerprints from it. +func (w *previewWatch) ensure() error { + w.mu.Lock() + defer w.mu.Unlock() + if w.watched { + return nil + } + kv, err := w.js.KeyValue(w.ctx, delivery.PreviewsBucket) + switch { + case errors.Is(err, jetstream.ErrBucketNotFound): + w.logger.Printf("the bucket %s does not exist yet: it is watched when the checker says it wrote a preview", delivery.PreviewsBucket) + return nil + case err != nil: + return fmt.Errorf("the bucket %s: %w", delivery.PreviewsBucket, err) + } + watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.ix.PutPreview(e.Key(), e.Value(), !opOf(e)) }) + if err != nil { + return err + } + w.watched = true + go func() { + w.errs <- follow(w.ctx, watcher, func(e jetstream.KeyValueEntry) { + if err := w.ix.ApplyPreview(w.ctx, e.Key(), e.Value(), !opOf(e)); err != nil { + w.logger.Printf("%s.%s could not be taken into the candidates: %v", delivery.PreviewsBucket, e.Key(), err) + } + }) + }() + return w.ix.RecomputeCandidates(w.ctx) +} + +// Written takes one preview.written event of the checker in: its bucket is watched if it is not. A subject that is not the +// checker's is terminated; a failure stays unacknowledged, and the server delivers it again. +func (w *previewWatch) Written(m delivered) { + if !strings.HasPrefix(m.Subject(), previewWrittenSubject) { + w.logger.Printf("%s: not the checker's preview.written, dropped", m.Subject()) + _ = m.Term() + return + } + if err := w.ensure(); err != nil { + w.logger.Printf("%s: left for redelivery: %v", m.Subject(), err) + return + } + _ = m.Ack() +} diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index c7b22bb..94177a4 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-indexer go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-indexer/go.sum b/modules/mesh-delivery-indexer/go.sum index 639aa71..7c963a0 100644 --- a/modules/mesh-delivery-indexer/go.sum +++ b/modules/mesh-delivery-indexer/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865 h1:VLir9NZbyi1LdH1YwWDIVRuL+lDC+AYlm6FZPlvmlm4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5 h1:1BiPKRPTYpgex8rSHWT9az0BteOafGWRDzd/Ba6GAcA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 80ab566..2f1bf70 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -48,8 +48,9 @@ type Indexer struct { Builds *Builds Logf func(format string, args ...any) - mu sync.Mutex // guards source and toolchains, which the watches and the forge messages all reach + mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach source sourceNow + previews map[string]delivery.Preview // by previews key (previews.go) toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go) recompute sync.Mutex // one Recompute at a time } @@ -102,6 +103,7 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error { } next := ix.next(cur, f, digests) if next != nil { + ix.withPreviews(ctx, next) if err := ix.refuseDisagreement(*next); err != nil { return nil, false, err } diff --git a/modules/mesh-delivery-indexer/internal/index/previews.go b/modules/mesh-delivery-indexer/internal/index/previews.go new file mode 100644 index 0000000..2aedb7d --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/previews.go @@ -0,0 +1,247 @@ +package index + +import ( + "context" + "fmt" + "maps" + "slices" + "strings" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The merge previews the checker writes (mesh-delivery-checker_previews; novox/hq ADR 0484 clause 3, issue 1445): a pull +// request's head merged onto its trunk's head, with the tree, the manifest digest and the lock hashes of each module +// directory. From one the indexer gives the candidate's touched module its new-form fingerprint, by the rules a module of +// the trunk is given its own (Recompute), or says why it has none. + +// The reasons a candidate's touched module has no fingerprint of the new form while a preview exists or is missing. +const ( + // NonePreviewStale is for a module whose preview was made onto a trunk head that is not the trunk's head now. + NonePreviewStale = "the merge preview is out of date: the trunk moved since it was made" + // NonePreviewNoModule is for a module whose directory the preview names no manifest for. + NonePreviewNoModule = "the merge preview names no manifest for the module's directory" + // NonePreviewNoManifest is for a module whose manifest at the merge the indexer does not hold. + NonePreviewNoManifest = "the manifest of the merge is not kept" + // NoneNoDirectory is for a module whose directory in its repository the builds do not tell. + NoneNoDirectory = "the module's directory in its repository is not known" +) + +// PutPreview takes a change of the previews bucket in without writing any candidate: a put of a preview (key +// delivery.PreviewKey) or its delete. It answers whether the previews held changed. A key or a value that is not a +// preview is said in the log and left out. +func (ix *Indexer) PutPreview(key string, value []byte, deleted bool) bool { + if _, _, _, err := delivery.ParsePreviewKey(key); err != nil { + ix.logf("previews.%s is not a preview key, left out: %v", key, err) + return false + } + ix.mu.Lock() + defer ix.mu.Unlock() + if deleted { + if _, had := ix.previews[key]; !had { + return false + } + delete(ix.previews, key) + return true + } + pv, err := delivery.DecodePreview(value) + if err != nil { + ix.logf("previews.%s is not a preview, left out: %v", key, err) + return false + } + if ix.previews == nil { + ix.previews = map[string]delivery.Preview{} + } + if cur, had := ix.previews[key]; had && delivery.SamePreview(cur, pv) { + return false + } + ix.previews[key] = pv + return true +} + +// ApplyPreview takes one change of the previews in (PutPreview) and, when the previews held changed, writes the +// candidates' fingerprints again (RecomputeCandidates). +func (ix *Indexer) ApplyPreview(ctx context.Context, key string, value []byte, deleted bool) error { + if !ix.PutPreview(key, value, deleted) { + return nil + } + return ix.RecomputeCandidates(ctx) +} + +// RecomputeCandidates writes again every candidate whose pull requests' fingerprints, from the previews held, are no longer +// what it holds. A candidate is written only when it differs, by compare-and-set on the revision read. +func (ix *Indexer) RecomputeCandidates(ctx context.Context) error { + if ix.Store == nil { + return nil + } + ix.mu.Lock() + none := len(ix.previews) == 0 + ix.mu.Unlock() + if none { + return nil // no preview held: every candidate has what the pull messages gave it + } + keys, err := ix.Store.Keys(ctx) + if err != nil { + return fmt.Errorf("listing the candidates: %w", err) + } + for _, key := range keys { + err := write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) { + if !found { + return nil, false, nil + } + c, err := delivery.DecodeCandidate(value) + if err != nil { + ix.logf("candidates.%s holds what is not a candidate, left: %v", key, err) + return nil, false, nil + } + if !ix.withPreviews(ctx, &c) { + return nil, false, nil + } + data, err := delivery.EncodeCandidate(c) + return data, false, err + }) + if err != nil { + return fmt.Errorf("candidates.%s: %w", key, err) + } + } + return nil +} + +// previewOf is the preview of a head merged onto the trunk head the source names now: found, or stale when only a preview +// onto another trunk head exists, or neither (found false, stale false) when the source is not known or none exists. +func (ix *Indexer) previewOf(repository, head string) (pv delivery.Preview, found, stale bool) { + ix.mu.Lock() + defer ix.mu.Unlock() + trunk := ix.source.heads[repository] + if trunk == "" { + return delivery.Preview{}, false, false + } + if key, err := delivery.PreviewKey(repository, head, trunk); err == nil { + if pv, ok := ix.previews[key]; ok { + return pv, true, false + } + } + for _, pv := range ix.previews { + if pv.Repository == repository && pv.Head == head { + return delivery.Preview{}, false, true + } + } + return delivery.Preview{}, false, false +} + +// withPreviews gives each touched module of the candidate's pull requests the fingerprint its preview gives it, or the +// reason it has none, and the candidate its fpset. It answers whether the candidate changed. A module of a pull request with +// no preview keeps what it had. +func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool { + changed := false + for i := range c.Pulls { + p := &c.Pulls[i] + pv, found, stale := ix.previewOf(p.Repository, p.Head.Head) + if !found && !stale { + continue + } + dirs := map[string]string{} + for _, k := range ix.Builds.Modules(p.Repository) { + dirs[k.Module] = k.Path + } + fps := maps.Clone(p.Fps) + for _, m := range p.Modules { + next := delivery.CandidateModule{Manifest: p.Fps[m].Manifest} + switch { + case stale: + next.None2 = NonePreviewStale + default: + dir, known := dirs[m] + if !known { + next.None2 = NoneNoDirectory + break + } + next = ix.candidateModule(ctx, m, dir, pv) + } + if fps == nil { + fps = map[string]delivery.CandidateModule{} + } + if fps[m] != next { + fps[m] = next + changed = true + } + } + if changed || !maps.Equal(fps, p.Fps) { + p.Fps = fps + } + } + if changed { + c.FPSet = delivery.FingerprintSet(c.Pulls) + } + return changed +} + +// candidateModule is the fingerprint of module m, whose directory is dir, at the preview: the module's inputs from the +// merge (tree, manifest, locks), its bases' and the builder's fingerprints as the trunk's modules hold them, and the toolchains +// the build modules agree on, by the rules Recompute applies to a module of the trunk. +func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delivery.Preview) delivery.CandidateModule { + pm, ok := pv.Modules[dir] + if !ok && dir == "" { + pm, ok = pv.Modules[delivery.RootPath] + } + if !ok { + return delivery.CandidateModule{None2: NonePreviewNoModule} + } + out := delivery.CandidateModule{Manifest: pm.Manifest} + if ix.Manifests == nil { + out.None2 = NonePreviewNoManifest + return out + } + value, _, found, err := ix.Manifests.Get(ctx, pm.Manifest) + if err != nil || !found { + out.None2 = NonePreviewNoManifest + return out + } + kept, err := delivery.DecodeManifest(value) + if err != nil { + out.None2 = NonePreviewNoManifest + return out + } + in := inputsOf(m, kept.JSON, pm.Tree, pm.Locks, ix.baseFP(ctx)) + if in.Kept == nil { + out.None2 = in.None2 + return out + } + if in.None2 != "" && in.None2 != delivery.ToolchainNoReport && in.None2 != None2Builder { + out.None2 = in.None2 // a context, or a base with no fingerprint + return out + } + tools := ix.agreed() + toolchains := map[string]delivery.Toolchain2{} + for _, artifact := range slices.Sorted(maps.Keys(in.Languages)) { + t := tools[in.Languages[artifact]] + if t.none != "" { + out.None2 = t.none + return out + } + toolchains[artifact] = t.tc + } + inputs := *in.Kept + if m == BuilderModule { + src, none := ix.builderInput() + if src == "" { + out.None2 = none + return out + } + inputs.Builder = src + } else { + fp, none := ix.baseFP(ctx)(BuilderModule) + if fp == "" { + out.None2 = strings.TrimSuffix(None2Builder+": "+none, ": ") + return out + } + inputs.Builder = fp + } + fp, none := delivery.SourceFingerprint2(inputs.With(m, toolchains)) + if fp == "" { + out.None2 = none + return out + } + out.FP2 = fp + return out +} diff --git a/modules/mesh-delivery-indexer/internal/index/previews_test.go b/modules/mesh-delivery-indexer/internal/index/previews_test.go new file mode 100644 index 0000000..f5484e6 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/previews_test.go @@ -0,0 +1,181 @@ +package index + +import ( + "context" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +const ( + trunkNow = "5555555555555555555555555555555555555555" + trunkOld = "4444444444444444444444444444444444444444" + mergeSha = "6666666666666666666666666666666666666666" +) + +// A trunk of the catalogue with avahi (and the builder, a base and others) merged, the toolchain agreed, and the gate's source +// naming the catalogue's trunk head: avahi has a fingerprint of the new form at tree-avahi. +func previewBed(t *testing.T, report bool) (*Indexer, *memStore, *memStore) { + t.Helper() + ix, mods := newFormIndexer(t, builderModules(t)...) + if report { + reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") + } + if err := ix.ApplySource(context.Background(), 5, delivery.Source{Group: "g", Heads: map[string]string{catalogueRepo: trunkNow}}); err != nil { + t.Fatal(err) + } + return ix, mods, ix.Store.(*memStore) +} + +// avahiOnTools is avahi's manifest as builderModules has it: standing on mesh-tools. +func avahiOnTools(t *testing.T) string { + return standingOn(t, catalogueManifest(t, "avahi"), "mesh-tools") +} + +// avahiPull is the pull message of the catalogue's pull request 301 touching avahi, with its manifest. +func avahiPull(t *testing.T, head string) delivery.Forge { + t.Helper() + f := manifestPull(t, 301, head, avahiOnTools(t)) + f.Manifests = map[string]string{"modules/avahi": avahiOnTools(t)} + return f +} + +func avahiPreview(t *testing.T, trunk, head string) delivery.Preview { + return delivery.Preview{Repository: catalogueRepo, Head: head, Trunk: trunk, Merge: mergeSha, Modules: map[string]delivery.PreviewModule{ + "modules/avahi": {Tree: "tree-avahi", Manifest: delivery.ManifestDigest([]byte(avahiOnTools(t))), Locks: map[string]string{"go.sum": avahiGoSum}}}} +} + +func applyPreview(t *testing.T, ix *Indexer, pv delivery.Preview) { + t.Helper() + key, err := delivery.PreviewKey(pv.Repository, pv.Head, pv.Trunk) + if err != nil { + t.Fatal(err) + } + data, err := delivery.EncodePreview(pv) + if err != nil { + t.Fatal(err) + } + if err := ix.ApplyPreview(context.Background(), key, data, false); err != nil { + t.Fatal(err) + } +} + +func fpOf(t *testing.T, st *memStore) delivery.CandidateModule { + t.Helper() + c := candidateOf(t, st) + if len(c.Pulls) != 1 { + t.Fatalf("candidate = %+v", c) + } + return c.Pulls[0].Fps["avahi"] +} + +func TestACandidatesModuleGetsTheFingerprintATrunkModuleOfTheSameTreeHas(t *testing.T) { // 1445/AC2 + ix, mods, cands := previewBed(t, true) + trunk, _ := moduleOf(t, mods, "avahi") + if trunk.FP2 == "" { + t.Fatalf("the trunk's avahi has no fingerprint of the new form: %+v", trunk) + } + if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil { + t.Fatal(err) + } + if got := fpOf(t, cands); got.None2 != NoneNoPreview { + t.Fatalf("before the preview: %+v", got) + } + applyPreview(t, ix, avahiPreview(t, trunkNow, h1)) + got := fpOf(t, cands) + if got.FP2 != trunk.FP2 || got.None2 != "" || got.Manifest != delivery.ManifestDigest([]byte(avahiOnTools(t))) { + t.Fatalf("candidate avahi = %+v, want the trunk's fingerprint %s", got, trunk.FP2) + } + if c := candidateOf(t, cands); c.FPSet != delivery.FingerprintSet(c.Pulls) || c.FPSet == "" { + t.Errorf("fpset = %q", c.FPSet) + } + // Another tree gives another fingerprint. + pv := avahiPreview(t, trunkNow, h1) + m := pv.Modules["modules/avahi"] + m.Tree = "tree-changed" + pv.Modules["modules/avahi"] = m + pv.Merge = "7777777777777777777777777777777777777777" + applyPreview(t, ix, pv) + if other := fpOf(t, cands); other.FP2 == "" || other.FP2 == trunk.FP2 { + t.Errorf("a changed tree: %+v", other) + } +} + +func TestAnUnknownToolchainLeavesTheCandidatesModuleNamingIt(t *testing.T) { // 1445/AC2 + ix, _, cands := previewBed(t, false) + if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil { + t.Fatal(err) + } + applyPreview(t, ix, avahiPreview(t, trunkNow, h1)) + if got := fpOf(t, cands); got.FP2 != "" || !strings.Contains(got.None2, delivery.ToolchainNoReport) { + t.Errorf("candidate avahi = %+v, want none2 naming %q", got, delivery.ToolchainNoReport) + } +} + +func TestAPreviewMadeOntoAnOldTrunkIsOutOfDate(t *testing.T) { // 1445/AC3 + ix, _, cands := previewBed(t, true) + if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil { + t.Fatal(err) + } + applyPreview(t, ix, avahiPreview(t, trunkOld, h1)) + if got := fpOf(t, cands); got.FP2 != "" || got.None2 != NonePreviewStale { + t.Errorf("candidate avahi = %+v, want it out of date", got) + } + // A preview onto the trunk head now makes it good again, and the trunk moving again makes it stale. + applyPreview(t, ix, avahiPreview(t, trunkNow, h1)) + if got := fpOf(t, cands); got.FP2 == "" { + t.Fatalf("after the right preview: %+v", got) + } + if err := ix.ApplySource(context.Background(), 6, delivery.Source{Group: "g", Heads: map[string]string{catalogueRepo: "8888888888888888888888888888888888888888"}}); err != nil { + t.Fatal(err) + } + if got := fpOf(t, cands); got.None2 != NonePreviewStale { + t.Errorf("after the trunk moved: %+v", got) + } +} + +func TestAPreviewHeardBeforeItsPullRequestGivesTheFingerprintWhenTheCandidateIsWritten(t *testing.T) { // 1445/AC2 + ix, mods, cands := previewBed(t, true) + applyPreview(t, ix, avahiPreview(t, trunkNow, h1)) + if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil { + t.Fatal(err) + } + trunk, _ := moduleOf(t, mods, "avahi") + if got := fpOf(t, cands); got.FP2 != trunk.FP2 { + t.Errorf("candidate avahi = %+v, want %s", got, trunk.FP2) + } + // The same pull request at a new head has no preview of it yet. + if err := ix.Apply(context.Background(), avahiPull(t, h2)); err != nil { + t.Fatal(err) + } + if got := fpOf(t, cands); got.FP2 != "" || got.None2 != NoneNoPreview { + t.Errorf("a new head: %+v, want no preview", got) + } +} + +func TestAPreviewNamingNoManifestOfTheModulesDirectoryGivesAReason(t *testing.T) { + ix, _, cands := previewBed(t, true) + if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil { + t.Fatal(err) + } + pv := avahiPreview(t, trunkNow, h1) + pv.Modules = map[string]delivery.PreviewModule{"modules/other": pv.Modules["modules/avahi"]} + applyPreview(t, ix, pv) + if got := fpOf(t, cands); got.None2 != NonePreviewNoModule || !strings.Contains(NonePreviewNoModule, "manifest") { + t.Errorf("candidate avahi = %+v", got) + } +} + +func TestTheSamePreviewAppliedAgainWritesNothing(t *testing.T) { + ix, _, cands := previewBed(t, true) + if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil { + t.Fatal(err) + } + applyPreview(t, ix, avahiPreview(t, trunkNow, h1)) + w := cands.writes + applyPreview(t, ix, avahiPreview(t, trunkNow, h1)) + if cands.writes != w { + t.Errorf("candidates written %d times for the same preview, want none", cands.writes-w) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/source.go b/modules/mesh-delivery-indexer/internal/index/source.go index 19a5b7b..2235218 100644 --- a/modules/mesh-delivery-indexer/internal/index/source.go +++ b/modules/mesh-delivery-indexer/internal/index/source.go @@ -44,7 +44,7 @@ func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Sourc ix.source = sourceNow{rev: rev, heads: maps.Clone(s.Heads)} ix.mu.Unlock() if ix.Modules == nil { - return nil + return ix.RecomputeCandidates(ctx) } repositories := slices.Sorted(maps.Keys(s.Heads)) for _, repository := range repositories { @@ -76,5 +76,6 @@ func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Sourc } } } - return nil + // A trunk that moved makes the previews made onto the old one out of date. + return ix.RecomputeCandidates(ctx) } diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 9b4dc04..ab29381 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -134,6 +134,14 @@ func (ix *Indexer) Recompute(ctx context.Context) error { return ix.recomputeWit // of what the bucket holds, are computed with the others, and are written once, with their final fingerprint or reason // (issue 1444: a merge writes each module once). A record of the overlay that has no kept inputs is written as it is. func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]delivery.Module) error { + if err := ix.recomputeModules(ctx, overlay); err != nil { + return err + } + // What the modules moved to reaches the candidates' fingerprints too (a base, the builder, a toolchain). + return ix.RecomputeCandidates(ctx) +} + +func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]delivery.Module) error { if ix.Modules == nil { return nil } diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 90d96c3..055782f 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -79,12 +79,12 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build if want := []state{{Name: "candidates", History: 10}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } - if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-controller.builds", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { + if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-controller.builds", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } // "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control // internal/broker/state.go:62-93). - if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-delivery-forge-bridge.forge.**"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-delivery-checker.preview.written", "mesh-delivery-forge-bridge.forge.**"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index fcfe983..cb58b8b 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -5,6 +5,7 @@ "consumes": [ "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", + "mesh-delivery-checker.preview.written", "mesh-delivery-forge-bridge.forge.**" ], "state": [ @@ -16,6 +17,7 @@ "mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-controller.builds", + "mesh-delivery-checker.previews", "mesh-delivery-gate.source" ], "own-secrets": { From 96ba6bd3dcfabd472b0342e24ad1ad001eabd933 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 21:20:42 +0200 Subject: [PATCH 06/19] Say the verdict in a line of at most 140 characters and owe the status mesh/check in the checker's bucket, which the forge bridge performs once per revision; the checker's stream checks wakes the readers of its buckets (issues 1682, 1683) --- .../cmd/mesh-delivery-checker/judge.go | 2 +- .../cmd/mesh-delivery-checker/main.go | 22 ++- .../internal/check/checker.go | 42 +++-- .../internal/check/checker_test.go | 20 ++ .../internal/check/examiner_test.go | 4 +- .../internal/check/judge.go | 4 +- .../internal/check/line.go | 171 +++++++++++++++++- .../internal/check/line_test.go | 134 ++++++++++++++ .../internal/check/preview.go | 2 +- .../mesh-delivery-checker/manifest_test.go | 4 +- modules/mesh-delivery-checker/module.json | 2 +- .../cmd/mesh-delivery-forge-bridge/gate.go | 29 ++- .../mesh-delivery-forge-bridge/gate_test.go | 54 ++++++ .../integration_test.go | 83 +++++++++ .../cmd/mesh-delivery-forge-bridge/late.go | 82 +++++++++ .../cmd/mesh-delivery-forge-bridge/main.go | 20 +- .../manifest_test.go | 4 +- .../mesh-delivery-forge-bridge/module.json | 4 +- .../cmd/mesh-delivery-indexer/previews.go | 6 +- 19 files changed, 646 insertions(+), 43 deletions(-) create mode 100644 modules/mesh-delivery-checker/internal/check/line_test.go create mode 100644 modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/late.go diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go index 0d55bda..46a0100 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go @@ -23,7 +23,7 @@ type judgeDeps struct { kept check.KV previews check.KV repos *repoModules - announce func(ctx context.Context, key string) + announce check.Announcer logf func(format string, args ...any) } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index 537680e..8b03ec9 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -32,9 +32,11 @@ import ( "mesh-delivery-checker/internal/check" ) -// previewWrittenSubject is the subject of the checker's preview.written event, on its stream previews (the bus names the -// stream PREVIEWS); the indexer's watch of the previews bucket is woken by it. -const previewWrittenSubject = "mesh.stream." + delivery.CheckerModule + ".event.preview.written" +// eventSubject is the subject of an event of the checker's stream checks (the bus names the stream CHECKS): the indexer's +// watch of the previews bucket and the forge bridge's watch of the owed bucket are woken by them. +func eventSubject(event string) string { + return "mesh.stream." + delivery.CheckerModule + ".event." + event +} const ( // defaultStepBound is how long the checker waits for one step of a build module (a fetch downloads the Go modules, a @@ -196,18 +198,20 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Link: link, Logf: logger.Printf, } + announce := func(ctx context.Context, event, key string) { + body, _ := json.Marshal(map[string]string{"key": key}) + if _, err := js.Publish(ctx, eventSubject(event), body); err != nil { + logger.Printf("%s of %s was written and the part that reads it could not be told: %v", event, key, err) + } + } repos := &repoModules{manifests: kvStore{buckets[delivery.ManifestsBucket]}, logf: logger.Printf} ch.Judge = newJudge(judgeDeps{ runner: runner, tools: tools, presence: presence, have: have, places: places, trunks: trunks, kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]}, repos: repos, logf: logger.Printf, - announce: func(ctx context.Context, key string) { - body, _ := json.Marshal(map[string]string{"key": key}) - if _, err := js.Publish(ctx, previewWrittenSubject, body); err != nil { - logger.Printf("previews.%s was written and the indexer could not be told: %v", key, err) - } - }, + announce: announce, }) + ch.Announce = announce // The toolchains first, so a check that starts from the first candidate finds the nodes. watch := newToolchainWatch(ctx, js, tools, ch.NodesChanged, logger) diff --git a/modules/mesh-delivery-checker/internal/check/checker.go b/modules/mesh-delivery-checker/internal/check/checker.go index 9084403..84753ed 100644 --- a/modules/mesh-delivery-checker/internal/check/checker.go +++ b/modules/mesh-delivery-checker/internal/check/checker.go @@ -18,6 +18,17 @@ const DefaultBound = 30 * time.Minute // is judged again when the nodes change, by a watch and never a timer (ADR 0437 §8). const NodesWords = "build node qualifies" +// The events the checker says on its stream checks (module.json): a preview or a status was written, so a part that reads +// the bucket, which is made when the checker is assigned and may not exist when that part starts, finds it (ADR 0437 §8: no +// timer). The body is {"key": ...}. +const ( + EventPreviewWritten = "preview.written" + EventOwedWritten = "owed.written" +) + +// Announcer says an event of the checker's stream, with the key written. +type Announcer func(ctx context.Context, event, key string) + // Outcome is what judging a case gives: the verdict (state, failure, levels, times, composition; the checker adds the // group, the heads and the trunk) and, for the line, what the check knows of the builds of the modules it touches. type Outcome struct { @@ -56,8 +67,10 @@ type Checker struct { Hold func() bool // Link is the base of the address a status points at; the verdict key follows. Link string - Now func() time.Time - Logf func(format string, args ...any) + // Announce says a status was written (EventOwedWritten); nil says nothing. + Announce Announcer + Now func() time.Time + Logf func(format string, args ...any) once sync.Once sem chan struct{} @@ -294,7 +307,7 @@ func (ch *Checker) try(ctx context.Context, key string, cs *Case) { return } began := ch.now() - ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, nil) + ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, nil, nil) out := ch.judge(ctx, cs) if ctx.Err() != nil { return @@ -322,7 +335,7 @@ func (ch *Checker) try(ctx context.Context, key string, cs *Case) { ch.nodesFail[cs.Key] = epoch ch.mu.Unlock() } - ch.writeOwed(ctx, cs, v, out.Builds) + ch.writeOwed(ctx, cs, v, out.Builds, out.Slowest) ch.logf("%s: %s", cs.Key, v.State) } @@ -360,7 +373,7 @@ func failure(machine, words string) delivery.Verdict { // writeOwed writes, for each pull request judged, the status the forge bridge performs: pending while the check runs, success // or failure at its end (ADR 0484 clause 2). A key already holding the same value is not written again, so the bridge // performs one act for each change; an older head's key is never touched. -func (ch *Checker) writeOwed(ctx context.Context, cs *Case, v delivery.Verdict, builds []BuildNote) { +func (ch *Checker) writeOwed(ctx context.Context, cs *Case, v delivery.Verdict, builds []BuildNote, slowest []string) { for _, p := range cs.Pulls { key, err := delivery.CheckerOwedKey(p.Repository, p.Number, p.Head) if err != nil { @@ -369,32 +382,35 @@ func (ch *Checker) writeOwed(ctx context.Context, cs *Case, v delivery.Verdict, } state := map[string]string{delivery.VerdictRunning: "pending", delivery.VerdictPass: "success", delivery.VerdictFail: "failure"}[v.State] o := delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: state, - Target: strings.TrimSuffix(ch.Link, "/") + "/" + cs.Key, Text: Line(v, cs, p, builds)} + Target: strings.TrimSuffix(ch.Link, "/") + "/" + cs.Key, Text: Line(v, cs, p, builds, slowest)} data, err := delivery.EncodeCheckerOwed(o) if err != nil { ch.logf("%s: the status of %s cannot be written: %v", cs.Key, key, err) continue } - if err := putOnce(ctx, ch.Owed, key, data, func(cur []byte) bool { + wrote, err := putOnce(ctx, ch.Owed, key, data, func(cur []byte) bool { held, err := delivery.DecodeCheckerOwed(cur) o.V = delivery.Version return err == nil && delivery.SameOwed(held, o) - }); err != nil { + }) + if err != nil { ch.logf("%s: the status of %s cannot be written: %v", cs.Key, key, err) + } else if wrote && ch.Announce != nil { + ch.Announce(ctx, EventOwedWritten, key) } } } // putOnce writes data under key by create, or by compare-and-set when the key exists, and writes nothing when same says the // key holds the value already. -func putOnce(ctx context.Context, kv KV, key string, data []byte, same func(cur []byte) bool) error { +func putOnce(ctx context.Context, kv KV, key string, data []byte, same func(cur []byte) bool) (bool, error) { for try := 0; try < 4; try++ { cur, rev, found, err := kv.Get(ctx, key) if err != nil { - return err + return false, err } if found && same(cur) { - return nil + return false, nil } if !found { _, err = kv.Create(ctx, key, data) @@ -404,9 +420,9 @@ func putOnce(ctx context.Context, kv KV, key string, data []byte, same func(cur if errors.Is(err, ErrConflict) { continue } - return err + return err == nil, err } - return fmt.Errorf("%s changed under every one of 4 tries", key) + return false, fmt.Errorf("%s changed under every one of 4 tries", key) } // Sweep closes, at start, every verdict left running for a candidate that no longer holds those heads: nothing will check diff --git a/modules/mesh-delivery-checker/internal/check/checker_test.go b/modules/mesh-delivery-checker/internal/check/checker_test.go index 4a791f4..f57ed71 100644 --- a/modules/mesh-delivery-checker/internal/check/checker_test.go +++ b/modules/mesh-delivery-checker/internal/check/checker_test.go @@ -365,3 +365,23 @@ func TestSweepClosesARunningKeyNoCandidateHoldsAndLeavesOneItDoes(t *testing.T) t.Errorf("the live key = %+v, want it left for the check to take over", v) } } + +func TestTheForgeBridgeIsToldOnceForEachChangeOfAStatus(t *testing.T) { // 1683 (the checker's half of AC5) + ctx, j := context.Background(), passJudge() + r := newRig(t, j) + var told []string + r.ch.Announce = func(_ context.Context, event, key string) { told = append(told, event+" "+key) } + cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1)) + r.ch.Candidate(ctx, "feat%2Fx", cand) + r.ch.Wait() + key, _ := delivery.CheckerOwedKey(repoA, 7, head1) + want := []string{EventOwedWritten + " " + key, EventOwedWritten + " " + key} // pending, then success + if len(told) != 2 || told[0] != want[0] || told[1] != want[1] { + t.Fatalf("told %v, want the status said when pending and when success", told) + } + r.ch.Candidate(ctx, "feat%2Fx", cand) // nothing changed: nothing written, nothing said + r.ch.Wait() + if len(told) != 2 { + t.Errorf("told %v after a check that changed nothing", told) + } +} diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go index 270a504..ba1109d 100644 --- a/modules/mesh-delivery-checker/internal/check/examiner_test.go +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -508,11 +508,11 @@ func TestAPreviewIsWrittenOnceAndAnExistingKeyStands(t *testing.T) { func TestTheIndexerIsToldOfAPreviewOnceWhenItIsCreated(t *testing.T) { // 1445/AC4 (the checker's half) b := newBench(t) var told []string - b.ex.Announce = func(_ context.Context, key string) { told = append(told, key) } + b.ex.Announce = func(_ context.Context, event, key string) { told = append(told, event+" "+key) } b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) key, _ := delivery.PreviewKey(repoA, head1, trunkA) - if len(told) != 1 || told[0] != key { + if len(told) != 1 || told[0] != EventPreviewWritten+" "+key { t.Errorf("told %v, want once of %s", told, key) } } diff --git a/modules/mesh-delivery-checker/internal/check/judge.go b/modules/mesh-delivery-checker/internal/check/judge.go index c29508d..0a9ce4b 100644 --- a/modules/mesh-delivery-checker/internal/check/judge.go +++ b/modules/mesh-delivery-checker/internal/check/judge.go @@ -50,8 +50,8 @@ type Examiner struct { // Previews is the previews bucket; nil writes none. Previews KV Modules BuildModules - // Announce says a preview was written under the key (the checker's preview.written event); nil says nothing. - Announce func(ctx context.Context, key string) + // Announce says a preview was written under the key (EventPreviewWritten); nil says nothing. + Announce Announcer // Trunks are the trunk heads of every repository, for the siblings a repository's tests read. Trunks interface{ Heads() map[string]string } Now func() time.Time diff --git a/modules/mesh-delivery-checker/internal/check/line.go b/modules/mesh-delivery-checker/internal/check/line.go index d2aaced..db675c2 100644 --- a/modules/mesh-delivery-checker/internal/check/line.go +++ b/modules/mesh-delivery-checker/internal/check/line.go @@ -2,13 +2,26 @@ package check import ( "fmt" + "strings" + "unicode/utf8" "git.novox.be/novox/mesh-sdk/go/delivery" ) -// Line is the words of a status for one pull request of a case (ADR 0447 clause 3, ADR 0484 clause 7). -func Line(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote) string { - return fmt.Sprintf("%s: %s", v.State, short12(p.Head)) +// MaxLine is the most characters a status's line has: the forge shows a status's description in a short space, and the full +// report is linked (ADR 0447 clause 3, ADR 0484 clause 7). A line that would be longer drops its lesser parts in a fixed +// order, and is cut last. +const MaxLine = 140 + +// seconds is a time in milliseconds in the unit a person reads: 350ms, 4.1s, 72s. +func seconds(ms int64) string { + switch { + case ms < 1000: + return fmt.Sprintf("%dms", ms) + case ms < 10000: + return fmt.Sprintf("%.1fs", float64(ms)/1000) + } + return fmt.Sprintf("%ds", (ms+500)/1000) } func short12(s string) string { @@ -17,3 +30,155 @@ func short12(s string) string { } return s } + +// firstFailed is what failed first, in words: a conflict, a refused step or a test, named before anything else. +func firstFailed(v delivery.Verdict) string { + f := v.Failed + switch { + case f.Test != "": + if f.Words != "" { + return f.Test + ": " + f.Words + } + return f.Test + case f.Machine != "" && f.Words != "": + return f.Machine + ": " + f.Words + case f.Machine != "": + return f.Machine + } + return f.Words +} + +// pieces are the parts of a line, from the most to the least needed. +type pieces struct { + head, commits, counts, times, over, builds, composition, group string +} + +func (p pieces) join(drop int) string { + parts := []string{p.head, p.commits, p.counts} + if drop < 3 && p.times != "" { + parts = append(parts, p.times) + } + if p.over != "" { + parts = append(parts, p.over) + } + if drop < 1 && p.builds != "" { + parts = append(parts, p.builds) + } + if drop < 2 && p.group != "" { + parts = append(parts, p.group) + } + if p.composition != "" { + parts = append(parts, p.composition) + } + out := make([]string, 0, len(parts)) + for _, s := range parts { + if s != "" { + out = append(out, s) + } + } + return strings.Join(out, " | ") +} + +func levelWords(name string, r delivery.LevelResult) string { + s := fmt.Sprintf("%s %d run, %d kept", name, r.Run, r.Kept) + if r.Skipped > 0 { + s += fmt.Sprintf(", %d skipped", r.Skipped) + } + return s +} + +func compositionWords(c *delivery.Composition) string { + switch { + case c == nil, c.State == delivery.CompositionNotRun: + return "composition not yet run" + case c.State == delivery.CompositionPass: + return "composition passed with " + short12(c.SDK) + } + return "composition failed: " + c.Words +} + +func buildWords(builds []BuildNote) string { + if len(builds) == 0 { + return "" + } + parts := make([]string, len(builds)) + for i, b := range builds { + parts[i] = b.Module + " " + b.State + } + return "builds: " + strings.Join(parts, ", ") +} + +// parts makes the pieces of the line of a pull request of a case. +func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string) pieces { + var pc pieces + commits := short12(p.Head) + " on " + short12(p.Trunk) + switch v.State { + case delivery.VerdictRunning: + pc.head = "checking " + commits + case delivery.VerdictPass: + pc.head = "pass " + commits + default: + pc.head = "failed: " + firstFailed(v) + pc.commits = commits + } + var counts []string + for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} { + if r, ok := v.Levels[level]; ok { + counts = append(counts, levelWords(level, r)) + } + } + pc.counts = strings.Join(counts, " | ") + if v.State != delivery.VerdictRunning { + t := v.Took + var times []string + for _, x := range []struct { + name string + ms int64 + }{{"queue", t.Queue}, {"clone", t.Clone}, {"unit", t.Unit}, {"int setup", t.IntegrationSetup}, {"int run", t.IntegrationRun}, {"composition", t.Composition}} { + if x.ms > 0 { + times = append(times, x.name+" "+seconds(x.ms)) + } + } + if len(times) > 0 { + pc.times = strings.Join(times, ", ") + } + } + if len(v.OverBound) > 0 { + pc.over = "over bound: " + strings.Join(v.OverBound, ", ") + if len(slowest) > 0 { + pc.over += " (slowest " + strings.Join(slowest, ", ") + ")" + } + } + pc.builds = buildWords(builds) + if v.State != delivery.VerdictRunning { + pc.composition = compositionWords(v.Composition) + } + if cs != nil && len(cs.Pulls) > 1 { + pc.group = fmt.Sprintf("group %s of %d pull requests", cs.Group, len(cs.Pulls)) + if v.State != delivery.VerdictPass { + pc.group = fmt.Sprintf("%s#%d in group %s", p.Repository, p.Number, cs.Group) + } + } + return pc +} + +// Report is the full line of a pull request's status, with no limit: what the read tools and the full report say. +func Report(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string) string { + return parts(v, cs, p, builds, slowest).join(0) +} + +// Line is the words of a status for one pull request of a case (ADR 0447 clause 3, ADR 0484 clause 7): the verdict first (a +// failure names the conflict, the refused step or the test before anything), the head and the trunk commit as twelve +// characters each, the tests run, kept and skipped per level, the time of each part, a level over its bound with its slowest +// tests, the builds of the touched modules, and that the composition was not yet run. It is at most MaxLine characters: the +// builds, then the group, then the times drop in that order, and the line is cut at last. +func Line(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string) string { + pc := parts(v, cs, p, builds, slowest) + for drop := 0; drop <= 3; drop++ { + if s := pc.join(drop); utf8.RuneCountInString(s) <= MaxLine { + return s + } + } + runes := []rune(pc.join(3)) + return string(runes[:MaxLine-1]) + "…" +} diff --git a/modules/mesh-delivery-checker/internal/check/line_test.go b/modules/mesh-delivery-checker/internal/check/line_test.go new file mode 100644 index 0000000..ccbce03 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/line_test.go @@ -0,0 +1,134 @@ +package check + +import ( + "strings" + "testing" + "unicode/utf8" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +func lineCase(t testing.TB, pulls ...delivery.CandidatePull) (*Case, Pull) { + cs := caseOf(t, pulls...) + return cs, cs.Pulls[0] +} + +func passVerdict() delivery.Verdict { + return delivery.Verdict{State: delivery.VerdictPass, + Levels: map[string]delivery.LevelResult{delivery.LevelUnit: {Run: 12, Kept: 301}, delivery.LevelIntegration: {Run: 3, Kept: 40}}, + Took: delivery.Took{Queue: 800, Clone: 4100, Unit: 400, IntegrationSetup: 3200, IntegrationRun: 12000}, + Composition: &delivery.Composition{State: delivery.CompositionNotRun}} +} + +func TestAPassOpensWithTheVerdictAndSaysTheCommitsTheCountsTheTimesAndThatTheCompositionWasNotRun(t *testing.T) { // 1300/AC1 + cs, p := lineCase(t, pullOf(repoA, 7, head1)) + full := Report(passVerdict(), cs, p, nil, nil) + for _, want := range []string{"pass 111111111111 on aaaaaaaaaaaa", "unit 12 run, 301 kept", "integration 3 run, 40 kept", "queue 800ms", "clone 4.1s", "unit 400ms", "int setup 3.2s", "int run 12s", "composition not yet run"} { + if !strings.Contains(full, want) { + t.Errorf("the report %q lacks %q", full, want) + } + } + if !strings.HasPrefix(full, "pass ") || strings.Contains(full, "composition passed") { + t.Errorf("report = %q", full) + } + line := Line(passVerdict(), cs, p, nil, nil) + if n := utf8.RuneCountInString(line); n > MaxLine { + t.Errorf("the line is %d characters, want at most %d: %q", n, MaxLine, line) + } + if !strings.HasPrefix(line, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(line, "composition not yet run") || !strings.Contains(line, "unit 12 run, 301 kept") { + t.Errorf("line = %q", line) + } +} + +func TestAFailureNamesTheTestTheConflictOrTheRefusedStepFirst(t *testing.T) { // 1300/AC2 + cs, p := lineCase(t, pullOf(repoA, 7, head1)) + for name, c := range map[string]struct { + v delivery.Verdict + want string + }{ + "test": {delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Test: "unit: m/a.TestX", Words: "want 4"}}, "failed: unit: m/a.TestX: want 4"}, + "conflict": {delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "shanks", Words: "novox/mesh-delivery#7 does not merge into the trunk: files in conflict: go.mod"}}, "failed: shanks: novox/mesh-delivery#7 does not merge into the trunk: files in conflict: go.mod"}, + "refused": {delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "mesh-build-go", Words: "no go build node qualifies: builders disagree"}}, "failed: mesh-build-go: no go build node qualifies"}, + } { + line := Line(c.v, cs, p, nil, nil) + if !strings.HasPrefix(line, c.want) || !strings.Contains(line, "111111111111 on aaaaaaaaaaaa") { + t.Errorf("%s: line = %q, want it to open with %q and name the commits", name, line, c.want) + } + } +} + +func TestALevelOverItsBoundIsNamedWithItsSlowestTestsAndTheVerdictIsNotChanged(t *testing.T) { // 1300/AC3 + cs, p := lineCase(t, pullOf(repoA, 7, head1)) + v := passVerdict() + v.OverBound = []string{"unit"} + line := Line(v, cs, p, nil, []string{"unit pkg.TestSlow 7.0s"}) + if !strings.HasPrefix(line, "pass ") || !strings.Contains(Report(v, cs, p, nil, []string{"unit pkg.TestSlow 7.0s"}), "over bound: unit (slowest unit pkg.TestSlow 7.0s)") { + t.Errorf("line = %q", line) + } +} + +func TestSkippedTestsAreSaidAndNeverAFlatPass(t *testing.T) { // 1300/AC6 + cs, p := lineCase(t, pullOf(repoA, 7, head1)) + v := passVerdict() + v.Levels[delivery.LevelUnit] = delivery.LevelResult{Run: 12, Kept: 301, Skipped: 2} + if line := Line(v, cs, p, nil, nil); !strings.Contains(line, "unit 12 run, 301 kept, 2 skipped") { + t.Errorf("line = %q", line) + } +} + +func TestAGroupsMembersEachCarryTheSameVerdict(t *testing.T) { // 1300/AC4 + cs := caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2)) + a := Line(passVerdict(), cs, cs.Pulls[0], nil, nil) + b := Line(passVerdict(), cs, cs.Pulls[1], nil, nil) + if !strings.HasPrefix(a, "pass ") || !strings.HasPrefix(b, "pass ") || !strings.Contains(Report(passVerdict(), cs, cs.Pulls[0], nil, nil), "group feat/x of 2 pull requests") { + t.Errorf("lines %q and %q", a, b) + } + if !strings.Contains(a, "111111111111") || !strings.Contains(b, "222222222222") { + t.Errorf("each member's line names its own head: %q and %q", a, b) + } +} + +func TestTheBuildsOfTheTouchedModulesAreSaidAndDroppedFirstWhenTheLineIsTooLong(t *testing.T) { // ADR 0484 clause 7 + cs, p := lineCase(t, pullOf(repoA, 7, head1)) + builds := []BuildNote{{"avahi", "built"}, {"mesh-tools", "waiting"}} + if r := Report(passVerdict(), cs, p, builds, nil); !strings.Contains(r, "builds: avahi built, mesh-tools waiting") { + t.Errorf("report = %q", r) + } + line := Line(passVerdict(), cs, p, builds, nil) + if utf8.RuneCountInString(line) > MaxLine { + t.Errorf("line %q is too long", line) + } + short := passVerdict() + short.Took = delivery.Took{} + if l := Line(short, cs, p, builds[:1], nil); !strings.Contains(l, "builds: avahi built") { + t.Errorf("a short line keeps its builds: %q", l) + } +} + +func TestALineIsNeverLongerThanTheLimit(t *testing.T) { // 1300/AC1 + cs, p := lineCase(t, pullOf(repoA, 7, head1)) + v := delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Test: "integration: " + strings.Repeat("pkg/", 50) + ".TestSomething", Words: strings.Repeat("because ", 40)}} + if line := Line(v, cs, p, nil, nil); utf8.RuneCountInString(line) > MaxLine || !strings.HasPrefix(line, "failed: integration: ") { + t.Errorf("line = %q (%d)", line, utf8.RuneCountInString(line)) + } +} + +func TestARunningCheckIsPending(t *testing.T) { + cs, p := lineCase(t, pullOf(repoA, 7, head1)) + if line := Line(delivery.Verdict{State: delivery.VerdictRunning}, cs, p, nil, nil); line != "checking 111111111111 on aaaaaaaaaaaa" { + t.Errorf("line = %q", line) + } +} + +func TestThePassedCompositionIsSaidOnlyWhenItWasRun(t *testing.T) { // ADR 0484 clause 6 + cs, p := lineCase(t, pullOf(repoA, 7, head1)) + v := passVerdict() + v.Composition = &delivery.Composition{State: delivery.CompositionPass, SDK: "afbe371003a570f088be4a394d83aa7a9458f7ab"} + if r := Report(v, cs, p, nil, nil); !strings.Contains(r, "composition passed with afbe371003a5") { + t.Errorf("report = %q", r) + } + v.Composition = nil + if r := Report(v, cs, p, nil, nil); !strings.Contains(r, "composition not yet run") || strings.Contains(r, "composition passed") { + t.Errorf("a verdict with no composition: %q", r) + } +} diff --git a/modules/mesh-delivery-checker/internal/check/preview.go b/modules/mesh-delivery-checker/internal/check/preview.go index e44a9d5..a037246 100644 --- a/modules/mesh-delivery-checker/internal/check/preview.go +++ b/modules/mesh-delivery-checker/internal/check/preview.go @@ -63,7 +63,7 @@ func (j *Examiner) writePreview(ctx context.Context, p Pull, fetched []Fetched) return } if err == nil && j.Announce != nil { - j.Announce(ctx, key) + j.Announce(ctx, EventPreviewWritten, key) } } } diff --git a/modules/mesh-delivery-checker/manifest_test.go b/modules/mesh-delivery-checker/manifest_test.go index 1517d59..08d68ba 100644 --- a/modules/mesh-delivery-checker/manifest_test.go +++ b/modules/mesh-delivery-checker/manifest_test.go @@ -111,8 +111,8 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 if m.SecretsOwner != "mesh-delivery-checker" { t.Errorf("secrets-owner = %q", m.SecretsOwner) } - // The checker says each time it wrote a preview, so the indexer's watch finds the bucket (ADR 0484 clause 3). - if want := []stream{{Name: "previews", Events: []string{"preview.**"}}}; !reflect.DeepEqual(m.Streams, want) { + // The checker says each time it wrote a preview or a status, so the indexer's and the forge bridge's watches find the buckets (ADR 0484 clauses 2 and 3). + if want := []stream{{Name: "checks", Events: []string{"preview.**", "owed.**"}}}; !reflect.DeepEqual(m.Streams, want) { t.Errorf("streams = %+v, want %+v", m.Streams, want) } for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats", "bus"} { diff --git a/modules/mesh-delivery-checker/module.json b/modules/mesh-delivery-checker/module.json index eef0cc1..bb08186 100644 --- a/modules/mesh-delivery-checker/module.json +++ b/modules/mesh-delivery-checker/module.json @@ -7,7 +7,7 @@ "mesh-build-node.toolchain.reported" ], "streams": [ - { "name": "previews", "events": ["preview.**"] } + { "name": "checks", "events": ["preview.**", "owed.**"] } ], "state": [ { "name": "kept", "history": 1 }, diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate.go index a98b7cd..410ed63 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate.go @@ -91,9 +91,34 @@ func (p *performer) refuse(ctx context.Context, doneKey string, rev uint64, a br return true } -// owed performs the act an owed key asks at revision rev, once. +// owed performs the act an owed key of the gate asks at revision rev, once. func (p *performer) owed(ctx context.Context, key string, rev uint64, value []byte) { - doneKey := "owed." + key + p.perform(ctx, "owed."+key, key, rev, value) +} + +// checkOwed performs the status the checker owes on a head (mesh-delivery-checker_owed; novox/hq ADR 0484 clause 2) at +// revision rev, once. Its keys are kept apart from the gate's, and a key that is not the checker's one act, the status +// mesh/check, is left. +func (p *performer) checkOwed(ctx context.Context, key string, rev uint64, value []byte) { + doneKey := "check." + key + if p.seen(ctx, doneKey, rev) { + return + } + if _, _, _, err := delivery.ParseCheckerOwedKey(key); err != nil { + p.logger.Printf("%s at revision %d is not a status of the checker, and is left: %v", key, rev, err) + p.keep(ctx, doneKey, rev) + return + } + if _, err := delivery.DecodeCheckerOwed(value); err != nil { + p.logger.Printf("%s at revision %d is not a status of the checker, and is left: %v", key, rev, err) + p.keep(ctx, doneKey, rev) + return + } + p.perform(ctx, doneKey, key, rev, value) +} + +// perform does the act of an owed key once for the revision, under the bookkeeping key doneKey. +func (p *performer) perform(ctx context.Context, doneKey, key string, rev uint64, value []byte) { if p.seen(ctx, doneKey, rev) { return } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go index 608685e..2a7b64b 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go @@ -208,3 +208,57 @@ func TestAMergeWithTwoHandedStopsAtTheFirstRefusal(t *testing.T) { // ADR 0441 t.Errorf("calls %d messages %d, want the first refusal to stop the pass", len(f.calls), len(pub.data)) } } + +// checkerOwed is the status the checker owes, as it writes it (mesh-sdk go/delivery CheckerOwedKey, EncodeCheckerOwed). +func checkerOwed(t *testing.T, number int, state, text string) (string, []byte) { + t.Helper() + key, err := delivery.CheckerOwedKey("novox/mesh-delivery", number, headA) + if err != nil { + t.Fatal(err) + } + v, err := delivery.EncodeCheckerOwed(delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: state, Text: text, Target: "https://view.example/checks/g.0123456789ab"}) + if err != nil { + t.Fatal(err) + } + return key, v +} + +func TestTheCheckersStatusIsSetOncePerRevisionAndAKeyOfTheGateIsNotItsKey(t *testing.T) { // 1683/AC5 (ADR 0484 clause 2) + f := &forgeFake{} + p, _, done, _ := newPerformer(f) + key, v := checkerOwed(t, 12, "success", "pass aaaaaaaaaaaa on bbbbbbbbbbbb") + p.checkOwed(context.Background(), key, 3, v) + p.checkOwed(context.Background(), key, 3, v) + if len(f.calls) != 1 || f.calls[0].key != "gitea.gitea_commit_status" || + f.calls[0].body != `{"context":"mesh/check","description":"pass aaaaaaaaaaaa on bbbbbbbbbbbb","owner":"novox","repo":"mesh-delivery","sha":"`+headA+`","state":"success","target_url":"https://view.example/checks/g.0123456789ab"}` { + t.Fatalf("calls = %+v", f.calls) + } + p.checkOwed(context.Background(), key, 4, v) + if len(f.calls) != 2 { + t.Errorf("a new revision was not performed: %+v", f.calls) + } + if _, ok := done["check."+key]; !ok { + t.Errorf("revision kept under %v, want check.%s", done, key) + } + // The gate's own owed bookkeeping is another key: the same owed key seen in the gate's bucket is its own act. + if _, ok := done["owed."+key]; ok { + t.Errorf("the checker's status was kept under the gate's key: %v", done) + } +} + +func TestAnythingButTheCheckersStatusInItsBucketIsLeft(t *testing.T) { // ADR 0484 clause 2: the checker owes one act + f := &forgeFake{} + p, _, done, logs := newPerformer(f) + // A comment, and a status of another context: the SDK refuses both to the checker's encoder, so they are written raw. + commentKey, commentValue := owedKV(t, "novox/mesh-delivery", 12, "comment.1", delivery.Owed{Act: delivery.OwedComment, Text: "hello"}) + gateKey, gateValue := owedKV(t, "novox/mesh-delivery", 12, "status=mesh/merge-gate", + delivery.Owed{Act: delivery.OwedStatus, Context: "mesh/merge-gate", State: "success", Text: "x", Target: "https://x.example"}) + p.checkOwed(context.Background(), commentKey, 1, commentValue) + p.checkOwed(context.Background(), gateKey, 1, gateValue) + if len(f.calls) != 0 { + t.Errorf("the forge was called: %+v", f.calls) + } + if !strings.Contains(logs.String(), commentKey) || !strings.Contains(logs.String(), gateKey) || len(done) != 2 { + t.Errorf("log %q, kept %v: both keys are named and kept so they are not tried again", logs.String(), done) + } +} diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go index 3281f5c..679a8ae 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go @@ -30,6 +30,10 @@ import ( // The gate's buckets mesh-delivery-gate_merges and mesh-delivery-gate_owed are read and watched (the manifest's reads), the // bucket performed is the bridge's own (its state), and the five forge tools of the gate's acts are invoked (issue 1341, 1342). // +// The checker's owed bucket is read like the gate's (reads) and the checker's stream checks is heard like EVENTS is (consumes +// mesh-delivery-checker.owed.written; ADR 0484 clause 2): those entries are written by analogy with the ones the controller +// composes for the gate's buckets and for the EVENTS consumer, not taken from a composed file. +// // The lists are what mesh-controller's broker.PermissionsFor answers (internal/broker/nats.go:577) for a module principal // with the manifest of module.json: consumes the three gitea.pull events, invokes gitea.gitea_open_pulls, // gitea.gitea_commit_trees and gitea.gitea_get_file, state heads, and the stream forge; `**` is written `>` on the wire (modstreams.go @@ -104,6 +108,14 @@ accounts { "mesh.mod.gitea.tool.gitea_add_comment" "mesh.mod.gitea.tool.gitea_add_comment.*" "mesh.stream.mesh-delivery-forge-bridge.event.forge.>" + "$JS.ACK.CHECKS.n1_mesh-delivery-forge-bridge.>" + "$JS.API.CONSUMER.INFO.CHECKS.n1_mesh-delivery-forge-bridge" + "$JS.API.CONSUMER.MSG.NEXT.CHECKS.n1_mesh-delivery-forge-bridge" + "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-checker_owed.>" + "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-checker_owed.>" + "$JS.API.DIRECT.GET.KV_mesh-delivery-checker_owed.>" + "$JS.API.STREAM.INFO.KV_mesh-delivery-checker_owed" + "$JS.FC.KV_mesh-delivery-checker_owed.>" ] } subscribe { allow: [ "_INBOX.n1.mesh-delivery-forge-bridge.>" @@ -205,6 +217,15 @@ func startBed(t *testing.T, openReply string, prepare func(ctx context.Context, if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "FORGE", Subjects: []string{"mesh.stream.mesh-delivery-forge-bridge.event.forge.>"}}); err != nil { t.Fatal(err) } + // The checker's stream checks and the bridge's durable consumer on its owed.written event; the checker's owed bucket is made + // only by the test that needs it, as when the checker is assigned after the bridge started. + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "CHECKS", Subjects: []string{"mesh.stream.mesh-delivery-checker.event.>"}}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateOrUpdateConsumer(ctx, "CHECKS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-forge-bridge", AckPolicy: jetstream.AckExplicitPolicy, + FilterSubjects: []string{"mesh.stream.mesh-delivery-checker.event.owed.written"}}); err != nil { + t.Fatal(err) + } // The bridge's state bucket, made by the controller from the manifest's state entry before the process starts. if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: headsBucket, History: 1}); err != nil { t.Fatal(err) @@ -706,3 +727,65 @@ func forgeMerged(t *testing.T, b *bed) []string { } func sha256Sum(b []byte) []byte { s := sha256.Sum256(b); return s[:] } + +// TestBridgePerformsTheCheckersStatusOncePerRevisionOfItsOwedKey: the checker's owed bucket is made after the bridge started; +// the checker writes the status mesh/check it owes on a head (the SDK's CheckerOwedKey and EncodeCheckerOwed, which the checker +// itself writes with) and says so on its stream; the bridge sets the status on the forge within 10 s, with the context +// mesh/check, the state and the line; the same revision is not set again, a new revision is (ADR 0484 clause 2). +func TestBridgePerformsTheCheckersStatusOncePerRevisionOfItsOwedKey(t *testing.T) { // 1683/AC5, ADR 0484 §2 + b := startBed(t, `{"result":{"pulls":[]}}`, nil) + calls := make(chan string, 8) + sub, err := b.ctl.Subscribe("mesh.mod.gitea.tool.gitea_commit_status", func(m *nats.Msg) { + calls <- string(m.Data) + _ = m.Respond([]byte(`{"result":{},"node":"n0"}`)) + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + if err := b.ctl.Flush(); err != nil { + t.Fatal(err) + } + kv, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: delivery.CheckerOwedBucket, History: delivery.CheckerOwedHistory}) + if err != nil { + t.Fatal(err) + } + key, err := delivery.CheckerOwedKey("novox/mesh-delivery", 12, head) + if err != nil { + t.Fatal(err) + } + put := func(state, text string) { + v, err := delivery.EncodeCheckerOwed(delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: state, Text: text, Target: "https://view.example/checks/g.0123456789ab"}) + if err != nil { + t.Fatal(err) + } + if _, err := kv.Put(b.ctx, key, v); err != nil { + t.Fatal(err) + } + if _, err := b.js.Publish(b.ctx, owedWrittenSubject, []byte(`{"key":"`+key+`"}`)); err != nil { + t.Fatal(err) + } + } + next := func() string { + select { + case got := <-calls: + return got + case <-time.After(10 * time.Second): + t.Fatal("the status was not set within 10 s") + } + return "" + } + put("pending", "checking 0123456789ab on aaaaaaaaaaaa") + if got := next(); !strings.Contains(got, `"context":"mesh/check"`) || !strings.Contains(got, `"state":"pending"`) || !strings.Contains(got, `"sha":"`+head+`"`) || !strings.Contains(got, `"repo":"mesh-delivery"`) { + t.Fatalf("first call = %s", got) + } + put("success", "pass 0123456789ab on aaaaaaaaaaaa | unit 2 run, 3 kept") + if got := next(); !strings.Contains(got, `"state":"success"`) || !strings.Contains(got, "unit 2 run, 3 kept") { + t.Fatalf("second call = %s", got) + } + select { + case extra := <-calls: + t.Errorf("a third call was made: %s", extra) + case <-time.After(500 * time.Millisecond): + } +} diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/late.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/late.go new file mode 100644 index 0000000..c25f87a --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/late.go @@ -0,0 +1,82 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log" + "strings" + "sync" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// checksStream is the stream the checker carries its events on, as the bus names it (declared by the checker as the stream +// checks, heard by this module through consumes; novox/hq ADR 0484 clause 2). +const checksStream = "CHECKS" + +// owedWrittenSubject is the subject of the checker's owed.written event. +const owedWrittenSubject = "mesh.stream." + delivery.CheckerModule + ".event.owed.written" + +// lateWatch watches a bucket that may be made after the bridge started: the checker's owed bucket. The bridge holds no timer +// (ADR 0437 §8), so it watches the bucket where it exists at start, and otherwise when the checker says it wrote a status +// (owed.written, handled by Written). +type lateWatch struct { + ctx context.Context + js interface { + KeyValue(ctx context.Context, bucket string) (jetstream.KeyValue, error) + } + bucket string + react func(ctx context.Context, key string, rev uint64, value []byte) + logger *log.Logger + errs chan error + mu sync.Mutex + watched bool +} + +// ensure starts the watch if the bucket exists and is not watched. +func (w *lateWatch) ensure() error { + w.mu.Lock() + defer w.mu.Unlock() + if w.watched { + return nil + } + kv, err := w.js.KeyValue(w.ctx, w.bucket) + switch { + case errors.Is(err, jetstream.ErrBucketNotFound): + w.logger.Printf("the bucket %s does not exist yet: it is watched when the checker says it wrote a status", w.bucket) + return nil + case err != nil: + return fmt.Errorf("the bucket %s: %w", w.bucket, err) + } + w.watched = true + go func() { + w.errs <- watch(w.ctx, kv, func(key string, rev uint64, value []byte) { + hctx, cancel := context.WithTimeout(w.ctx, syncDeadline) + defer cancel() + w.react(hctx, key, rev, value) + }) + }() + return nil +} + +// Written takes one owed.written event of the checker in: its bucket is watched if it is not. A subject that is not the +// checker's is terminated; a failure stays unacknowledged, and the server delivers it again. +func (w *lateWatch) Written(m interface { + Subject() string + Ack() error + Term() error +}) { + if !strings.HasPrefix(m.Subject(), owedWrittenSubject) { + w.logger.Printf("%s: not the checker's owed.written, dropped", m.Subject()) + _ = m.Term() + return + } + if err := w.ensure(); err != nil { + w.logger.Printf("%s: left for redelivery: %v", m.Subject(), err) + return + } + _ = m.Ack() +} diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go index 0438f1c..fc1aaff 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go @@ -202,7 +202,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return fmt.Errorf("the bucket %s: %w", performedBucket, err) } perf := &performer{call: call, pub: rec, done: kvRevisions{performedKV}, logger: logger} - watching := make(chan error, 2) + watching := make(chan error, 3) for _, w := range []struct { bucket string react func(ctx context.Context, key string, rev uint64, value []byte) @@ -225,6 +225,24 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro }) }() } + // The checker's statuses (mesh/check, unrequired until release R4f): its bucket is made when the checker is assigned, which + // may be after the bridge started, and the bridge holds no timer; the checker says on its stream each time it wrote a + // status, and the bucket is then watched (late.go). + late := &lateWatch{ctx: ctx, js: js, bucket: delivery.CheckerOwedBucket, react: perf.checkOwed, logger: logger, errs: watching} + if err := late.ensure(); err != nil { + return err + } + switch ccons, err := js.Consumer(ctx, checksStream, cred.Node+"_"+cred.Module); { + case err != nil: + logger.Printf("the consumer %s_%s on %s does not exist yet, so the checker's statuses are not heard until the bridge starts again: %v", cred.Node, cred.Module, checksStream, err) + default: + cc, err := ccons.Consume(func(m jetstream.Msg) { late.Written(m) }, + jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) })) + if err != nil { + return err + } + defer cc.Stop() + } if err := said(ctx); err != nil { return err } diff --git a/modules/mesh-delivery-forge-bridge/manifest_test.go b/modules/mesh-delivery-forge-bridge/manifest_test.go index d6682cc..40fb8a8 100644 --- a/modules/mesh-delivery-forge-bridge/manifest_test.go +++ b/modules/mesh-delivery-forge-bridge/manifest_test.go @@ -86,10 +86,10 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1 if !reflect.DeepEqual(m.Streams, []stream{{Name: "forge", Events: []string{"forge.**"}}}) { t.Errorf("streams = %+v", m.Streams) } - if want := []string{"gitea.pull.closed", "gitea.pull.merged", "gitea.pull.updated"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + if want := []string{"gitea.pull.closed", "gitea.pull.merged", "gitea.pull.updated", "mesh-delivery-checker.owed.written"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } - if want := []string{"mesh-delivery-gate.merges", "mesh-delivery-gate.owed"}; !reflect.DeepEqual(sorted(m.Reads), want) { + if want := []string{"mesh-delivery-checker.owed", "mesh-delivery-gate.merges", "mesh-delivery-gate.owed"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } if want := []string{"gitea.gitea_add_comment", "gitea.gitea_commit_status", "gitea.gitea_commit_trees", "gitea.gitea_delivery_view", diff --git a/modules/mesh-delivery-forge-bridge/module.json b/modules/mesh-delivery-forge-bridge/module.json index 25b68be..d411e5f 100644 --- a/modules/mesh-delivery-forge-bridge/module.json +++ b/modules/mesh-delivery-forge-bridge/module.json @@ -5,7 +5,8 @@ "consumes": [ "gitea.pull.updated", "gitea.pull.merged", - "gitea.pull.closed" + "gitea.pull.closed", + "mesh-delivery-checker.owed.written" ], "streams": [ { "name": "forge", "events": ["forge.**"] } @@ -15,6 +16,7 @@ { "name": "performed", "history": 1 } ], "reads": [ + "mesh-delivery-checker.owed", "mesh-delivery-gate.merges", "mesh-delivery-gate.owed" ], diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go index fcf6a31..c42221c 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go @@ -15,9 +15,9 @@ import ( "mesh-delivery-indexer/internal/index" ) -// PreviewsStream is the stream the checker carries its preview.written event on, as the bus names it (declared by the -// checker as the stream previews, heard by this module through consumes; novox/hq ADR 0484 clause 3). -const PreviewsStream = "PREVIEWS" +// PreviewsStream is the stream the checker carries its events on, as the bus names it (declared by the +// checker as the stream checks, heard by this module through consumes; novox/hq ADR 0484 clause 3). +const PreviewsStream = "CHECKS" // previewWrittenSubject is the subject of the checker's preview.written event. const previewWrittenSubject = "mesh.stream." + delivery.CheckerModule + ".event.preview.written" From 64599db3fa75a4cd636f5e27f5fbdc6702906b44 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 21:23:22 +0200 Subject: [PATCH 07/19] Capture the fetch-with-base fixtures from mesh-catalog's code, and prove the checker whole against a real bus with fake build modules (issues 1679, 1287, 1288, 1289, 1678) --- .../mesh-delivery-checker/integration_test.go | 497 ++++++++++++++++++ .../internal/check/examiner_test.go | 14 +- .../internal/check/testdata/captured/README | 10 +- .../testdata/captured/go-fetch-clean.json | 1 + .../testdata/captured/go-fetch-conflict.json | 1 + .../testdata/captured/node-fetch-clean.json | 1 + 6 files changed, 514 insertions(+), 10 deletions(-) create mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go create mode 100644 modules/mesh-delivery-checker/internal/check/testdata/captured/go-fetch-clean.json create mode 100644 modules/mesh-delivery-checker/internal/check/testdata/captured/go-fetch-conflict.json create mode 100644 modules/mesh-delivery-checker/internal/check/testdata/captured/node-fetch-clean.json diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go new file mode 100644 index 0000000..f8e50db --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go @@ -0,0 +1,497 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the +// controller's user and the checker's own principal .. It carries no per-principal permissions: the grants +// the controller composes from the manifest's state and reads (only the checker writes its buckets) are the controller's, +// and are proved by its own module check, not here. +const busConf = ` +listen: 127.0.0.1:-1 +jetstream { store_dir: %q } +accounts { + MESH { + jetstream: enabled + users = [ + { user: "ctl", password: "pw" } + { user: "n1.mesh-delivery-checker", password: "pw" } + ] + } +} +` + +const ( + repo = "novox/mesh-delivery" + head1 = "1111111111111111111111111111111111111111" + head2 = "2222222222222222222222222222222222222222" + trunk = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + merged = "fedcba9876543210fedcba9876543210fedcba98" // the merge commit of the captured fetch answer + digU = "aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11" + digI = "bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22" +) + +var captured = filepath.Join("..", "..", "internal", "check", "testdata", "captured") + +func readCaptured(t *testing.T, name string) map[string]any { + t.Helper() + raw, err := os.ReadFile(filepath.Join(captured, name+".json")) + if err != nil { + t.Fatal(err) + } + var v map[string]any + if err := json.Unmarshal(raw, &v); err != nil { + t.Fatal(err) + } + return v +} + +// fakeBuildModules serve the tools of mesh-build-go and mesh-build-node as the real ones do (ADR 0469 clause 2): a call on +// mesh.mod..call... answers at once with the step and its key, and the step's end is written to +// the key . of the module's steps bucket. What the steps end with are the results the build modules' own code +// marshalled (internal/check/testdata/captured, README). +type fakeBuildModules struct { + js jetstream.JetStream + ctx context.Context + n atomic.Int64 + conflict atomic.Bool + mu sync.Mutex + calls map[string]int // by `.` + testTool *testing.T +} + +func (f *fakeBuildModules) count(tool string) int { + f.mu.Lock() + defer f.mu.Unlock() + return f.calls[tool] +} + +func (f *fakeBuildModules) result(t *testing.T, tool string, args map[string]any) json.RawMessage { + level, _ := args["level"].(string) + var v any + switch { + case strings.HasSuffix(tool, "_fetch"): + name := "go-fetch-clean" + if f.conflict.Load() { + name = "go-fetch-conflict" + } + fetched := readCaptured(t, name) + fetched["commit"], fetched["base"] = args["commit"], args["base"] + v = fetched + case strings.HasSuffix(tool, "_digest"): + d := map[string]string{"unit": digU, "integration": digI}[level] + v = map[string]any{"level": level, "packages": map[string]string{"fixture/slow": d}} + case strings.HasSuffix(tool, "_test"): + r := readCaptured(t, "go-test-slow") + r["level"] = level + v = r + } + data, _ := json.Marshal(v) + return data +} + +func (f *fakeBuildModules) serve(t *testing.T, nc *nats.Conn, module string) { + steps, err := f.js.CreateKeyValue(f.ctx, jetstream.KeyValueConfig{Bucket: delivery.StepsBucket(module), History: 1}) + if err != nil { + t.Fatal(err) + } + sub, err := nc.Subscribe("mesh.mod."+module+".call.>", func(m *nats.Msg) { + tokens := strings.Split(m.Subject, ".") // mesh mod call + tool, node := tokens[4], tokens[5] + f.mu.Lock() + f.calls[module+"."+tool]++ + f.mu.Unlock() + var args map[string]any + _ = json.Unmarshal(m.Data, &args) + id := fmt.Sprintf("s%d", f.n.Add(1)) + key, _ := delivery.StepKey(node, id) + put := func(s delivery.Step) { + data, err := delivery.EncodeStep(s) + if err == nil { + _, err = steps.Put(f.ctx, key, data) + } + if err != nil { + t.Errorf("writing step %s: %v", key, err) + } + } + step := delivery.Step{ID: id, Tool: tool, Node: node, State: delivery.StepRunning, Started: time.Now().UTC()} + put(step) + answer, _ := json.Marshal(map[string]any{"result": map[string]any{"step": id, "key": key, "state": "running"}, "error": "", "node": node}) + _ = m.Respond(answer) + go func() { + time.Sleep(20 * time.Millisecond) + step.Ended = time.Now().UTC() + step.State, step.Result = delivery.StepPassed, f.result(t, tool, args) + put(step) + }() + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) +} + +type bed struct { + ctx context.Context + js jetstream.JetStream + buckets map[string]jetstream.KeyValue + tools map[string]jetstream.KeyValue + fake *fakeBuildModules + srvURL string + announce []string + mu sync.Mutex +} + +// startBed starts the bus, the buckets the controller makes from the checker's and the other parts' manifests, the build +// modules' streams and the checker's durable consumers on them, one node's toolchains, the gate's source naming the trunk, and +// the fake build modules. It runs no checker. +func startBed(t *testing.T) *bed { + dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "c") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + conf := filepath.Join(dir, "bus.conf") + if err := os.WriteFile(conf, []byte(fmt.Sprintf(busConf, filepath.Join(dir, "js"))), 0o600); err != nil { + t.Fatal(err) + } + opts, err := server.ProcessConfigFile(conf) + if err != nil { + t.Fatal(err) + } + opts.NoLog, opts.NoSigs = true, true + srv, err := server.NewServer(opts) + if err != nil { + t.Fatal(err) + } + go srv.Start() + if !srv.ReadyForConnections(10 * time.Second) { + t.Fatal("the bus did not come up") + } + t.Cleanup(srv.Shutdown) + ctl, err := nats.Connect(srv.ClientURL(), nats.UserInfo("ctl", "pw")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(ctl.Close) + js, err := jetstream.New(ctl) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + t.Cleanup(cancel) + b := &bed{ctx: ctx, js: js, buckets: map[string]jetstream.KeyValue{}, tools: map[string]jetstream.KeyValue{}, srvURL: srv.ClientURL()} + for module, stream := range toolchainStreams { + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{"mesh.stream." + module + ".event.toolchain.reported"}}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-checker", AckPolicy: jetstream.AckExplicitPolicy}); err != nil { + t.Fatal(err) + } + kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket(module), History: 1}) + if err != nil { + t.Fatal(err) + } + b.tools[module] = kv + } + // The checker's stream of events, which the indexer and the forge bridge hear. + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "CHECKS", Subjects: []string{"mesh.stream.mesh-delivery-checker.event.>"}}); err != nil { + t.Fatal(err) + } + for name, history := range map[string]int{delivery.CheckerVerdictsBucket: delivery.CheckerVerdictsHistory, delivery.CheckerKeptBucket: delivery.CheckerKeptHistory, + delivery.CheckerOwedBucket: delivery.CheckerOwedHistory, delivery.PreviewsBucket: delivery.PreviewsHistory, delivery.CandidatesBucket: delivery.CandidatesHistory, + delivery.ModulesBucket: delivery.ModulesHistory, delivery.ManifestsBucket: delivery.ManifestsHistory, delivery.SourceBucket: delivery.SourceHistory, + delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory} { + kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: uint8(history)}) + if err != nil { + t.Fatal(err) + } + b.buckets[name] = kv + } + b.setOnline(t, "n1", true) + for _, module := range []string{"mesh-build-go", "mesh-build-node"} { + b.setHealth(t, "n1", module) + } + b.fake = &fakeBuildModules{js: js, ctx: ctx, calls: map[string]int{}} + b.fake.serve(t, ctl, "mesh-build-go") + b.fake.serve(t, ctl, "mesh-build-node") + b.report(t, "mesh-build-go", "n1", "go", "go1.27.1", "1") + b.report(t, "mesh-build-node", "n1", "typescript", "tsc5", "1") + b.setTrunk(t, map[string]string{repo: trunk}) + return b +} + +func (b *bed) setTrunk(t *testing.T, heads map[string]string) { + t.Helper() + data, err := delivery.EncodeSource(delivery.Source{Group: "g", Heads: heads}) + if err != nil { + t.Fatal(err) + } + if _, err := b.buckets[delivery.SourceBucket].Put(b.ctx, delivery.SourceMainKey, data); err != nil { + t.Fatal(err) + } +} + +func (b *bed) setOnline(t *testing.T, node string, online bool) { + t.Helper() + data, _ := delivery.EncodePresence(delivery.Presence{Online: online}) + if _, err := b.buckets[delivery.PresenceBucket].Put(b.ctx, node, data); err != nil { + t.Fatal(err) + } +} + +func (b *bed) setHealth(t *testing.T, node, module string) { + t.Helper() + data, err := delivery.EncodeHave(delivery.Have{Phase: delivery.PhaseRunning, Health: delivery.HealthOK}) + if err != nil { + t.Fatal(err) + } + key, _ := delivery.HaveKey(node, module) + if _, err := b.buckets[delivery.HaveBucket].Put(b.ctx, key, data); err != nil { + t.Fatal(err) + } +} + +func (b *bed) report(t *testing.T, module, node, language, version, recipe string) { + t.Helper() + data, err := delivery.EncodeToolchain(delivery.Toolchain{Language: language, Version: version, Recipe: recipe, At: time.Now().UTC()}) + if err != nil { + t.Fatal(err) + } + key, _ := delivery.ToolchainKey(node, language) + if _, err := b.tools[module].Put(b.ctx, key, data); err != nil { + t.Fatal(err) + } + if _, err := b.js.Publish(b.ctx, "mesh.stream."+module+".event.toolchain.reported", []byte(`{}`)); err != nil { + t.Fatal(err) + } +} + +// candidate writes candidates. as the indexer does (the SDK's EncodeCandidate is the indexer's own encoder). +func (b *bed) candidate(t *testing.T, group string, pulls ...delivery.CandidatePull) string { + t.Helper() + heads := make([]delivery.Head, len(pulls)) + for i, p := range pulls { + heads[i] = p.Head + } + hash, err := delivery.HeadsHash(heads) + if err != nil { + t.Fatal(err) + } + data, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: group, HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)}) + if err != nil { + t.Fatal(err) + } + key, _ := delivery.CandidatesKey(group) + if _, err := b.buckets[delivery.CandidatesBucket].Put(b.ctx, key, data); err != nil { + t.Fatal(err) + } + vk, _ := delivery.CheckerVerdictKey(group, hash) + return vk +} + +func pull(repository string, number int, head string) delivery.CandidatePull { + return delivery.CandidatePull{Head: delivery.Head{Repository: repository, Number: number, Head: head}, Modules: []string{}} +} + +// start runs the program as the checker of a node. +func (b *bed) start(t *testing.T, env map[string]string) { + t.Helper() + broker := filepath.Join(t.TempDir(), "broker") + cred, _ := json.Marshal(credential{URL: b.srvURL, Node: "n1", Module: "mesh-delivery-checker", User: "n1.mesh-delivery-checker", Password: "pw"}) + if err := os.WriteFile(broker, cred, 0o600); err != nil { + t.Fatal(err) + } + e := map[string]string{"MESH_BROKER_FILE": broker} + for k, v := range env { + e[k] = v + } + runCtx, stop := context.WithCancel(b.ctx) + done := make(chan error, 1) + go func() { done <- run(runCtx, func(k string) string { return e[k] }, io.Discard) }() + t.Cleanup(func() { stop(); <-done }) +} + +// verdict reads the verdict key until pred holds of it, within 20 s. +func (b *bed) verdict(t *testing.T, key string, pred func(delivery.Verdict) bool) delivery.Verdict { + t.Helper() + var last delivery.Verdict + for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + e, err := b.buckets[delivery.CheckerVerdictsBucket].Get(b.ctx, key) + if err != nil { + continue + } + if last, err = delivery.DecodeVerdict(e.Value()); err == nil && pred(last) { + return last + } + } + t.Fatalf("verdicts.%s = %+v within 20 s", key, last) + return last +} + +func ended(v delivery.Verdict) bool { return v.State != delivery.VerdictRunning } + +// owed reads the status the checker owes on a head. +func (b *bed) owed(t *testing.T, number int, head, state string) delivery.Owed { + t.Helper() + key, err := delivery.CheckerOwedKey(repo, number, head) + if err != nil { + t.Fatal(err) + } + var last delivery.Owed + for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + e, err := b.buckets[delivery.CheckerOwedBucket].Get(b.ctx, key) + if err != nil { + continue + } + if last, err = delivery.DecodeCheckerOwed(e.Value()); err == nil && last.State == state { + return last + } + } + t.Fatalf("owed.%s = %+v within 20 s, want the state %s", key, last, state) + return last +} + +// TestCheckerJudgesACandidateAndOwesItsStatus: a candidate the indexer wrote, one pull request of mesh-delivery, is fetched +// merged onto the trunk the gate names, digested and tested at both levels on the node, kept, previewed; the verdict passes +// with the counts of both levels and the composition not run, and the status mesh/check is owed on the head as success. +func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300/AC5, 1445/AC1 + b := startBed(t) + key := b.candidate(t, "feat/x", pull(repo, 7, head1)) + b.start(t, nil) + v := b.verdict(t, key, ended) + if v.State != delivery.VerdictPass || v.Levels[delivery.LevelUnit].Run != 2 || v.Levels[delivery.LevelIntegration].Run != 2 || v.Trunk[repo] != trunk { + t.Fatalf("verdict = %+v", v) + } + if v.Composition == nil || v.Composition.State != delivery.CompositionNotRun || len(v.Machines) != 1 || v.Machines[0] != "n1" { + t.Errorf("composition %+v machines %v", v.Composition, v.Machines) + } + o := b.owed(t, 7, head1, "success") + if o.Context != delivery.StatusContextCheck || !strings.HasPrefix(o.Text, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(o.Text, "unit 2 run, 0 kept") { + t.Errorf("owed = %+v", o) + } + for tool, want := range map[string]int{"mesh-build-go.build_go_fetch": 1, "mesh-build-go.build_go_digest": 2, "mesh-build-go.build_go_test": 2} { + if got := b.fake.count(tool); got != want { + t.Errorf("%s called %d times, want %d", tool, got, want) + } + } + // What passed is kept once under each level's digest. + for _, k := range []string{"unit." + digU, "integration." + digI} { + if _, err := b.buckets[delivery.CheckerKeptBucket].Get(b.ctx, k); err != nil { + t.Errorf("kept.%s: %v", k, err) + } + } + // The preview of the head merged onto the trunk, with the tree and the manifest digest the fetch named. + pk, _ := delivery.PreviewKey(repo, head1, trunk) + e, err := b.buckets[delivery.PreviewsBucket].Get(b.ctx, pk) + if err != nil { + t.Fatalf("previews.%s: %v", pk, err) + } + pv, err := delivery.DecodePreview(e.Value()) + if err != nil || pv.Merge != merged || pv.Modules["modules/m"].Tree != "1111111111111111111111111111111111111111" { + t.Errorf("preview = %+v, %v", pv, err) + } +} + +// TestACheckOfEqualInputsRunsNothing: a second candidate (another head, the same digests) is checked with every package +// taken from the first's kept verdicts: no test step runs and no second kept key appears. +func TestACheckOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6 + b := startBed(t) + first := b.candidate(t, "feat/x", pull(repo, 7, head1)) + b.start(t, nil) + b.verdict(t, first, ended) + tests, kept := b.fake.count("mesh-build-go.build_go_test"), keptCount(t, b) + second := b.candidate(t, "feat/y", pull(repo, 8, head2)) + v := b.verdict(t, second, ended) + if v.State != delivery.VerdictPass || v.Levels[delivery.LevelUnit].Run != 0 || v.Levels[delivery.LevelUnit].Kept != 2 || v.Levels[delivery.LevelIntegration].Run != 0 { + t.Fatalf("verdict = %+v, want everything taken from the kept verdicts", v) + } + if got := b.fake.count("mesh-build-go.build_go_test"); got != tests { + t.Errorf("build_go_test called %d times, was %d: a test ran for equal inputs", got, tests) + } + if got := keptCount(t, b); got != kept { + t.Errorf("%d kept keys, was %d", got, kept) + } +} + +func keptCount(t *testing.T, b *bed) int { + t.Helper() + keys, err := b.buckets[delivery.CheckerKeptBucket].Keys(b.ctx) + if err != nil { + return 0 + } + return len(keys) +} + +// TestAHeadThatDoesNotMergeFailsNamingTheFilesAndRunsNoTest: the fetch answers the files in conflict; the verdict fails naming +// them, no test or digest step is run, no preview is written and the status is owed as failure. +func TestAHeadThatDoesNotMergeFailsNamingTheFilesAndRunsNoTest(t *testing.T) { // 1289/AC4 + b := startBed(t) + b.fake.conflict.Store(true) + key := b.candidate(t, "feat/x", pull(repo, 7, head1)) + b.start(t, nil) + v := b.verdict(t, key, ended) + if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "files in conflict: a.go, sub/b.go") { + t.Fatalf("verdict = %+v", v) + } + if o := b.owed(t, 7, head1, "failure"); !strings.HasPrefix(o.Text, "failed: n1: ") || !strings.Contains(o.Text, "files in conflict: a.go, sub/b.go") { + t.Errorf("owed = %+v", o) + } + if n := b.fake.count("mesh-build-go.build_go_digest") + b.fake.count("mesh-build-go.build_go_test"); n != 0 { + t.Errorf("%d digest and test steps ran after a conflict", n) + } + if keys, _ := b.buckets[delivery.PreviewsBucket].Keys(b.ctx); len(keys) != 0 { + t.Errorf("previews %v for a head that does not merge", keys) + } +} + +// TestAnHqPullRequestGetsNoVerdictAndAnotherDoes: hq's pull request is judged by no one (ADR 0484 clause 8), the other one in +// the same run is. +func TestAnHqPullRequestGetsNoVerdictAndAnotherDoes(t *testing.T) { // ADR 0484 clause 8 + b := startBed(t) + hq := b.candidate(t, "feat/hq", pull("novox/hq", 3, head1)) + other := b.candidate(t, "feat/x", pull(repo, 7, head2)) + b.start(t, nil) + b.verdict(t, other, ended) + if _, err := b.buckets[delivery.CheckerVerdictsBucket].Get(b.ctx, hq); err == nil { + t.Error("hq's pull request has a verdict") + } +} + +// TestACheckLeftRunningIsTakenOverWhenTheCheckerStarts: the verdict key holds running, left by a checker that stopped; the +// started checker judges it again and ends it (ADR 0478 clause 6, as the builder's). +func TestACheckLeftRunningIsTakenOverWhenTheCheckerStarts(t *testing.T) { // N6/AC2 + b := startBed(t) + key := b.candidate(t, "feat/x", pull(repo, 7, head1)) + group, heads, _ := delivery.ParseCheckerVerdictKey(key) + left, err := delivery.EncodeVerdict(delivery.Verdict{Group: group, Heads: heads, Trunk: map[string]string{repo: trunk}, State: delivery.VerdictRunning}) + if err != nil { + t.Fatal(err) + } + if _, err := b.buckets[delivery.CheckerVerdictsBucket].Create(b.ctx, key, left); err != nil { + t.Fatal(err) + } + b.start(t, nil) + if v := b.verdict(t, key, ended); v.State != delivery.VerdictPass { + t.Errorf("verdict = %+v", v) + } +} diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go index ba1109d..39e1ac8 100644 --- a/modules/mesh-delivery-checker/internal/check/examiner_test.go +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -116,8 +116,8 @@ type bench struct { // the captured slow fixture's result at both levels, on shanks. func newBench(t testing.TB) *bench { b := &bench{steps: &fakeSteps{}, kept: newMemKV(), prev: newMemKV()} - b.fetch = Fetched{Workspace: "ws1", Commit: head1, Modules: []string{"."}, Base: trunkA, Merge: "9999999999999999999999999999999999999999", - Changed: []string{"README.md"}} + b.fetch = captured[Fetched](t, "go-fetch-clean") + b.fetch.Modules, b.fetch.Changed, b.fetch.Trees = []string{"."}, []string{"README.md"}, nil one := captured[Digested](t, "go-digest") b.digest = map[string]Digested{ delivery.LevelUnit: {Level: "unit", Packages: map[string]string{"fixture/slow": digA}}, @@ -268,9 +268,9 @@ func TestAReAskRunsOnlyWhatFailedAndWhatChanged(t *testing.T) { // 1288/AC2, AC3 func TestANodePackageIsNeverKeptAndRunsEveryCheck(t *testing.T) { // 1288/AC4, 1292/AC1 b := newBench(t) b.ex.Modules = langs{repoA: {buildrun.NodeBuildModule}} - b.fetch.Modules = []string{"modules/ui"} + b.fetch = captured[Fetched](t, "node-fetch-clean") b.test = func(c stepCall) (any, error) { - return Tested{Level: c.args["level"].(string), Module: "modules/ui", Passed: true, Elapsed: 0.4, Bound: 5}, nil + return Tested{Level: c.args["level"].(string), Module: "modules/m", Passed: true, Elapsed: 0.4, Bound: 5}, nil } cs := caseOf(t, pullOf(repoA, 7, head1)) for i := 0; i < 2; i++ { @@ -338,10 +338,10 @@ func TestTheChangesOwnPassingTestRunsFirstAndTheRestFollow(t *testing.T) { // 12 func TestAHeadThatDoesNotMergeEndsFailedNamingTheFilesAndRunsNoTest(t *testing.T) { // 1289/AC4 b := newBench(t) - b.fetch = Fetched{Workspace: "ws1", Commit: head1, Base: trunkA, Conflicts: []string{"go.mod", "internal/x.go"}} + b.fetch = captured[Fetched](t, "go-fetch-conflict") out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) f := out.Verdict.Failed - if out.Verdict.State != delivery.VerdictFail || f.Machine != "shanks" || !strings.Contains(f.Words, "files in conflict: go.mod, internal/x.go") { + if out.Verdict.State != delivery.VerdictFail || f.Machine != "shanks" || !strings.Contains(f.Words, "files in conflict: a.go, sub/b.go") { t.Fatalf("verdict = %+v", out.Verdict) } if n := len(b.steps.tool("digest")) + len(b.steps.tool("test")); n != 0 { @@ -483,7 +483,7 @@ func TestACleanMergeWritesItsPreviewWithTheTreesManifestsAndLocks(t *testing.T) func TestAHeadThatDoesNotMergeWritesNoPreview(t *testing.T) { // 1445/AC5 b := newBench(t) - b.fetch = Fetched{Workspace: "ws1", Commit: head1, Base: trunkA, Conflicts: []string{"go.mod"}} + b.fetch = captured[Fetched](t, "go-fetch-conflict") b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) if len(b.prev.vals) != 0 { t.Errorf("previews %v written for a head that does not merge", b.prev.vals) diff --git a/modules/mesh-delivery-checker/internal/check/testdata/captured/README b/modules/mesh-delivery-checker/internal/check/testdata/captured/README index e39f215..b8023fe 100644 --- a/modules/mesh-delivery-checker/internal/check/testdata/captured/README +++ b/modules/mesh-delivery-checker/internal/check/testdata/captured/README @@ -4,6 +4,10 @@ test beside the module's test_test.go and digest_test.go (workspaceFor + runUnit slow and empty, and digestFixture + digests on testdata/repos/digest) at mesh-catalog origin/main cdbdb474, captured on 2026-10-21 for issue 1679 and not kept there. The fixtures' package names are the build module's own (fixture/failing/bad...). -The fields a fetch with a base adds (base, merge, conflicts, changed, trees) are the contract the lead relayed to the -mesh-catalog agent (ADR 0484 clause 3, 2026-10-21); mesh-catalog's fetch step is not on main yet, so the fetch fixtures in -the tests are written from that contract and are to be replaced by a capture once its fetch_test can marshal one. +go-fetch-clean.json and go-fetch-conflict.json are what mesh-build-go's fetch step ends with (FetchResult, cmd/mesh-build-go +fetch.go and preview.go) for a fetch with a base, the Result bytes of the step in mesh-catalog's own tests +TestFetchWithABaseMergesAndAnswersThePreview and TestFetchWithAConflictEndsPassedAndNamesTheFiles (preview_test.go), copied +into a throwaway test that wrote them out, at mesh-catalog feat/r4c5-checker d5804927 (the git commands are the test's fake, +so the shas are its constants; the shape and the field names are the module's own marshalling). node-fetch-clean.json is the +FetchResult json.Marshal of mesh-build-node's fetch (fetch.go, preview.go) in its test TestFetchWithABaseMergesAndAnswersThePreview +(cmd/mesh-build-node/preview_test.go) at the same commit. Captured on 2026-10-21 for issue 1679. diff --git a/modules/mesh-delivery-checker/internal/check/testdata/captured/go-fetch-clean.json b/modules/mesh-delivery-checker/internal/check/testdata/captured/go-fetch-clean.json new file mode 100644 index 0000000..ce2e947 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/testdata/captured/go-fetch-clean.json @@ -0,0 +1 @@ +{"workspace":"mvii1iwu5414e4","commit":"0123456789abcdef0123456789abcdef01234567","modules":["."],"base":"abcdef0123456789abcdef0123456789abcdef01","merge":"fedcba9876543210fedcba9876543210fedcba98","changed":["a.go","modules/m/main.go"],"trees":{"modules/m":{"tree":"1111111111111111111111111111111111111111","manifest":"ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356","locks":{"go.sum":"c5fc83c01e92404452b986527d239140ccf9a48b88e0c268fbf38c2e1429e9c9"}}}} diff --git a/modules/mesh-delivery-checker/internal/check/testdata/captured/go-fetch-conflict.json b/modules/mesh-delivery-checker/internal/check/testdata/captured/go-fetch-conflict.json new file mode 100644 index 0000000..e1d8a99 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/testdata/captured/go-fetch-conflict.json @@ -0,0 +1 @@ +{"workspace":"mvii1iwz5050eb","commit":"0123456789abcdef0123456789abcdef01234567","modules":["."],"base":"abcdef0123456789abcdef0123456789abcdef01","conflicts":["a.go","sub/b.go"]} diff --git a/modules/mesh-delivery-checker/internal/check/testdata/captured/node-fetch-clean.json b/modules/mesh-delivery-checker/internal/check/testdata/captured/node-fetch-clean.json new file mode 100644 index 0000000..4c8eedb --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/testdata/captured/node-fetch-clean.json @@ -0,0 +1 @@ +{"workspace":"mvii1sxt979f67","commit":"0123456789abcdef0123456789abcdef01234567","modules":["modules/m"],"installed":[],"refused":[],"base":"abcdef0123456789abcdef0123456789abcdef01","merge":"0123456789abcdef0123456789abcdef01234567","changed":["modules/m/package.json"],"trees":{"modules/m":{"tree":"0123456789abcdef0123456789abcdef01234567","manifest":"0581b699d9d05517e69b21f5cedbeb2eee8aba2b205280b561be731594d7feef"}}} From e554094b74992cb7f1d76cea8ec1dcada10b5b67 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 21:25:00 +0200 Subject: [PATCH 08/19] The builder uses mesh-sdk's buildrun package for the caller, the steps, the places, the presence, toolchain and have sets and the choice of nodes, so the checker shares them (issue 1671) --- .../cmd/mesh-delivery-builder/main.go | 13 +- .../cmd/mesh-delivery-builder/settings.go | 7 +- .../mesh-delivery-builder/settings_test.go | 5 +- .../cmd/mesh-delivery-builder/toolchains.go | 17 +- .../internal/build/choose.go | 88 --------- .../internal/build/choose_test.go | 95 --------- .../internal/build/claim.go | 5 +- .../internal/build/fakes_test.go | 94 +++++++++ .../internal/build/health.go | 56 ------ .../internal/build/places.go | 171 ---------------- .../internal/build/places_test.go | 186 ------------------ .../internal/build/plan.go | 13 +- .../internal/build/plan_test.go | 9 +- .../internal/build/presence.go | 53 ----- .../internal/build/presence_test.go | 43 ---- .../internal/build/round4_test.go | 45 ++--- .../internal/build/run.go | 27 +-- .../internal/build/run_test.go | 23 +-- .../internal/build/steps.go | 67 ------- .../internal/build/steps_test.go | 116 ----------- .../internal/build/toolchains.go | 64 ------ .../internal/build/toolchains_test.go | 51 ----- 22 files changed, 175 insertions(+), 1073 deletions(-) delete mode 100644 modules/mesh-delivery-builder/internal/build/choose.go delete mode 100644 modules/mesh-delivery-builder/internal/build/choose_test.go create mode 100644 modules/mesh-delivery-builder/internal/build/fakes_test.go delete mode 100644 modules/mesh-delivery-builder/internal/build/health.go delete mode 100644 modules/mesh-delivery-builder/internal/build/places.go delete mode 100644 modules/mesh-delivery-builder/internal/build/places_test.go delete mode 100644 modules/mesh-delivery-builder/internal/build/presence.go delete mode 100644 modules/mesh-delivery-builder/internal/build/presence_test.go delete mode 100644 modules/mesh-delivery-builder/internal/build/steps.go delete mode 100644 modules/mesh-delivery-builder/internal/build/steps_test.go delete mode 100644 modules/mesh-delivery-builder/internal/build/toolchains.go delete mode 100644 modules/mesh-delivery-builder/internal/build/toolchains_test.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 9db5436..b5d2d34 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -13,6 +13,7 @@ import ( "encoding/json" "errors" "fmt" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "io" "log" "os" @@ -162,16 +163,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return fmt.Errorf("the build capacity setting: %w", err) } - tools := &build.ToolchainSet{} - presence := &build.PresenceSet{} - have := &build.HaveSet{} + tools := &buildrun.ToolchainSet{} + presence := &buildrun.PresenceSet{} + have := &buildrun.HaveSet{} b := &build.Builder{ Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module}, - Runner: build.Runner{Caller: caller, Steps: stepsWaiter{js}, Bound: bound}, + Runner: buildrun.Runner{Caller: caller, Steps: stepsWaiter{js}, Bound: bound}, Toolchains: tools, Presence: presence, Health: have, - Places: &build.Places{Capacity: capacity}, + Places: &buildrun.Places{Capacity: capacity}, Manifests: manifestReader{manifests}, Nodes: nodesPerFingerprint, Logf: logger.Printf, @@ -184,7 +185,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // A build module says each time it wrote its toolchains key: its stream, and the builder's durable consumer on it, are // made by the controller. Where the stream is not there yet the builder says so and is started again with the build // module. - for _, module := range []string{build.GoBuildModule, build.NodeBuildModule} { + for _, module := range []string{buildrun.GoBuildModule, buildrun.NodeBuildModule} { stream := toolchainStreams[module] cons, err := js.Consumer(ctx, stream, cred.Node+"_"+cred.Module) if err != nil { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/settings.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/settings.go index 36ead03..e0255f1 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/settings.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/settings.go @@ -3,9 +3,8 @@ package main import ( "encoding/json" "fmt" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "os" - - "mesh-delivery-builder/internal/build" ) // capacityFrom is the build capacity of each node (ADR 0478 clause 3): the module's setting `capacity`, `= 0 && (qualifies == nil || qualifies(e.Node)) { // a capacity of 0 keeps the builder off a node - nodes = append(nodes, e.Node) - } - } - if len(nodes) == 0 { - sort.Strings(reporting) - return nil, fmt.Errorf("no %s build node qualifies: %s report the toolchain, and none is online with a healthy build module and a build capacity", language, strings.Join(reporting, ", ")) - } - return places.choose(nodes, want), nil -} - -// DigestsDifferPrefix opens the error of a build whose nodes built an artifact to different digests (ADR 0478 clause 4): -// such a key is not built again as a new revision, since the same source on the same toolchains gave two answers and -// building again settles nothing; it waits for another fingerprint. -const DigestsDifferPrefix = "digests differ:" - -// Agree is the one digest every node built an artifact to (ADR 0478 clause 4), or an error naming each node and its -// digest when they differ. One node's digest agrees with itself. -func Agree(artifact string, byNode map[string]string) (string, error) { - nodes := make([]string, 0, len(byNode)) - for n := range byNode { - nodes = append(nodes, n) - } - sort.Strings(nodes) - var first string - for i, n := range nodes { - if i == 0 { - first = byNode[n] - continue - } - if byNode[n] != first { - msg := fmt.Sprintf("%s the nodes built %s to different digests:", DigestsDifferPrefix, artifact) - for _, n := range nodes { - msg += fmt.Sprintf(" %s %s;", n, byNode[n]) - } - return "", fmt.Errorf("%s", msg[:len(msg)-1]) - } - } - return first, nil -} diff --git a/modules/mesh-delivery-builder/internal/build/choose_test.go b/modules/mesh-delivery-builder/internal/build/choose_test.go deleted file mode 100644 index 88a6b53..0000000 --- a/modules/mesh-delivery-builder/internal/build/choose_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package build - -import ( - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/delivery" -) - -func tcs(language, version string, nodes ...string) []delivery.NodeToolchain { - var out []delivery.NodeToolchain - for _, n := range nodes { - out = append(out, delivery.NodeToolchain{Node: n, Toolchain: delivery.Toolchain{Language: language, Version: version, Recipe: "1"}}) - } - return out -} - -func TestChooseTakesTheAgreedNodesWithFewestRunningStepsTiesByName(t *testing.T) { // 1607/AC4, ADR 0478 clause 3 - entries := tcs("go", "go1.27.1", "novox", "ace", "g14", "shanks") - nodes, tc, err := ChooseNodes("go", entries, placesRunning(map[string]int{"novox": 3, "ace": 1, "g14": 1, "shanks": 0}), nil, 2) - if err != nil { - t.Fatal(err) - } - if len(nodes) != 2 || nodes[0] != "shanks" || nodes[1] != "ace" { - t.Errorf("nodes = %v, want [shanks ace]: fewest steps first, a tie by name", nodes) - } - if tc.Version != "go1.27.1" || tc.Recipe != "1" || tc.Language != "go" { - t.Errorf("toolchain = %+v", tc) - } -} - -func TestChooseWithOneNodeAnswersOneNode(t *testing.T) { // 1607/AC4 - nodes, _, err := ChooseNodes("go", tcs("go", "go1.27.1", "novox"), nil, nil, 2) - if err != nil || len(nodes) != 1 || nodes[0] != "novox" { - t.Errorf("nodes = %v, err %v, want [novox]", nodes, err) - } -} - -func TestChooseFailsNamingTheLanguageAndTheVersionsWhenNoNodeQualifies(t *testing.T) { // 1607/AC4 - _, _, err := ChooseNodes("typescript", nil, nil, nil, 2) - if err == nil || !strings.Contains(err.Error(), "typescript") || !strings.Contains(err.Error(), delivery.ToolchainNoReport) { - t.Errorf("no report: error = %v, want it to name typescript and %q", err, delivery.ToolchainNoReport) - } - entries := append(tcs("go", "go1.27.1", "novox"), tcs("go", "go1.26.0", "ace")...) - _, _, err = ChooseNodes("go", entries, nil, nil, 2) - if err == nil || !strings.Contains(err.Error(), "go1.27.1") || !strings.Contains(err.Error(), "go1.26.0") || !strings.Contains(err.Error(), "go") { - t.Errorf("disagreeing nodes: error = %v, want both versions named", err) - } -} - -func TestAgreeNeedsEqualDigestsAndNamesBothNodesWhenTheyDiffer(t *testing.T) { // 1607/AC4, ADR 0478 clause 4 - d, err := Agree("tool", map[string]string{"ace": digest, "novox": digest}) - if err != nil || d != digest { - t.Errorf("Agree = %q, %v, want %q", d, err, digest) - } - _, err = Agree("tool", map[string]string{"ace": digest, "novox": "sha256:" + hex64('9')}) - if err == nil || !strings.Contains(err.Error(), "ace") || !strings.Contains(err.Error(), "novox") || !strings.Contains(err.Error(), digest) || !strings.Contains(err.Error(), hex64('9')) || !strings.Contains(err.Error(), "tool") { - t.Errorf("differing digests: error = %v, want the artifact, both nodes and both digests", err) - } - if d, err := Agree("tool", map[string]string{"ace": digest}); err != nil || d != digest { - t.Errorf("one node: %q, %v", d, err) - } -} - -func TestChooseSkipsNodesThatAreNotOnlineAndFailsNamingThemWhenNoneIs(t *testing.T) { // 1607 review 3, ADR 0478 clause 3 - entries := tcs("go", "go1.27.1", "ace", "g14", "novox") - online := func(n string) bool { return n != "g14" } // a sleeping laptop - nodes, tc, err := ChooseNodes("go", entries, placesRunning(map[string]int{"novox": 5}), online, 2) - if err != nil || len(nodes) != 2 || nodes[0] != "ace" || nodes[1] != "novox" || tc.Version != "go1.27.1" { - t.Errorf("nodes = %v, toolchain %+v, err %v, want [ace novox] with g14 left out although it runs nothing", nodes, tc, err) - } - _, tc, err = ChooseNodes("go", entries, nil, func(string) bool { return false }, 2) - if err == nil || !strings.Contains(err.Error(), "online") || !strings.Contains(err.Error(), "g14") || !strings.Contains(err.Error(), "go") { - t.Errorf("none online: error = %v, want it to name the nodes that report and that none is online", err) - } - if tc.Version != "go1.27.1" { - t.Errorf("none online: the agreed toolchain %+v is still answered", tc) - } -} - -// placesRunning is Places with these steps counted as running, for a test of the choice. -func placesRunning(running map[string]int) *Places { - return &Places{running: running, changed: make(chan struct{})} -} - -func TestChoosePrefersNodesBelowTheirCapacity(t *testing.T) { // round 3 - entries := tcs("go", "go1.27.1", "ace", "novox", "shanks") - // shanks has capacity 3 with 2 running: room; ace has the default 1 with 1 running: none; novox has none running. - p := placesRunning(map[string]int{"shanks": 2, "ace": 1}) - p.Capacity = map[string]int{"shanks": 3} - nodes, _, err := ChooseNodes("go", entries, p, nil, 2) - if err != nil || len(nodes) != 2 || nodes[0] != "novox" || nodes[1] != "shanks" { - t.Errorf("nodes = %v, err %v, want [novox shanks]: the nodes with room, ace at capacity left out", nodes, err) - } -} diff --git a/modules/mesh-delivery-builder/internal/build/claim.go b/modules/mesh-delivery-builder/internal/build/claim.go index 6d16939..9ab0285 100644 --- a/modules/mesh-delivery-builder/internal/build/claim.go +++ b/modules/mesh-delivery-builder/internal/build/claim.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "strings" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -70,7 +71,7 @@ type ClaimOptions struct { // when a second node qualifies now (ClaimOptions.MoreNodes): that is taken as it stands, with the done value, and its // builder writes the new revision when the second build ends (ADR 0478 clause 4); // - a failed key is taken again as a new revision of the same key (ADR 0429 clause 18), except one failed for a kind no -// builder builds yet (RefusedPrefix, ADR 0478 clause 2) or for digests that differ (DigestsDifferPrefix, clause 4), +// builder builds yet (RefusedPrefix, ADR 0478 clause 2) or for digests that differ (buildrun.DigestsDifferPrefix, clause 4), // which wait for another fingerprint; // - a key left queued or building by this builder is taken as it stands (ADR 0478 clause 5: a stopped builder takes its // own again when it starts), and so is one left by another builder when ClaimOptions.TakeOver (clause 6). @@ -97,7 +98,7 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string return &Claim{Key: key, Rev: rev, Build: cur}, nil } return nil, nil - case cur.State == delivery.BuildFailed && (strings.HasPrefix(cur.Error, RefusedPrefix) || strings.HasPrefix(cur.Error, DigestsDifferPrefix)): + case cur.State == delivery.BuildFailed && (strings.HasPrefix(cur.Error, RefusedPrefix) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix)): return nil, nil case cur.State == delivery.BuildFailed: return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) }) diff --git a/modules/mesh-delivery-builder/internal/build/fakes_test.go b/modules/mesh-delivery-builder/internal/build/fakes_test.go new file mode 100644 index 0000000..2b993db --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/fakes_test.go @@ -0,0 +1,94 @@ +package build + +// The fakes of the build modules the builder tests run against: a caller answering a long tool at once with the step and its key, +// and the steps watch. The caller, the steps watch and the runner themselves are the SDK's (mesh-sdk go/delivery/buildrun, issue 1671). + +import ( + "context" + "encoding/json" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +type call struct { + node, key string + body any +} + +// fakeCaller answers a long tool as the build modules do: at once, with the step and its key. +type fakeCaller struct { + calls []call + answer string + err error +} + +func (f *fakeCaller) CallOn(node, key string, body any) (json.RawMessage, error) { + f.calls = append(f.calls, call{node, key, body}) + if f.err != nil { + return nil, f.err + } + return json.RawMessage(f.answer), nil +} + +// fakeSteps holds the steps values by `/`; Wait on a key it does not hold blocks until the context ends. +type fakeSteps struct { + mu sync.Mutex + steps map[string]delivery.Step + // during, when set, is called as a wait starts and the function it answers as the wait ends. + during func(key string) func() +} + +func (f *fakeSteps) Wait(ctx context.Context, module, key string) (delivery.Step, error) { + f.mu.Lock() + s, ok := f.steps[module+"/"+key] + during := f.during + f.mu.Unlock() + if ok { + if during != nil { + defer during(key)() + } + return s, nil + } + <-ctx.Done() + return delivery.Step{}, context.Cause(ctx) +} + +func stepAt(node, id, state, reason string, result string) delivery.Step { + s := delivery.Step{ID: id, Tool: "build_go_fetch", Node: node, State: state, Reason: reason} + if result != "" { + s.Result = json.RawMessage(result) + } + return s +} + +// presenceValue is a watchman presence value, as its writer encodes it (mesh-sdk go/delivery EncodePresence). +func presenceValue(t *testing.T, online bool) []byte { + t.Helper() + b, err := delivery.EncodePresence(delivery.Presence{Online: online}) + if err != nil { + t.Fatal(err) + } + return b +} + +// tcValue is a toolchains value, as the build module's writer encodes it (mesh-sdk go/delivery EncodeToolchain). +func tcValue(t *testing.T, language, version string) []byte { + t.Helper() + b, err := delivery.EncodeToolchain(delivery.Toolchain{Language: language, Version: version, Recipe: "1", At: time.Now()}) + if err != nil { + t.Fatal(err) + } + return b +} + +// tcs are the toolchain entries of a language on nodes, as the toolchains set gives them. +func tcs(language, version string, nodes ...string) []delivery.NodeToolchain { + var out []delivery.NodeToolchain + for _, n := range nodes { + out = append(out, delivery.NodeToolchain{Node: n, Toolchain: delivery.Toolchain{Language: language, Version: version, Recipe: "1"}}) + } + return out +} diff --git a/modules/mesh-delivery-builder/internal/build/health.go b/modules/mesh-delivery-builder/internal/build/health.go deleted file mode 100644 index d75e40b..0000000 --- a/modules/mesh-delivery-builder/internal/build/health.go +++ /dev/null @@ -1,56 +0,0 @@ -package build - -import ( - "sync" - - "git.novox.be/novox/mesh-sdk/go/delivery" -) - -// HaveSet holds what each node's node-engine says of each module on it (node-engine_have, key `.`): the -// builder reads it to know whether a node's build module is running and healthy, since a build module whose installed -// toolchain is not the one its recipe builds with is unhealthy (mesh-build-go build_go_health) and its steps would -// refuse (ADR 0478 clause 3). A module the node-engine has written no key of is not healthy. Safe for use from the watch -// and from a build at once. -type HaveSet struct { - mu sync.Mutex - healthy map[string]bool // by `.` -} - -// Put takes a change of the have bucket in and answers whether a module's healthy state changed. A node's own key -// (`.node`) and a value that is not a have are left out. -func (s *HaveSet) Put(key string, value []byte, deleted bool) bool { - node, module, isNode, err := delivery.ParseHaveKey(key) - if err != nil || isNode { - return false - } - ok := false - if !deleted { - h, err := delivery.DecodeHave(value) - if err != nil { - return false - } - ok = h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK - } - k := node + "." + module - s.mu.Lock() - defer s.mu.Unlock() - if s.healthy[k] == ok { - return false - } - if s.healthy == nil { - s.healthy = map[string]bool{} - } - if ok { - s.healthy[k] = true - } else { - delete(s.healthy, k) - } - return true -} - -// Healthy is whether the module on the node was last seen running and healthy. -func (s *HaveSet) Healthy(node, module string) bool { - s.mu.Lock() - defer s.mu.Unlock() - return s.healthy[node+"."+module] -} diff --git a/modules/mesh-delivery-builder/internal/build/places.go b/modules/mesh-delivery-builder/internal/build/places.go deleted file mode 100644 index 65d41fb..0000000 --- a/modules/mesh-delivery-builder/internal/build/places.go +++ /dev/null @@ -1,171 +0,0 @@ -package build - -import ( - "context" - "fmt" - "sort" - "strconv" - "strings" - "sync" -) - -// DefaultCapacity is the build capacity of a node that has no setting: one build step at a time. -const DefaultCapacity = 1 - -// ParseCapacity reads the builder's capacity setting: `node=steps` pairs separated by commas, such as -// `shanks=3,ace=2`. A node without a pair has DefaultCapacity. A pair with no node, a count that is not a whole number of -// at least 0, and a node named twice are refused. A count of 0 keeps the builder off the node: it is never chosen. -func ParseCapacity(s string) (map[string]int, error) { - out := map[string]int{} - for _, pair := range strings.Split(s, ",") { - pair = strings.TrimSpace(pair) - if pair == "" { - continue - } - node, count, ok := strings.Cut(pair, "=") - node, count = strings.TrimSpace(node), strings.TrimSpace(count) - n, err := strconv.Atoi(count) - if !ok || node == "" || err != nil || n < 0 { - return nil, fmt.Errorf("the capacity %q is not =", pair) - } - if _, dup := out[node]; dup { - return nil, fmt.Errorf("the capacity names %s twice", node) - } - out[node] = n - } - return out, nil -} - -// Places counts the build steps the builder runs on each node and holds a lane back while its node is at its -// capacity (the operator's decision of 2026-10-21; ADR 0478 clause 3). A lane is the fetch and the builds of one build -// module on one node, run one step at a time, so a lane holds one place while it runs. A node is chosen for a lane by -// reserving a place on it at the moment of the choice (choose), so that builds choosing at once see each other's -// choices and none waits on a full node while another has room. A lane waits for its place by watching the ends of the -// other lanes, not by a timer: ending a lane wakes the waiting ones. -type Places struct { - // Capacity is the number of steps at once by node; a node not in it has DefaultCapacity. - Capacity map[string]int - - mu sync.Mutex - running map[string]int // lanes holding a place - reserved map[string]int // lanes chosen for the node that do not hold a place yet - changed chan struct{} // closed, and replaced, each time a lane ends -} - -// CapacityOf is how many steps the builder runs on the node at once: DefaultCapacity for a node the setting does not -// name, and 0 for one it names so, where the builder never builds. -func (p *Places) CapacityOf(node string) int { - if n, ok := p.Capacity[node]; ok && n >= 0 { - return n - } - return DefaultCapacity -} - -// Running is the steps in flight now, by node. -func (p *Places) Running() map[string]int { - p.mu.Lock() - defer p.mu.Unlock() - out := make(map[string]int, len(p.running)) - for n, c := range p.running { - if c > 0 { - out[n] = c - } - } - return out -} - -// Reserved is the lanes chosen for a node that hold no place yet, by node. -func (p *Places) Reserved() map[string]int { - p.mu.Lock() - defer p.mu.Unlock() - out := make(map[string]int, len(p.reserved)) - for n, c := range p.reserved { - if c > 0 { - out[n] = c - } - } - return out -} - -func (p *Places) init() { - if p.running == nil { - p.running = map[string]int{} - } - if p.reserved == nil { - p.reserved = map[string]int{} - } - if p.changed == nil { - p.changed = make(chan struct{}) - } -} - -// Acquire takes a place on the node, waiting while the node is at its capacity, and answers the function that gives it -// back. A place reserved for the node by a choice is the one taken. It returns an error when ctx ends first, and then -// gives up the reservation too. -func (p *Places) Acquire(ctx context.Context, node string) (release func(), err error) { - for { - p.mu.Lock() - p.init() - if p.running[node] < p.CapacityOf(node) { - p.running[node]++ - if p.reserved[node] > 0 { - p.reserved[node]-- - } - p.mu.Unlock() - var once sync.Once - return func() { - once.Do(func() { - p.mu.Lock() - p.running[node]-- - close(p.changed) - p.changed = make(chan struct{}) - p.mu.Unlock() - }) - }, nil - } - wait := p.changed - p.mu.Unlock() - select { - case <-wait: - case <-ctx.Done(): - p.Cancel(node) - return nil, fmt.Errorf("waiting for a place on %s: %w", node, ctx.Err()) - } - } -} - -// Cancel gives up a place reserved for the node that will not be taken. -func (p *Places) Cancel(node string) { - p.mu.Lock() - defer p.mu.Unlock() - p.init() - if p.reserved[node] > 0 { - p.reserved[node]-- - } -} - -// choose orders the nodes (those with room first, counting the lanes running and the lanes reserved, then the fewest of -// them, then by name), keeps the first want and reserves a place on each, all at once. -func (p *Places) choose(nodes []string, want int) []string { - p.mu.Lock() - defer p.mu.Unlock() - p.init() - load := func(n string) int { return p.running[n] + p.reserved[n] } - sort.SliceStable(nodes, func(i, j int) bool { - ri, rj := load(nodes[i]) < p.CapacityOf(nodes[i]), load(nodes[j]) < p.CapacityOf(nodes[j]) - if ri != rj { - return ri - } - if load(nodes[i]) != load(nodes[j]) { - return load(nodes[i]) < load(nodes[j]) - } - return nodes[i] < nodes[j] - }) - if len(nodes) > want { - nodes = nodes[:want] - } - for _, n := range nodes { - p.reserved[n]++ - } - return nodes -} diff --git a/modules/mesh-delivery-builder/internal/build/places_test.go b/modules/mesh-delivery-builder/internal/build/places_test.go deleted file mode 100644 index 27c7d8d..0000000 --- a/modules/mesh-delivery-builder/internal/build/places_test.go +++ /dev/null @@ -1,186 +0,0 @@ -package build - -import ( - "context" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -func TestParseCapacityReadsNodeEqualsCountPairs(t *testing.T) { - c, err := ParseCapacity(" shanks=3, ace=2 ") - if err != nil || c["shanks"] != 3 || c["ace"] != 2 || len(c) != 2 { - t.Errorf("capacity = %v, %v", c, err) - } - if c, err := ParseCapacity(""); err != nil || len(c) != 0 { - t.Errorf("empty: %v, %v", c, err) - } - for _, bad := range []string{"shanks", "shanks=x", "=2", "a=1,a=2", "shanks=-1"} { - if _, err := ParseCapacity(bad); err == nil { - t.Errorf("%q accepted", bad) - } - } -} - -func acquireAsync(p *Places, node string) chan func() { - got := make(chan func(), 1) - go func() { - release, err := p.Acquire(context.Background(), node) - if err != nil { - close(got) - return - } - got <- release - }() - return got -} - -func TestThreeStepsOnANodeOfCapacityTwoRunTwoAtOnceAndTheThirdStartsWhenOneEnds(t *testing.T) { // round 3 - p := &Places{Capacity: map[string]int{"ace": 2}} - a, b := acquireAsync(p, "ace"), acquireAsync(p, "ace") - ra, rb := <-a, <-b - third := acquireAsync(p, "ace") - select { - case <-third: - t.Fatal("a third step started on a node of capacity 2 with two running") - case <-time.After(50 * time.Millisecond): - } - if p.Running()["ace"] != 2 { - t.Errorf("running = %v, want 2 on ace", p.Running()) - } - ra() - select { - case r := <-third: - r() - case <-time.After(2 * time.Second): - t.Fatal("the third step did not start when one ended") - } - rb() -} - -func TestANodeWithNoSettingRunsOneStepAtATime(t *testing.T) { // round 3 - p := &Places{Capacity: map[string]int{"shanks": 3}} - first := <-acquireAsync(p, "g14") - second := acquireAsync(p, "g14") - select { - case <-second: - t.Fatal("two steps on a node without a setting") - case <-time.After(50 * time.Millisecond): - } - first() - r := <-second - r() - if p.CapacityOf("g14") != 1 || p.CapacityOf("shanks") != 3 { - t.Errorf("capacities = %d, %d", p.CapacityOf("g14"), p.CapacityOf("shanks")) - } -} - -func TestAWaitingStepStopsWhenItsContextEnds(t *testing.T) { // round 3 - p := &Places{} - r := <-acquireAsync(p, "ace") - defer r() - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan error, 1) - go func() { _, err := p.Acquire(ctx, "ace"); done <- err }() - cancel() - select { - case err := <-done: - if err == nil { - t.Error("Acquire returned no error for a cancelled wait") - } - case <-time.After(2 * time.Second): - t.Fatal("a cancelled wait did not return") - } -} - -func TestStepsOfSeveralBuildsNeverExceedANodesCapacity(t *testing.T) { // round 3, through the builder - r := newRig(t, catalogue.Artifact{Name: "tool", Kind: "bundle", Language: "go"}) - r.b.Places = &Places{Capacity: map[string]int{"ace": 2}} // novox has none: one at a time - var now, most atomic.Int32 - var mu sync.Mutex - perNode := map[string]int{} - r.bm.steps.during = func(key string) func() { - node, _, _ := strings.Cut(key, ".") - mu.Lock() - perNode[node]++ - if int32(perNode[node]) > most.Load() && node == "ace" { - most.Store(int32(perNode[node])) - } - if node == "novox" && perNode[node] > 1 { - t.Errorf("%d steps at once on novox, which has capacity 1", perNode[node]) - } - mu.Unlock() - now.Add(1) - time.Sleep(15 * time.Millisecond) - return func() { mu.Lock(); perNode[node]--; mu.Unlock(); now.Add(-1) } - } - for _, m := range []string{"m1", "m2", "m3", "m4"} { - r.b.Module(context.Background(), m, moduleValue(t, "modules/"+m)) - } - r.b.Wait() - if most.Load() > 2 { - t.Errorf("%d steps at once on ace, which has capacity 2", most.Load()) - } - if most.Load() < 2 { - t.Errorf("at most %d steps at once on ace: the capacity was not used", most.Load()) - } -} - -func TestTwoNodeAgreementStillTakesTwoDifferentNodesWhenCapacitiesAreSet(t *testing.T) { // round 3, ADR 0478 clause 4 - r := newRig(t, catalogue.Artifact{Name: "tool", Kind: "bundle", Language: "go"}) - r.b.Places = &Places{Capacity: map[string]int{"ace": 2, "novox": 3}} - v := r.consider(t, "modules/m") - if v.State != "done" || !strings.Contains(v.Step, "ace and novox") { - t.Errorf("build = %+v, want done on ace and novox", v) - } -} - -// The operator's decision of 2026-10-21 13:40: the builder builds on shanks and ace only; a capacity of 0 keeps it off a node. -func TestACapacityOfZeroKeepsTheBuilderOffTheNodeAndANodeNotNamedKeepsOne(t *testing.T) { // issue 1661 - c, err := ParseCapacity("shanks=3,ace=2,g14=0,novox=0") - if err != nil || c["g14"] != 0 || c["novox"] != 0 || c["shanks"] != 3 || len(c) != 4 { - t.Fatalf("capacity = %v, %v", c, err) - } - p := &Places{Capacity: c} - if p.CapacityOf("g14") != 0 || p.CapacityOf("novox") != 0 || p.CapacityOf("ace") != 2 || p.CapacityOf("other") != DefaultCapacity { - t.Errorf("capacities = %d %d %d %d, want 0 0 2 and the default for a node not named", p.CapacityOf("g14"), p.CapacityOf("novox"), p.CapacityOf("ace"), p.CapacityOf("other")) - } - if _, err := ParseCapacity("shanks=-1"); err == nil { - t.Error("a negative capacity accepted") - } - entries := tcs("go", "go1.27.1", "ace", "g14", "novox", "shanks", "other") - for i := 0; i < 6; i++ { - nodes, _, err := ChooseNodes("go", entries, p, nil, 2) - if err != nil { - t.Fatal(err) - } - for _, n := range nodes { - if n == "g14" || n == "novox" { - t.Fatalf("choice %d: %v includes a node of capacity 0", i, nodes) - } - } - } - // Only nodes of capacity 0 report: nothing qualifies, and the build says why. - _, _, err = ChooseNodes("go", tcs("go", "go1.27.1", "g14", "novox"), p, nil, 2) - if err == nil || !strings.Contains(err.Error(), "capacity") { - t.Errorf("only capacity-0 nodes: error = %v, want it to name the capacity", err) - } -} - -func TestABuildNeverRunsOnANodeOfCapacityZero(t *testing.T) { // issue 1661 - r := newRig(t, catalogue.Artifact{Name: "tool", Kind: "bundle", Language: "go"}) // ace and novox report - r.b.Places = &Places{Capacity: map[string]int{"novox": 0}} - v := r.consider(t, "modules/m") - if v.State != "done" || !strings.Contains(v.Step, "on one node, ace") { - t.Fatalf("build = %+v, want done on ace alone", v) - } - for _, c := range r.bm.calls { - if c.node == "novox" { - t.Errorf("a step was called on novox, whose capacity is 0: %+v", c) - } - } -} diff --git a/modules/mesh-delivery-builder/internal/build/plan.go b/modules/mesh-delivery-builder/internal/build/plan.go index d765825..edebb4e 100644 --- a/modules/mesh-delivery-builder/internal/build/plan.go +++ b/modules/mesh-delivery-builder/internal/build/plan.go @@ -5,17 +5,12 @@ package build import ( "fmt" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "sort" "git.novox.be/novox/mesh-sdk/go/catalogue" ) -// The build modules a task is given to, and the language whose toolchain each reports (the indexer's ToolchainModules). -const ( - GoBuildModule = "mesh-build-go" - NodeBuildModule = "mesh-build-node" -) - // RefusedPrefix opens the error of a build key that is failed for a kind no builder builds yet (ADR 0478 clause 2): such // a key is not built again as a new revision. const RefusedPrefix = "refused kind:" @@ -79,11 +74,11 @@ func PlanOf(m catalogue.Manifest) (Plan, error) { for _, a := range artifacts { switch { case a.Kind == catalogue.ArtifactArchive: - p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: GoBuildModule, Language: "go"}) + p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: buildrun.GoBuildModule, Language: "go"}) case a.Kind == catalogue.ArtifactBundle && a.Language == "go": - p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: GoBuildModule, Language: "go"}) + p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: buildrun.GoBuildModule, Language: "go"}) case a.Kind == catalogue.ArtifactBundle && a.Language == "typescript": - p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: NodeBuildModule, Language: "typescript"}) + p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: buildrun.NodeBuildModule, Language: "typescript"}) case a.Kind == catalogue.ArtifactBundle: return Plan{}, &Refused{Artifact: a.Name, What: fmt.Sprintf("bundle of language %q", a.Language)} default: diff --git a/modules/mesh-delivery-builder/internal/build/plan_test.go b/modules/mesh-delivery-builder/internal/build/plan_test.go index bbeb0c3..3760917 100644 --- a/modules/mesh-delivery-builder/internal/build/plan_test.go +++ b/modules/mesh-delivery-builder/internal/build/plan_test.go @@ -2,6 +2,7 @@ package build import ( "errors" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "strings" "testing" @@ -23,9 +24,9 @@ func TestPlanRoutesEachKindToItsBuildModule(t *testing.T) { // 1607/AC2 t.Fatal(err) } want := []Task{ - {Artifact: "files", Kind: "archive", Module: GoBuildModule, Language: "go"}, - {Artifact: "tool", Kind: "bundle", Module: GoBuildModule, Language: "go"}, - {Artifact: "web", Kind: "bundle", Module: NodeBuildModule, Language: "typescript"}, + {Artifact: "files", Kind: "archive", Module: buildrun.GoBuildModule, Language: "go"}, + {Artifact: "tool", Kind: "bundle", Module: buildrun.GoBuildModule, Language: "go"}, + {Artifact: "web", Kind: "bundle", Module: buildrun.NodeBuildModule, Language: "typescript"}, } if len(p.Tasks) != len(want) { t.Fatalf("tasks = %+v, want %+v", p.Tasks, want) @@ -35,7 +36,7 @@ func TestPlanRoutesEachKindToItsBuildModule(t *testing.T) { // 1607/AC2 t.Errorf("task %d = %+v, want %+v", i, p.Tasks[i], want[i]) } } - if modules := p.Modules(); len(modules) != 2 || modules[0] != GoBuildModule || modules[1] != NodeBuildModule { + if modules := p.Modules(); len(modules) != 2 || modules[0] != buildrun.GoBuildModule || modules[1] != buildrun.NodeBuildModule { t.Errorf("modules = %v", modules) } } diff --git a/modules/mesh-delivery-builder/internal/build/presence.go b/modules/mesh-delivery-builder/internal/build/presence.go deleted file mode 100644 index b1131aa..0000000 --- a/modules/mesh-delivery-builder/internal/build/presence.go +++ /dev/null @@ -1,53 +0,0 @@ -package build - -import ( - "sync" - - "git.novox.be/novox/mesh-sdk/go/delivery" -) - -// PresenceSet holds which nodes are online, as the watchman writes it (mesh-delivery-watchman_presence, key ``, the -// node-engine's connection): a node asleep or off the bus is offline and its build module cannot answer. A node the -// watchman has no key of is not online. Safe for use from the watch and from a build at once. -type PresenceSet struct { - mu sync.Mutex - online map[string]bool -} - -// Put takes a change of the presence bucket in and answers whether a node's online state changed. A part's key and a -// value that is not a presence are left out. -func (s *PresenceSet) Put(key string, value []byte, deleted bool) bool { - node, part, _, err := delivery.ParsePresenceKey(key) - if err != nil || part != "" { - return false - } - on := false - if !deleted { - p, err := delivery.DecodePresence(value) - if err != nil { - return false - } - on = p.Online - } - s.mu.Lock() - defer s.mu.Unlock() - if s.online[node] == on { - return false - } - if s.online == nil { - s.online = map[string]bool{} - } - if on { - s.online[node] = true - } else { - delete(s.online, node) - } - return true -} - -// Online is whether the watchman last saw the node online. -func (s *PresenceSet) Online(node string) bool { - s.mu.Lock() - defer s.mu.Unlock() - return s.online[node] -} diff --git a/modules/mesh-delivery-builder/internal/build/presence_test.go b/modules/mesh-delivery-builder/internal/build/presence_test.go deleted file mode 100644 index c1f3de7..0000000 --- a/modules/mesh-delivery-builder/internal/build/presence_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package build - -import ( - "testing" - - "git.novox.be/novox/mesh-sdk/go/delivery" -) - -func presenceValue(t *testing.T, online bool) []byte { - t.Helper() - b, err := delivery.EncodePresence(delivery.Presence{Online: online}) - if err != nil { - t.Fatal(err) - } - return b -} - -func TestPresenceSetKnowsOnlyNodesTheWatchmanSawOnline(t *testing.T) { // 1607 review 3 - var s PresenceSet - if s.Online("ace") { - t.Error("a node the watchman never wrote is online") - } - if !s.Put("ace", presenceValue(t, true), false) || !s.Online("ace") { - t.Error("ace online was not taken in") - } - if s.Put("ace", presenceValue(t, true), false) { - t.Error("the same presence again counted as a change") - } - if !s.Put("ace", presenceValue(t, false), false) || s.Online("ace") { - t.Error("ace going offline was not taken in") - } - s.Put("g14", presenceValue(t, true), false) - if !s.Put("g14", nil, true) || s.Online("g14") { - t.Error("a deleted key is still online") - } - // A part's instance (part..) is not a node. - if s.Put("part.mesh-delivery-builder.novox", presenceValue(t, true), false) || s.Online("part") { - t.Error("a part's key was taken for a node") - } - if s.Put("novox", []byte("junk"), false) { - t.Error("junk counted") - } -} diff --git a/modules/mesh-delivery-builder/internal/build/round4_test.go b/modules/mesh-delivery-builder/internal/build/round4_test.go index fc7f638..24f962f 100644 --- a/modules/mesh-delivery-builder/internal/build/round4_test.go +++ b/modules/mesh-delivery-builder/internal/build/round4_test.go @@ -2,6 +2,7 @@ package build import ( "context" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "strings" "testing" @@ -40,14 +41,14 @@ func TestAnArchiveOnlyModuleIsBuiltAlthoughItsFingerprintNamesNoToolchain(t *tes func TestAnArchiveOnlyModuleNeedsNoAgreedToolchainButNeedsNodesThatQualify(t *testing.T) { // review round 4, 1 r := newRig(t, archiveA) - r.tools.Put(GoBuildModule, "ace.go", tcValue(t, "go", "go1.26.0"), false) // the nodes disagree: a bundle would wait - r.b.Health = &HaveSet{} // no build module is healthy anywhere + r.tools.Put(buildrun.GoBuildModule, "ace.go", tcValue(t, "go", "go1.26.0"), false) // the nodes disagree: a bundle would wait + r.b.Health = &buildrun.HaveSet{} // no build module is healthy anywhere r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", nil)) r.b.Wait() if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.Contains(v.Error, "healthy") { t.Errorf("build = %+v, want failed: no node's build module is healthy", v) } - have := &HaveSet{} + have := &buildrun.HaveSet{} have.Put("novox.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) r.b.Health = have r.b.Reconsider(context.Background()) @@ -60,11 +61,11 @@ func TestAnArchiveOnlyModuleNeedsNoAgreedToolchainButNeedsNodesThatQualify(t *te func TestABundleAndAnArchiveOfOneBuildModuleStillNeedTheAgreedToolchain(t *testing.T) { // review round 4, 1 p, err := PlanOf(catalogue.Manifest{Module: "m", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle, archiveA}}}) - if err != nil || !p.NeedsToolchain(GoBuildModule) { - t.Errorf("a plan with a Go bundle: %v, %v", p.NeedsToolchain(GoBuildModule), err) + if err != nil || !p.NeedsToolchain(buildrun.GoBuildModule) { + t.Errorf("a plan with a Go bundle: %v, %v", p.NeedsToolchain(buildrun.GoBuildModule), err) } p, _ = PlanOf(catalogue.Manifest{Module: "m", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{archiveA}}}) - if p.NeedsToolchain(GoBuildModule) { + if p.NeedsToolchain(buildrun.GoBuildModule) { t.Error("a plan of an archive needs a toolchain") } } @@ -90,7 +91,7 @@ func TestDifferingDigestsFailTheKeyWithThePrefixAndNothingBuildsItAgain(t *testi func TestClaimLeavesAKeyFailedForDifferingDigestsAlone(t *testing.T) { // review round 4, 2 st := newMem() - seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, Error: DigestsDifferPrefix + " the nodes built tool to different digests: ace x; novox y"}) + seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, Error: buildrun.DigestsDifferPrefix + " the nodes built tool to different digests: ace x; novox y"}) writes := st.writes got, err := (Claimer{Store: st, By: "novox.a"}).Claim(context.Background(), "m", fp1, "", ClaimOptions{TakeOver: true}) if err != nil || got != nil || st.writes != writes { @@ -103,10 +104,10 @@ func TestADoneKeyOfOneNodeIsBuiltAgainOnASecondNodeAndTheDigestsCompared(t *test r := newRig(t, goBundle) r.pres.Put("ace", presenceValue(t, false), false) // only novox qualifies v := r.consider(t, "modules/m") - if v.State != delivery.BuildDone || !strings.HasPrefix(v.Step, OneNodePrefix+GoBuildModule+"): ") || !strings.Contains(v.Step, "on one node, novox") { + if v.State != delivery.BuildDone || !strings.HasPrefix(v.Step, OneNodePrefix+buildrun.GoBuildModule+"): ") || !strings.Contains(v.Step, "on one node, novox") { t.Fatalf("build = %+v, want done on one node, marked", v) } - if got := OneNodeModules(v); len(got) != 1 || got[0] != GoBuildModule { + if got := OneNodeModules(v); len(got) != 1 || got[0] != buildrun.GoBuildModule { t.Errorf("OneNodeModules = %v", got) } rev, calls := r.st.revs[r.key], len(r.bm.callsOf("build_go_build")) @@ -141,7 +142,7 @@ func TestASecondNodeThatDisagreesFailsTheDoneKeyWithTheDigestsPrefix(t *testing. r.pres.Put("ace", presenceValue(t, true), false) r.b.Reconsider(context.Background()) r.b.Wait() - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, DigestsDifferPrefix) { + if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, buildrun.DigestsDifferPrefix) { t.Errorf("build = %+v, want failed with digests differ", v) } } @@ -161,7 +162,7 @@ func TestASecondBuildThatFailsForAnotherReasonLeavesTheDoneKeyStanding(t *testin // 4. A step never waits on a full node while another qualifying node has room. func TestChoosesMadeAtOnceSeeEachOthersReservationsSoNoLaneWaitsWhileANodeHasRoom(t *testing.T) { // review round 4, 4 - p := &Places{Capacity: map[string]int{"ace": 2, "g14": 1, "novox": 1, "shanks": 3}} + p := &buildrun.Places{Capacity: map[string]int{"ace": 2, "g14": 1, "novox": 1, "shanks": 3}} entries := tcs("go", "go1.27.1", "ace", "g14", "novox", "shanks") total := 2 + 1 + 1 + 3 var waiting int @@ -170,7 +171,7 @@ func TestChoosesMadeAtOnceSeeEachOthersReservationsSoNoLaneWaitsWhileANodeHasRoo for _, n := range []string{"ace", "g14", "novox", "shanks"} { room += p.CapacityOf(n) - p.Running()[n] - p.Reserved()[n] } - nodes, _, err := ChooseNodes("go", entries, p, nil, 2) + nodes, _, err := buildrun.ChooseNodes("go", entries, p, nil, 2) if err != nil || len(nodes) != 2 { t.Fatalf("choice %d: %v, %v", i, nodes, err) } @@ -199,8 +200,8 @@ func TestChoosesMadeAtOnceSeeEachOthersReservationsSoNoLaneWaitsWhileANodeHasRoo } func TestACancelledReservationFreesTheChoiceAndAnAcquireTakesItsReservation(t *testing.T) { // review round 4, 4 - p := &Places{} - nodes, _, _ := ChooseNodes("go", tcs("go", "go1.27.1", "ace"), p, nil, 1) + p := &buildrun.Places{} + nodes, _, _ := buildrun.ChooseNodes("go", tcs("go", "go1.27.1", "ace"), p, nil, 1) if len(nodes) != 1 || p.Reserved()["ace"] != 1 { t.Fatalf("nodes %v, reserved %v", nodes, p.Reserved()) } @@ -209,7 +210,7 @@ func TestACancelledReservationFreesTheChoiceAndAnAcquireTakesItsReservation(t *t t.Fatalf("acquire: %v, reserved %v, running %v", err, p.Reserved(), p.Running()) } release() - ChooseNodes("go", tcs("go", "go1.27.1", "ace"), p, nil, 1) + buildrun.ChooseNodes("go", tcs("go", "go1.27.1", "ace"), p, nil, 1) p.Cancel("ace") if len(p.Reserved()) != 0 { t.Errorf("reserved after Cancel = %v", p.Reserved()) @@ -218,25 +219,25 @@ func TestACancelledReservationFreesTheChoiceAndAnAcquireTakesItsReservation(t *t // 5. A node whose build module is not healthy is not chosen. func TestHaveSetKnowsOnlyBuildModulesRunningAndHealthy(t *testing.T) { // review round 4, 5 - var s HaveSet - if s.Healthy("ace", GoBuildModule) { + var s buildrun.HaveSet + if s.Healthy("ace", buildrun.GoBuildModule) { t.Error("a module with no have is healthy") } - if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) || !s.Healthy("ace", GoBuildModule) { + if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) || !s.Healthy("ace", buildrun.GoBuildModule) { t.Error("a running, healthy module is not") } if s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) { t.Error("the same state counted as a change") } - if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthFailed), false) || s.Healthy("ace", GoBuildModule) { + if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthFailed), false) || s.Healthy("ace", buildrun.GoBuildModule) { t.Error("a module whose health failed is healthy") } s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) - if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseFailed, delivery.HealthOK), false) || s.Healthy("ace", GoBuildModule) { + if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseFailed, delivery.HealthOK), false) || s.Healthy("ace", buildrun.GoBuildModule) { t.Error("a module whose phase failed is healthy") } s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) - if !s.Put("ace.mesh-build-go", nil, true) || s.Healthy("ace", GoBuildModule) { + if !s.Put("ace.mesh-build-go", nil, true) || s.Healthy("ace", buildrun.GoBuildModule) { t.Error("a deleted have is healthy") } if s.Put("ace.node", []byte(`{}`), false) || s.Put("ace.mesh-build-go", []byte(`not json`), false) { @@ -246,7 +247,7 @@ func TestHaveSetKnowsOnlyBuildModulesRunningAndHealthy(t *testing.T) { // review func TestANodeWhoseBuildModuleIsNotHealthyIsNotChosen(t *testing.T) { // review round 4, 5 r := newRig(t, goBundle) - have := &HaveSet{} + have := &buildrun.HaveSet{} have.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthFailed), false) // go is not the recipe's have.Put("novox.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) r.b.Health = have diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 53de028..66877e3 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "sort" "strings" "sync" @@ -12,13 +13,13 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" ) -// ToolchainEntries are the nodes' toolchain entries of a language (ToolchainSet). +// ToolchainEntries are the nodes' toolchain entries of a language (buildrun.ToolchainSet). type ToolchainEntries interface { Entries(language string) []delivery.NodeToolchain } // toolPrefix is the start of each build module's tool names: fetch and build. -var toolPrefix = map[string]string{GoBuildModule: "build_go_", NodeBuildModule: "build_node_"} +var toolPrefix = map[string]string{buildrun.GoBuildModule: "build_go_", buildrun.NodeBuildModule: "build_node_"} // Builder builds, dark, every new-form fingerprint the indexer writes (novox/hq ADR 0453 clause 4, ADR 0478): it is // handed each modules value as the watch of the indexer's bucket sees it, and for each module with a fingerprint and an @@ -27,7 +28,7 @@ var toolPrefix = map[string]string{GoBuildModule: "build_go_", NodeBuildModule: // change of the toolchains. type Builder struct { Claimer Claimer - Runner Runner + Runner buildrun.Runner Toolchains ToolchainEntries // Presence says which nodes are online (the watchman's); a node that is not is not chosen. Presence interface{ Online(node string) bool } @@ -37,8 +38,8 @@ type Builder struct { Health interface { Healthy(node, module string) bool } - // Places holds a step back while its node is at its build capacity; a Builder with none runs one step at a time on each node. - Places *Places + // buildrun.Places holds a step back while its node is at its build capacity; a Builder with none runs one step at a time on each node. + Places *buildrun.Places // Nodes is how many build nodes build each fingerprint: two until release R4e (ADR 0478 clause 4). Nodes int // Limit is how many builds run at once; four when 0. @@ -75,7 +76,7 @@ func (b *Builder) init() { b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{} b.secondAt = map[string]uint64{} if b.Places == nil { - b.Places = &Places{} + b.Places = &buildrun.Places{} } }) } @@ -293,7 +294,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { return } result.By, result.Manifest = b.Claimer.By, m.Manifest - if cl.Build.State == delivery.BuildDone && result.State == delivery.BuildFailed && !strings.HasPrefix(result.Error, DigestsDifferPrefix) { + if cl.Build.State == delivery.BuildDone && result.State == delivery.BuildFailed && !strings.HasPrefix(result.Error, buildrun.DigestsDifferPrefix) { // A done key built again on a second node keeps its done value when the new build fails for another reason than // the digests: the first build stands, and a later change of the nodes looks again. b.logf("%s: built again on a second node and failed, the done build stands: %s", cl.Key, result.Error) @@ -318,7 +319,7 @@ func (b *Builder) stale(m delivery.Module, plan Plan) string { if !plan.NeedsToolchain(bm) { continue // an archive is packed with no toolchain } - language := LanguageOf[bm] + language := buildrun.LanguageOf[bm] tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language)) if none != "" { return fmt.Sprintf("the %s toolchain is not agreed: %s", language, none) @@ -367,12 +368,12 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p } } for _, bm := range plan.Modules() { - language := LanguageOf[bm] + language := buildrun.LanguageOf[bm] entries := b.Toolchains.Entries(language) qualifies := func(node string) bool { return b.qualifies(node, bm) } var nodes []string if plan.NeedsToolchain(bm) { - ns, tc, err := ChooseNodes(language, entries, b.Places, qualifies, b.Nodes) + ns, tc, err := buildrun.ChooseNodes(language, entries, b.Places, qualifies, b.Nodes) if err != nil { giveUpLanes() return failed("%s", err) @@ -386,7 +387,7 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p nodes = ns } else { // An archive is packed with no toolchain (the fingerprint names none): any node of the build module that qualifies. - ns, err := ChooseAnyNodes(language, entries, b.Places, qualifies, b.Nodes) + ns, err := buildrun.ChooseAnyNodes(language, entries, b.Places, qualifies, b.Nodes) if err != nil { giveUpLanes() return failed("%s", err) @@ -434,7 +435,7 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p byNode[l.node] = d } } - d, err := Agree(t.Artifact, byNode) + d, err := buildrun.Agree(t.Artifact, byNode) if err != nil { return failed("%s", err) } @@ -463,7 +464,7 @@ func (b *Builder) moreNodes(modules []string) bool { for _, bm := range modules { seen := map[string]bool{} n := 0 - for _, e := range b.Toolchains.Entries(LanguageOf[bm]) { + for _, e := range b.Toolchains.Entries(buildrun.LanguageOf[bm]) { if !seen[e.Node] && b.qualifies(e.Node, bm) { seen[e.Node] = true n++ diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 34c89a6..9764790 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "strings" "sync" "testing" @@ -94,22 +95,22 @@ type rig struct { b *Builder st *memStore bm *buildModules - tools *ToolchainSet - pres *PresenceSet + tools *buildrun.ToolchainSet + pres *buildrun.PresenceSet mf *fakeManifests key string } func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig { t.Helper() - r := &rig{st: newMem(), bm: newBuildModules(), tools: &ToolchainSet{}, pres: &PresenceSet{}, mf: &fakeManifests{byDigest: map[string]catalogue.Manifest{mdigest('a'): {Module: "m", Build: &catalogue.Build{Artifacts: artifacts}}}}} + r := &rig{st: newMem(), bm: newBuildModules(), tools: &buildrun.ToolchainSet{}, pres: &buildrun.PresenceSet{}, mf: &fakeManifests{byDigest: map[string]catalogue.Manifest{mdigest('a'): {Module: "m", Build: &catalogue.Build{Artifacts: artifacts}}}}} for _, n := range []string{"ace", "novox"} { - r.tools.Put(GoBuildModule, n+".go", tcValue(t, "go", "go1.27.1"), false) - r.tools.Put(NodeBuildModule, n+".typescript", tcValue(t, "typescript", "tsc5"), false) + r.tools.Put(buildrun.GoBuildModule, n+".go", tcValue(t, "go", "go1.27.1"), false) + r.tools.Put(buildrun.NodeBuildModule, n+".typescript", tcValue(t, "typescript", "tsc5"), false) r.pres.Put(n, presenceValue(t, true), false) } r.b = &Builder{ - Claimer: Claimer{Store: r.st, By: "novox.builder"}, Runner: Runner{Caller: r.bm, Steps: r.bm.steps, Bound: time.Second}, + Claimer: Claimer{Store: r.st, By: "novox.builder"}, Runner: buildrun.Runner{Caller: r.bm, Steps: r.bm.steps, Bound: time.Second}, Toolchains: r.tools, Presence: r.pres, Manifests: r.mf, Nodes: 2, Logf: t.Logf, } var err error @@ -202,7 +203,7 @@ func TestDifferingDigestsFailTheBuildNamingBothNodes(t *testing.T) { // 1607/AC4 func TestOneQualifyingNodeBuildsOnceAndTheKeySaysSo(t *testing.T) { // 1607/AC4, ADR 0478 clause 4 r := newRig(t, goBundle) - r.tools.Put(GoBuildModule, "ace.go", nil, true) + r.tools.Put(buildrun.GoBuildModule, "ace.go", nil, true) v := r.consider(t, "modules/m") if v.State != delivery.BuildDone || !strings.Contains(v.Step, "one node") || !strings.Contains(v.Step, "novox") { t.Errorf("build = %+v, want done with its step saying one node, novox", v) @@ -340,8 +341,8 @@ func TestAKeyIsBuiltOnlyWithTheToolchainItsFingerprintNamesAndNothingIsWrittenOt t.Errorf("keys %v and %d calls for a recipe that differs or a value that names no toolchain", r.st.vals, len(r.bm.calls)) } // The toolchain moves under a key that is not built yet: the old fingerprint stays unbuilt, the new one builds. - r.tools.Put(GoBuildModule, "ace.go", tcValue(t, "go", "go1.27.2"), false) - r.tools.Put(GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.2"), false) + r.tools.Put(buildrun.GoBuildModule, "ace.go", tcValue(t, "go", "go1.27.2"), false) + r.tools.Put(buildrun.GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.2"), false) r.b.Reconsider(context.Background()) r.b.Wait() if len(r.st.vals) != 0 { @@ -356,7 +357,7 @@ func TestAKeyIsBuiltOnlyWithTheToolchainItsFingerprintNamesAndNothingIsWrittenOt func TestNodesTheyDisagreeOnLeaveTheKeyUnwritten(t *testing.T) { // 1615/AC4 r := newRig(t, goBundle) - r.tools.Put(GoBuildModule, "ace.go", tcValue(t, "go", "go1.26.0"), false) + r.tools.Put(buildrun.GoBuildModule, "ace.go", tcValue(t, "go", "go1.26.0"), false) r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) r.b.Wait() if len(r.st.vals) != 0 || len(r.bm.calls) != 0 { @@ -376,7 +377,7 @@ func TestARootModulesPathIsDotAndIsGivenToTheBuildTools(t *testing.T) { // 1615/ func TestANodeThatIsNotOnlineIsNotChosenAndComingBackWakesTheBuild(t *testing.T) { // 1607 review 3 r := newRig(t, goBundle) - r.tools.Put(GoBuildModule, "g14.go", tcValue(t, "go", "go1.27.1"), false) // a laptop that reports its toolchain and sleeps + r.tools.Put(buildrun.GoBuildModule, "g14.go", tcValue(t, "go", "go1.27.1"), false) // a laptop that reports its toolchain and sleeps v := r.consider(t, "modules/m") if v.State != delivery.BuildDone { t.Fatalf("build = %+v", v) diff --git a/modules/mesh-delivery-builder/internal/build/steps.go b/modules/mesh-delivery-builder/internal/build/steps.go deleted file mode 100644 index 3a84936..0000000 --- a/modules/mesh-delivery-builder/internal/build/steps.go +++ /dev/null @@ -1,67 +0,0 @@ -package build - -import ( - "context" - "encoding/json" - "fmt" - "time" - - "git.novox.be/novox/mesh-sdk/go/delivery" -) - -// Caller calls a tool of a module on one named machine (mesh-sdk process.Conn.CallOn; ADR 0469 clause 3). -type Caller interface { - CallOn(node, key string, body any) (json.RawMessage, error) -} - -// Steps waits for the end of a build module's long step: the value of its key `.` in the module's steps state -// (ADR 0469 clause 2) once its state is not running. It is a watch of that key, not a poll; it returns when the step -// ends or ctx does. -type Steps interface { - Wait(ctx context.Context, module, key string) (delivery.Step, error) -} - -// Runner runs a build module's long steps: the call answers at once with the step's key, and the result is read from that -// key when the step ends (ADR 0469 clause 2). -type Runner struct { - Caller Caller - Steps Steps - // Bound is how long a step may run before the builder stops waiting for it. - Bound time.Duration -} - -// Run calls tool of the build module on node with args, waits for the step's end and answers its result. A step that -// failed or was refused is an error naming the node, the state and the reason. -func (r Runner) Run(ctx context.Context, module, node, tool string, args map[string]any) (json.RawMessage, error) { - raw, err := r.Caller.CallOn(node, module+"."+tool, args) - if err != nil { - return nil, fmt.Errorf("%s on %s: %w", tool, node, err) - } - var answer struct { - Step string `json:"step"` - Key string `json:"key"` - State string `json:"state"` - } - if err := json.Unmarshal(raw, &answer); err != nil { - return nil, fmt.Errorf("%s on %s answered what is not a step: %w", tool, node, err) - } - if answer.Key == "" { - return nil, fmt.Errorf("%s on %s answered no step key: %s", tool, node, raw) - } - if n, _, err := delivery.ParseStepKey(answer.Key); err != nil || n != node { - return nil, fmt.Errorf("%s on %s answered the step key %q, which is not a key of that node", tool, node, answer.Key) - } - wait, cancel := context.WithTimeoutCause(ctx, r.Bound, fmt.Errorf("the step %s did not end within %s", answer.Key, r.Bound)) - defer cancel() - step, err := r.Steps.Wait(wait, module, answer.Key) - if err != nil { - if cause := context.Cause(wait); cause != nil && ctx.Err() == nil { - err = cause - } - return nil, fmt.Errorf("%s on %s: %w", tool, node, err) - } - if step.State != delivery.StepPassed { - return nil, fmt.Errorf("%s on %s: the step %s %s: %s", tool, node, answer.Key, step.State, step.Reason) - } - return step.Result, nil -} diff --git a/modules/mesh-delivery-builder/internal/build/steps_test.go b/modules/mesh-delivery-builder/internal/build/steps_test.go deleted file mode 100644 index 4a0503b..0000000 --- a/modules/mesh-delivery-builder/internal/build/steps_test.go +++ /dev/null @@ -1,116 +0,0 @@ -package build - -import ( - "context" - "encoding/json" - "errors" - "strings" - "sync" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/delivery" -) - -type call struct { - node, key string - body any -} - -// fakeCaller answers a long tool as the build modules do: at once, with the step and its key. -type fakeCaller struct { - calls []call - answer string - err error -} - -func (f *fakeCaller) CallOn(node, key string, body any) (json.RawMessage, error) { - f.calls = append(f.calls, call{node, key, body}) - if f.err != nil { - return nil, f.err - } - return json.RawMessage(f.answer), nil -} - -// fakeSteps holds the steps values by `/`; Wait on a key it does not hold blocks until the context ends. -type fakeSteps struct { - mu sync.Mutex - steps map[string]delivery.Step - // during, when set, is called as a wait starts and the function it answers as the wait ends. - during func(key string) func() -} - -func (f *fakeSteps) Wait(ctx context.Context, module, key string) (delivery.Step, error) { - f.mu.Lock() - s, ok := f.steps[module+"/"+key] - during := f.during - f.mu.Unlock() - if ok { - if during != nil { - defer during(key)() - } - return s, nil - } - <-ctx.Done() - return delivery.Step{}, context.Cause(ctx) -} - -func stepAt(node, id, state, reason string, result string) delivery.Step { - s := delivery.Step{ID: id, Tool: "build_go_fetch", Node: node, State: state, Reason: reason} - if result != "" { - s.Result = json.RawMessage(result) - } - return s -} - -func TestRunCallsTheToolOnTheNodeAndReadsTheStepsKey(t *testing.T) { // 1607/AC5 - caller := &fakeCaller{answer: `{"step":"abc1","key":"ace.abc1","state":"running"}`} - steps := &fakeSteps{steps: map[string]delivery.Step{"mesh-build-go/ace.abc1": stepAt("ace", "abc1", delivery.StepPassed, "", `{"workspace":"w1","commit":"c"}`)}} - res, err := Runner{Caller: caller, Steps: steps, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_fetch", map[string]any{"repository": "novox/r", "commit": "c"}) - if err != nil { - t.Fatal(err) - } - if len(caller.calls) != 1 || caller.calls[0].node != "ace" || caller.calls[0].key != "mesh-build-go.build_go_fetch" { - t.Errorf("calls = %+v, want one on ace for mesh-build-go.build_go_fetch", caller.calls) - } - var fr struct{ Workspace string } - if err := json.Unmarshal(res, &fr); err != nil || fr.Workspace != "w1" { - t.Errorf("result = %s (%v), want the step's result", res, err) - } -} - -func TestRunFailsOnAFailedOrRefusedStepNamingItsReason(t *testing.T) { // 1607/AC5 - for _, state := range []string{delivery.StepFailed, delivery.StepRefused} { - caller := &fakeCaller{answer: `{"step":"abc1","key":"ace.abc1","state":"running"}`} - steps := &fakeSteps{steps: map[string]delivery.Step{"mesh-build-go/ace.abc1": stepAt("ace", "abc1", state, "go is go1.26.0, the recipe builds with go1.27.1", "")}} - _, err := Runner{Caller: caller, Steps: steps, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_build", nil) - if err == nil || !strings.Contains(err.Error(), "go1.27.1") || !strings.Contains(err.Error(), state) || !strings.Contains(err.Error(), "ace") { - t.Errorf("%s: error = %v, want the node, the state and the reason", state, err) - } - } -} - -func TestRunStopsWaitingAtTheBound(t *testing.T) { // 1607/AC5 - caller := &fakeCaller{answer: `{"step":"abc1","key":"ace.abc1","state":"running"}`} - _, err := Runner{Caller: caller, Steps: &fakeSteps{}, Bound: 20 * time.Millisecond}.Run(context.Background(), "mesh-build-go", "ace", "build_go_build", nil) - if err == nil || !strings.Contains(err.Error(), "ace.abc1") || !strings.Contains(err.Error(), "did not end") { - t.Errorf("error = %v, want the step key and that it did not end within its bound", err) - } -} - -func TestRunRefusesAnAnswerWithNoKeyAndAKeyOfAnotherNode(t *testing.T) { // 1607/AC5 - for _, answer := range []string{`{"state":"running"}`, `{"step":"abc1","key":"novox.abc1","state":"running"}`, `not json`} { - _, err := Runner{Caller: &fakeCaller{answer: answer}, Steps: &fakeSteps{}, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_fetch", nil) - if err == nil { - t.Errorf("answer %s: no error", answer) - } - } -} - -func TestRunReturnsTheCallsOwnError(t *testing.T) { // 1607/AC5 - boom := errors.New("no responders") - _, err := Runner{Caller: &fakeCaller{err: boom}, Steps: &fakeSteps{}, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_fetch", nil) - if !errors.Is(err, boom) || !strings.Contains(err.Error(), "ace") { - t.Errorf("error = %v, want the call's error naming the node", err) - } -} diff --git a/modules/mesh-delivery-builder/internal/build/toolchains.go b/modules/mesh-delivery-builder/internal/build/toolchains.go deleted file mode 100644 index 44af6e6..0000000 --- a/modules/mesh-delivery-builder/internal/build/toolchains.go +++ /dev/null @@ -1,64 +0,0 @@ -package build - -import ( - "sync" - - "git.novox.be/novox/mesh-sdk/go/delivery" -) - -// LanguageOf is the language each build module's toolchains state holds (the indexer's ToolchainModules, reversed). -var LanguageOf = map[string]string{GoBuildModule: "go", NodeBuildModule: "typescript"} - -// ToolchainSet holds what the build modules' toolchains buckets hold: each node's entry of a language, as the watches -// put them in. Safe for use from the watches and from a build at once. -type ToolchainSet struct { - mu sync.Mutex - by map[string]map[string]delivery.Toolchain // language -> node -> entry -} - -// Put takes a change of a build module's toolchains bucket in: a put of a node's entry (key .) or its -// delete. It answers whether the language's entries changed (a node's version or recipe, or a node appearing or going). -// A key or a value that is not an entry of the module's language is left out. -func (s *ToolchainSet) Put(module, key string, value []byte, deleted bool) bool { - language, ok := LanguageOf[module] - if !ok { - return false - } - node, keyLanguage, err := delivery.ParseToolchainKey(key) - if err != nil || keyLanguage != language { - return false - } - s.mu.Lock() - defer s.mu.Unlock() - if deleted { - if _, had := s.by[language][node]; !had { - return false - } - delete(s.by[language], node) - return true - } - tc, err := delivery.DecodeToolchain(value) - if err != nil || tc.Language != language { - return false - } - if s.by == nil { - s.by = map[string]map[string]delivery.Toolchain{} - } - if s.by[language] == nil { - s.by[language] = map[string]delivery.Toolchain{} - } - cur, had := s.by[language][node] - s.by[language][node] = tc - return !had || cur.Version != tc.Version || cur.Recipe != tc.Recipe -} - -// Entries are the nodes' entries of a language, in no particular order. -func (s *ToolchainSet) Entries(language string) []delivery.NodeToolchain { - s.mu.Lock() - defer s.mu.Unlock() - var out []delivery.NodeToolchain - for node, tc := range s.by[language] { - out = append(out, delivery.NodeToolchain{Node: node, Toolchain: tc}) - } - return out -} diff --git a/modules/mesh-delivery-builder/internal/build/toolchains_test.go b/modules/mesh-delivery-builder/internal/build/toolchains_test.go deleted file mode 100644 index 83b95ca..0000000 --- a/modules/mesh-delivery-builder/internal/build/toolchains_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package build - -import ( - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/delivery" -) - -func tcValue(t *testing.T, language, version string) []byte { - t.Helper() - b, err := delivery.EncodeToolchain(delivery.Toolchain{Language: language, Version: version, Recipe: "1", At: time.Now()}) - if err != nil { - t.Fatal(err) - } - return b -} - -func TestToolchainSetKeepsEachNodesEntryByLanguageAndSaysWhenOneChanged(t *testing.T) { // 1607/AC6 - var s ToolchainSet - if !s.Put(GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.1"), false) { - t.Error("a first entry did not count as a change") - } - if s.Put(GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.1"), false) { - t.Error("the same entry again counted as a change") - } - s.Put(GoBuildModule, "ace.go", tcValue(t, "go", "go1.27.1"), false) - s.Put(NodeBuildModule, "ace.typescript", tcValue(t, "typescript", "tsc5"), false) - if got := s.Entries("go"); len(got) != 2 { - t.Errorf("go entries = %+v, want two", got) - } - if got := s.Entries("typescript"); len(got) != 1 || got[0].Node != "ace" || got[0].Version != "tsc5" { - t.Errorf("typescript entries = %+v", got) - } - if !s.Put(GoBuildModule, "ace.go", nil, true) || len(s.Entries("go")) != 1 { - t.Error("a delete did not remove the node's entry") - } -} - -func TestToolchainSetLeavesOutWhatIsNotAToolchainOfTheModulesLanguage(t *testing.T) { // 1607/AC6 - var s ToolchainSet - if s.Put(GoBuildModule, "novox.typescript", tcValue(t, "typescript", "x"), false) { - t.Error("a typescript toolchain in the go build module's bucket counted") - } - if s.Put(GoBuildModule, "novox.go", []byte("not json"), false) || s.Put("other-module", "novox.go", tcValue(t, "go", "x"), false) { - t.Error("garbage or an unknown module counted") - } - if len(s.Entries("go")) != 0 { - t.Error("an entry was kept") - } -} From a2a5167d72ce3c16229738b12ea7faaea78a2789 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 21:25:45 +0200 Subject: [PATCH 09/19] Write a preview of the conflicts for a head that does not merge, and give the candidate's modules the reason; pin mesh-sdk to the R4c-5a head (issue 1445) --- modules/mesh-delivery-builder/go.mod | 2 +- modules/mesh-delivery-builder/go.sum | 4 ++-- .../cmd/mesh-delivery-checker/integration_test.go | 9 ++++++--- modules/mesh-delivery-checker/go.mod | 2 +- modules/mesh-delivery-checker/go.sum | 6 ++---- .../internal/check/examiner_test.go | 12 +++++++++--- .../mesh-delivery-checker/internal/check/judge.go | 1 + .../internal/check/preview.go | 15 ++++++++++++++- modules/mesh-delivery-forge-bridge/go.mod | 2 +- modules/mesh-delivery-forge-bridge/go.sum | 4 ++-- modules/mesh-delivery-indexer/go.mod | 2 +- modules/mesh-delivery-indexer/go.sum | 4 ++-- .../internal/index/previews.go | 5 +++++ .../internal/index/previews_test.go | 12 ++++++++++++ 14 files changed, 59 insertions(+), 21 deletions(-) diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index b8b9bd8..59b27dd 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-builder go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-builder/go.sum b/modules/mesh-delivery-builder/go.sum index 7c963a0..3515fb8 100644 --- a/modules/mesh-delivery-builder/go.sum +++ b/modules/mesh-delivery-builder/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5 h1:1BiPKRPTYpgex8rSHWT9az0BteOafGWRDzd/Ba6GAcA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go index f8e50db..977b02d 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go @@ -443,7 +443,7 @@ func keptCount(t *testing.T, b *bed) int { } // TestAHeadThatDoesNotMergeFailsNamingTheFilesAndRunsNoTest: the fetch answers the files in conflict; the verdict fails naming -// them, no test or digest step is run, no preview is written and the status is owed as failure. +// them, no test or digest step is run, a preview of the conflicts alone is written and the status is owed as failure. func TestAHeadThatDoesNotMergeFailsNamingTheFilesAndRunsNoTest(t *testing.T) { // 1289/AC4 b := startBed(t) b.fake.conflict.Store(true) @@ -459,8 +459,11 @@ func TestAHeadThatDoesNotMergeFailsNamingTheFilesAndRunsNoTest(t *testing.T) { / if n := b.fake.count("mesh-build-go.build_go_digest") + b.fake.count("mesh-build-go.build_go_test"); n != 0 { t.Errorf("%d digest and test steps ran after a conflict", n) } - if keys, _ := b.buckets[delivery.PreviewsBucket].Keys(b.ctx); len(keys) != 0 { - t.Errorf("previews %v for a head that does not merge", keys) + pk, _ := delivery.PreviewKey(repo, head1, trunk) + if e, err := b.buckets[delivery.PreviewsBucket].Get(b.ctx, pk); err != nil { + t.Errorf("previews.%s: %v", pk, err) + } else if pv, err := delivery.DecodePreview(e.Value()); err != nil || strings.Join(pv.Conflicts, ",") != "a.go,sub/b.go" || pv.Merge != "" { + t.Errorf("preview = %+v, %v, want the files in conflict alone", pv, err) } } diff --git a/modules/mesh-delivery-checker/go.mod b/modules/mesh-delivery-checker/go.mod index 5035651..9c911a8 100644 --- a/modules/mesh-delivery-checker/go.mod +++ b/modules/mesh-delivery-checker/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-checker go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-checker/go.sum b/modules/mesh-delivery-checker/go.sum index 6f440f0..3515fb8 100644 --- a/modules/mesh-delivery-checker/go.sum +++ b/modules/mesh-delivery-checker/go.sum @@ -1,7 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865 h1:VLir9NZbyi1LdH1YwWDIVRuL+lDC+AYlm6FZPlvmlm4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5 h1:1BiPKRPTYpgex8rSHWT9az0BteOafGWRDzd/Ba6GAcA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go index 39e1ac8..f5b239b 100644 --- a/modules/mesh-delivery-checker/internal/check/examiner_test.go +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -481,12 +481,18 @@ func TestACleanMergeWritesItsPreviewWithTheTreesManifestsAndLocks(t *testing.T) } } -func TestAHeadThatDoesNotMergeWritesNoPreview(t *testing.T) { // 1445/AC5 +func TestAHeadThatDoesNotMergeWritesAPreviewOfItsConflictsAlone(t *testing.T) { // 1445/AC5 b := newBench(t) b.fetch = captured[Fetched](t, "go-fetch-conflict") b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) - if len(b.prev.vals) != 0 { - t.Errorf("previews %v written for a head that does not merge", b.prev.vals) + key, _ := delivery.PreviewKey(repoA, head1, trunkA) + raw, _, found, _ := b.prev.Get(context.Background(), key) + if !found { + t.Fatalf("no preview for a head that does not merge; the bucket holds %v", b.prev.vals) + } + pv, err := delivery.DecodePreview(raw) + if err != nil || strings.Join(pv.Conflicts, ",") != "a.go,sub/b.go" || pv.Merge != "" || len(pv.Modules) != 0 { + t.Errorf("preview = %+v, %v, want the files in conflict and no merge", pv, err) } } diff --git a/modules/mesh-delivery-checker/internal/check/judge.go b/modules/mesh-delivery-checker/internal/check/judge.go index 0a9ce4b..1785c5d 100644 --- a/modules/mesh-delivery-checker/internal/check/judge.go +++ b/modules/mesh-delivery-checker/internal/check/judge.go @@ -226,6 +226,7 @@ func (st *state) fetchAll(ctx context.Context) *delivery.Verdict { return fail(node, fmt.Sprintf("the fetch of %s#%d answered no workspace: %s", p.Repository, p.Number, raw)) } if len(t.fetched.Conflicts) > 0 { + st.j.writePreview(ctx, p, append(merged, t.fetched)) return fail(node, fmt.Sprintf("%s#%d does not merge into the trunk: files in conflict: %s", p.Repository, p.Number, strings.Join(t.fetched.Conflicts, ", "))) } t.changed = t.fetched.Changed diff --git a/modules/mesh-delivery-checker/internal/check/preview.go b/modules/mesh-delivery-checker/internal/check/preview.go index a037246..5d4cf56 100644 --- a/modules/mesh-delivery-checker/internal/check/preview.go +++ b/modules/mesh-delivery-checker/internal/check/preview.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "regexp" + "slices" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -15,10 +16,22 @@ var digest64 = regexp.MustCompile(`^[0-9a-f]{64}$`) // key (ADR 0484 clause 3): the merge commit and, for each module directory the fetches named with a manifest, its tree, the // digest of its manifest and the hashes of its lock files. The fetches of the repository's build modules merge the same // trees; the first one's merge commit is kept. A module directory with no tree or no manifest digest is left out, so the -// indexer gives its module no fingerprint and says why. It answers false when no fetch merged. +// indexer gives its module no fingerprint and says why. A head that does not merge has a preview of the files in conflict +// alone, so the indexer says the candidate's modules have no fingerprint because it does not merge (issue 1445). It +// answers false when no fetch was made. func PreviewOf(p Pull, fetched []Fetched) (delivery.Preview, bool) { var merge string + var conflicts []string mods := map[string]delivery.PreviewModule{} + for _, f := range fetched { + if len(f.Conflicts) > 0 { + conflicts = append(conflicts, f.Conflicts...) + } + } + if len(conflicts) > 0 { + slices.Sort(conflicts) + return delivery.Preview{Repository: p.Repository, Head: p.Head, Trunk: p.Trunk, Conflicts: slices.Compact(conflicts)}, true + } for _, f := range fetched { if f.Merge == "" { continue diff --git a/modules/mesh-delivery-forge-bridge/go.mod b/modules/mesh-delivery-forge-bridge/go.mod index d073e5f..383758b 100644 --- a/modules/mesh-delivery-forge-bridge/go.mod +++ b/modules/mesh-delivery-forge-bridge/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-forge-bridge/go.sum b/modules/mesh-delivery-forge-bridge/go.sum index 7c963a0..3515fb8 100644 --- a/modules/mesh-delivery-forge-bridge/go.sum +++ b/modules/mesh-delivery-forge-bridge/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5 h1:1BiPKRPTYpgex8rSHWT9az0BteOafGWRDzd/Ba6GAcA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index 94177a4..a97df59 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-indexer go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-indexer/go.sum b/modules/mesh-delivery-indexer/go.sum index 7c963a0..3515fb8 100644 --- a/modules/mesh-delivery-indexer/go.sum +++ b/modules/mesh-delivery-indexer/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5 h1:1BiPKRPTYpgex8rSHWT9az0BteOafGWRDzd/Ba6GAcA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/internal/index/previews.go b/modules/mesh-delivery-indexer/internal/index/previews.go index 2aedb7d..ddd0a51 100644 --- a/modules/mesh-delivery-indexer/internal/index/previews.go +++ b/modules/mesh-delivery-indexer/internal/index/previews.go @@ -19,6 +19,9 @@ import ( const ( // NonePreviewStale is for a module whose preview was made onto a trunk head that is not the trunk's head now. NonePreviewStale = "the merge preview is out of date: the trunk moved since it was made" + // NonePreviewConflict opens the reason of a module whose pull request's head does not merge into the trunk; the files + // in conflict follow. + NonePreviewConflict = "does not merge: the head conflicts with the trunk in " // NonePreviewNoModule is for a module whose directory the preview names no manifest for. NonePreviewNoModule = "the merge preview names no manifest for the module's directory" // NonePreviewNoManifest is for a module whose manifest at the merge the indexer does not hold. @@ -150,6 +153,8 @@ func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool switch { case stale: next.None2 = NonePreviewStale + case len(pv.Conflicts) > 0: + next.None2 = NonePreviewConflict + strings.Join(pv.Conflicts, ", ") default: dir, known := dirs[m] if !known { diff --git a/modules/mesh-delivery-indexer/internal/index/previews_test.go b/modules/mesh-delivery-indexer/internal/index/previews_test.go index f5484e6..5d32716 100644 --- a/modules/mesh-delivery-indexer/internal/index/previews_test.go +++ b/modules/mesh-delivery-indexer/internal/index/previews_test.go @@ -179,3 +179,15 @@ func TestTheSamePreviewAppliedAgainWritesNothing(t *testing.T) { t.Errorf("candidates written %d times for the same preview, want none", cands.writes-w) } } + +func TestAPreviewOfConflictsGivesTheCandidatesModuleTheReasonItDoesNotMerge(t *testing.T) { // 1445/AC5 + ix, _, cands := previewBed(t, true) + if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil { + t.Fatal(err) + } + applyPreview(t, ix, delivery.Preview{Repository: catalogueRepo, Head: h1, Trunk: trunkNow, Conflicts: []string{"modules/avahi/main.go", "go.mod"}}) + got := fpOf(t, cands) + if got.FP2 != "" || !strings.HasPrefix(got.None2, "does not merge") || !strings.Contains(got.None2, "go.mod, modules/avahi/main.go") && !strings.Contains(got.None2, "modules/avahi/main.go, go.mod") { + t.Errorf("candidate avahi = %+v, want it not merging and naming the files", got) + } +} From 7433b0d047be9b48c6a09c478360da65217c240c Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 21:26:46 +0200 Subject: [PATCH 10/19] Start no check while a delivery builds: the checker reads the controller's bucket of the plans in their build phase, absent as empty (issue 1685) --- .../cmd/mesh-delivery-checker/hold.go | 76 +++++++++++++++++++ .../mesh-delivery-checker/integration_test.go | 28 +++++++ .../cmd/mesh-delivery-checker/main.go | 4 + .../internal/check/hold.go | 40 ++++++++++ .../internal/check/hold_test.go | 60 +++++++++++++++ 5 files changed, 208 insertions(+) create mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker/hold.go create mode 100644 modules/mesh-delivery-checker/internal/check/hold.go create mode 100644 modules/mesh-delivery-checker/internal/check/hold_test.go diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/hold.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/hold.go new file mode 100644 index 0000000..dbbbfc7 --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/hold.go @@ -0,0 +1,76 @@ +package main + +import ( + "context" + "errors" + "log" + "sync" + + "github.com/nats-io/nats.go/jetstream" + + "mesh-delivery-checker/internal/check" +) + +// holdWatch holds the checker back while a delivery builds (ADR 0484 clause 12): it watches the controller's bucket of the +// delivery plans in their build phase and reads it as empty where it is absent (issue 1652 makes it) or cannot be read, +// saying so once. The bucket may come into being after the checker started and the checker holds no timer (ADR 0437 §8), so +// each time a check is about to start and the bucket is not watched yet, the checker looks for it. +type holdWatch struct { + ctx context.Context + js bucketOpener + hold check.PlanHold + changed func(ctx context.Context) // the last plan ended: the checks that waited may start + logger *log.Logger + errs chan error + mu sync.Mutex + watched bool + said bool +} + +func newHoldWatch(ctx context.Context, js bucketOpener, changed func(context.Context), logger *log.Logger) *holdWatch { + return &holdWatch{ctx: ctx, js: js, changed: changed, logger: logger, errs: make(chan error, 1)} +} + +// Held says whether a delivery builds now. +func (w *holdWatch) Held() bool { + w.ensure() + return w.hold.Held() +} + +func (w *holdWatch) say(format string, args ...any) { + if !w.said { + w.said = true + w.logger.Printf(format, args...) + } +} + +// ensure starts the watch if the bucket exists and is not watched. +func (w *holdWatch) ensure() { + w.mu.Lock() + defer w.mu.Unlock() + if w.watched { + return + } + kv, err := w.js.KeyValue(w.ctx, check.PlansBuildingBucket) + switch { + case errors.Is(err, jetstream.ErrBucketNotFound): + w.say("the bucket %s does not exist yet: it is read as empty, so no check is held back", check.PlansBuildingBucket) + return + case err != nil: + w.say("the bucket %s cannot be read, so it is read as empty: %v", check.PlansBuildingBucket, err) + return + } + watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.hold.Put(e.Key(), !opOf(e)) }) + if err != nil { + w.say("the bucket %s cannot be watched, so it is read as empty: %v", check.PlansBuildingBucket, err) + return + } + w.watched, w.said = true, false + go func() { + w.errs <- follow(w.ctx, watcher, func(e jetstream.KeyValueEntry) { + if w.hold.Put(e.Key(), !opOf(e)) && !w.hold.Held() { + w.changed(w.ctx) + } + }) + }() +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go index 977b02d..c446fde 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go @@ -498,3 +498,31 @@ func TestACheckLeftRunningIsTakenOverWhenTheCheckerStarts(t *testing.T) { // N6/ t.Errorf("verdict = %+v", v) } } + +// TestNoCheckStartsWhileADeliveryBuildsAndOneStartsWhenItEnds: the controller's bucket of the plans in their build phase +// holds a plan's key; the candidate waits, no step is called; the key deleted, the check starts and passes (ADR 0484 clause 12). +func TestNoCheckStartsWhileADeliveryBuildsAndOneStartsWhenItEnds(t *testing.T) { // N8/AC1 + b := startBed(t) + plans, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: "mesh-controller_plans-building", History: 1}) + if err != nil { + t.Fatal(err) + } + if _, err := plans.Put(b.ctx, "plan-1", []byte(`{"plan":"plan-1","since":"2026-10-21T19:00:00Z","seen":1}`)); err != nil { + t.Fatal(err) + } + key := b.candidate(t, "feat/x", pull(repo, 7, head1)) + b.start(t, nil) + time.Sleep(1500 * time.Millisecond) // a wait for nothing to happen: there is no event to wait on + if _, err := b.buckets[delivery.CheckerVerdictsBucket].Get(b.ctx, key); err == nil { + t.Fatal("a check started while a delivery builds") + } + if n := b.fake.count("mesh-build-go.build_go_fetch"); n != 0 { + t.Fatalf("%d fetches while a delivery builds", n) + } + if err := plans.Delete(b.ctx, "plan-1"); err != nil { + t.Fatal(err) + } + if v := b.verdict(t, key, ended); v.State != delivery.VerdictPass { + t.Errorf("verdict = %+v", v) + } +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index 8b03ec9..0d1186d 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -198,6 +198,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Link: link, Logf: logger.Printf, } + holds := newHoldWatch(ctx, js, ch.Reconsider, logger) + ch.Hold = holds.Held announce := func(ctx context.Context, event, key string) { body, _ := json.Marshal(map[string]string{"key": key}) if _, err := js.Publish(ctx, eventSubject(event), body); err != nil { @@ -333,5 +335,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err case err := <-watch.errs: return err + case err := <-holds.errs: + return err } } diff --git a/modules/mesh-delivery-checker/internal/check/hold.go b/modules/mesh-delivery-checker/internal/check/hold.go new file mode 100644 index 0000000..eb8984b --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/hold.go @@ -0,0 +1,40 @@ +package check + +import "sync" + +// PlansBuildingBucket is the controller's bucket of the delivery plans in their build phase, one key for each plan, written +// by the controller alone (issue 1652): a plan key there means a delivery builds on the build nodes, and no new check starts +// meanwhile (ADR 0484 clause 12, ADR 0483 clause 6). The bucket may be absent: the controller's issue 1652 is delivered +// before R4c-5b, and if it is not, the bucket is read as empty. +const PlansBuildingBucket = "mesh-controller_plans-building" + +// PlanHold holds which delivery plans build now, from the controller's bucket (a watch puts the changes in). Safe for use +// from the watch and from a check at once. +type PlanHold struct { + mu sync.Mutex + plans map[string]bool +} + +// Put takes a change of the bucket in: a put of a plan's key, or its delete. It answers whether a delivery builds now +// differently than before, so a watch wakes the checker when the last plan ended. +func (h *PlanHold) Put(key string, deleted bool) (changed bool) { + h.mu.Lock() + defer h.mu.Unlock() + before := len(h.plans) > 0 + if deleted { + delete(h.plans, key) + } else { + if h.plans == nil { + h.plans = map[string]bool{} + } + h.plans[key] = true + } + return before != (len(h.plans) > 0) +} + +// Held says whether any delivery plan builds now. +func (h *PlanHold) Held() bool { + h.mu.Lock() + defer h.mu.Unlock() + return len(h.plans) > 0 +} diff --git a/modules/mesh-delivery-checker/internal/check/hold_test.go b/modules/mesh-delivery-checker/internal/check/hold_test.go new file mode 100644 index 0000000..38cc8bc --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/hold_test.go @@ -0,0 +1,60 @@ +package check + +import ( + "context" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +func TestADeliveryThatBuildsHoldsTheChecksAndTheEndOfTheLastPlanSaysSo(t *testing.T) { // N8/AC1 + var h PlanHold + if h.Held() { + t.Fatal("an empty bucket holds") + } + if !h.Put("plan-1", false) || !h.Held() { + t.Error("a first plan did not hold") + } + if h.Put("plan-2", false) { + t.Error("a second plan changed whether a delivery builds") + } + if h.Put("plan-1", true) || !h.Held() { + t.Error("one of two plans ending released the hold") + } + if !h.Put("plan-2", true) || h.Held() { + t.Error("the last plan ending did not release the hold") + } + if h.Put("plan-3", true) { + t.Error("the delete of a plan never put changed the hold") + } +} + +func TestARunningCheckIsNotStoppedWhenADeliveryStartsToBuild(t *testing.T) { // ADR 0484 clause 12, ADR 0483 clause 4 + ctx := context.Background() + var h PlanHold + j := &judgeFunc{f: func(_ context.Context, _ *Case, call int) (Outcome, error) { + if call == 1 { + h.Put("plan-1", false) // a delivery starts while this check runs + } + return Outcome{Verdict: passing()}, nil + }} + r := newRig(t, j) + r.ch.Hold = h.Held + r.ch.Candidate(ctx, "feat/x", candidateOf(t, "feat/x", pullOf(repoA, 7, head1))) + r.ch.Wait() + if v := r.verdict(t, keyOf(t, "feat/x", candidateOf(t, "feat/x", pullOf(repoA, 7, head1)))); v.State != delivery.VerdictPass { + t.Errorf("verdict = %+v, want the running check to end as it was", v) + } + // A new candidate waits while the plan builds. + r.ch.Candidate(ctx, "feat/y", candidateOf(t, "feat/y", pullOf(repoA, 8, head2))) + r.ch.Wait() + if j.count() != 1 { + t.Errorf("judged %d times while a delivery builds, want only the one already running", j.count()) + } + h.Put("plan-1", true) + r.ch.Reconsider(ctx) + r.ch.Wait() + if j.count() != 2 { + t.Errorf("judged %d times after the delivery ended, want the waiting candidate too", j.count()) + } +} From 66908d9bbc1f20a188ae9a08ccba00f8fbdeee03 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 21:28:38 +0200 Subject: [PATCH 11/19] The builder builds the fingerprints a candidate carries, once each, from the head merged onto the trunk (issue 1684) --- .../mesh-delivery-builder/integration_test.go | 72 ++++++++++-- .../cmd/mesh-delivery-builder/main.go | 19 +++- .../internal/build/candidates.go | 104 ++++++++++++++++++ .../internal/build/candidates_test.go | 96 ++++++++++++++++ .../internal/build/run.go | 40 ++++++- .../mesh-delivery-builder/manifest_test.go | 2 +- modules/mesh-delivery-builder/module.json | 1 + 7 files changed, 316 insertions(+), 18 deletions(-) create mode 100644 modules/mesh-delivery-builder/internal/build/candidates.go create mode 100644 modules/mesh-delivery-builder/internal/build/candidates_test.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index 305ed79..926260a 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -56,6 +56,7 @@ type fakeBuildModules struct { failBuild atomic.Bool mu sync.Mutex calls map[string]int // by `.` + fetches []map[string]any } func (f *fakeBuildModules) count(tool string) int { @@ -83,6 +84,11 @@ func (f *fakeBuildModules) serve(t *testing.T, nc *nats.Conn, module string) { f.mu.Unlock() var args map[string]any _ = json.Unmarshal(m.Data, &args) + if strings.HasSuffix(tool, "_fetch") { + f.mu.Lock() + f.fetches = append(f.fetches, args) + f.mu.Unlock() + } id := fmt.Sprintf("s%d", f.n.Add(1)) key, _ := delivery.StepKey(node, id) put := func(s delivery.Step) { @@ -119,16 +125,17 @@ func (f *fakeBuildModules) serve(t *testing.T, nc *nats.Conn, module string) { } type bed struct { - ctx context.Context - js jetstream.JetStream - modules jetstream.KeyValue - builds jetstream.KeyValue - tools map[string]jetstream.KeyValue - presence jetstream.KeyValue - have jetstream.KeyValue - fake *fakeBuildModules - srvURL string - dir string + ctx context.Context + js jetstream.JetStream + modules jetstream.KeyValue + candidates jetstream.KeyValue + builds jetstream.KeyValue + tools map[string]jetstream.KeyValue + presence jetstream.KeyValue + have jetstream.KeyValue + fake *fakeBuildModules + srvURL string + dir string } // startBed starts the bus, the buckets the controller makes from the builder's and the build modules' manifests, the @@ -193,6 +200,7 @@ func startBed(t *testing.T) *bed { return kv } b.modules = mk(delivery.ModulesBucket, delivery.ModulesHistory) + b.candidates = mk(delivery.CandidatesBucket, delivery.CandidatesHistory) mk(delivery.ManifestsBucket, delivery.ManifestsHistory) b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory) presence := mk(delivery.PresenceBucket, delivery.PresenceHistory) @@ -522,3 +530,47 @@ func TestAKeyLeftBuildingForAModuleThatIsGoneIsClosedAtStart(t *testing.T) { t.Errorf("build = %+v, want it closed naming who left it", v) } } + +// TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk: the trunk's record of netcheck carries a fingerprint of its +// own; a candidate the indexer wrote gives netcheck the fingerprint fp at a pull request's head; the program writes +// builds.netcheck. done, its fetch given the head, the ref that carries it and the trunk commit (issue 1684). +func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1 + b := startBed(t) + other := "src2:" + strings.Repeat("e", 64) + digest := b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck") + data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: commit, FP: "src1:" + strings.Repeat("f", 64), FP2: other, + Manifest: digest, Path: "modules/netcheck", Toolchains: map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}}}) + if err != nil { + t.Fatal(err) + } + if _, err := b.modules.Put(b.ctx, "netcheck", data); err != nil { + t.Fatal(err) + } + const head = "1111111111111111111111111111111111111111" + pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 7, Head: head}, Modules: []string{"netcheck"}, + Fps: map[string]delivery.CandidateModule{"netcheck": {FP2: fp, Manifest: digest}}}} + hash, _ := delivery.HeadsHash([]delivery.Head{pulls[0].Head}) + cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)}) + if err != nil { + t.Fatal(err) + } + if _, err := b.candidates.Put(b.ctx, "feat%2Fx", cand); err != nil { + t.Fatal(err) + } + b.start(t, "n1", nil) + v := b.await(t, "netcheck", isState(delivery.BuildDone)) + if len(v.Artifacts) != 2 { + t.Fatalf("build = %+v", v) + } + b.fake.mu.Lock() + defer b.fake.mu.Unlock() + found := false + for _, f := range b.fake.fetches { + if f["commit"] == head && f["ref"] == "refs/pull/7/head" && f["base"] == commit { + found = true + } + } + if !found { + t.Errorf("fetches = %v, want one of the candidate's head with its ref and the trunk commit", b.fake.fetches) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index b5d2d34..f2b61e6 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -141,6 +141,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } + candidates, err := bucket(delivery.CandidatesBucket) + if err != nil { + return err + } // Calls to the build modules go out as this module's own account, naming the machine (mesh-sdk process.CallOn); the // buckets are read and written on the connection above. account, err := process.ReadAccount(path) @@ -256,9 +260,22 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := b.Sweep(ctx); err != nil { return err } + // The fingerprints the indexer gives the candidates' modules are built too (issue 1684), once the trunk's modules are known. + eachCandidate := func(e jetstream.KeyValueEntry) { + if opOf(e) { + b.Candidate(ctx, e.Key(), e.Value()) + } else { + b.ForgetCandidate(e.Key()) + } + } + cw, err := settle(ctx, candidates, eachCandidate) + if err != nil { + return err + } logger.Printf("started on %s", cred.Node) - errs := make(chan error, 1) + errs := make(chan error, 2) go func() { errs <- follow(ctx, w, each) }() + go func() { errs <- follow(ctx, cw, eachCandidate) }() select { case <-ctx.Done(): b.Wait() diff --git a/modules/mesh-delivery-builder/internal/build/candidates.go b/modules/mesh-delivery-builder/internal/build/candidates.go new file mode 100644 index 0000000..93e7183 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/candidates.go @@ -0,0 +1,104 @@ +package build + +import ( + "context" + "fmt" + "sort" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" +) + +// How a candidate's fingerprint is built (novox/hq ADR 0484 clause 3, issue 1684): the indexer writes, for each module a +// pull request touches, the fingerprint of the new form it has at the head merged onto the trunk (candidates.), and +// the builder builds each, once, as a key of its own (builds..) that the checker's line reads. The +// build is the trunk module's: the same build modules, the same nodes and capacity; what differs is where the source comes +// from: the fetch step is given the pull request's head, the ref that carries it and the trunk commit to merge it onto, as +// the checker's fetch is, and builds the merge's module directory. + +// via is how a build's fetch step reaches its source when it is a candidate's: the ref that carries the head and the trunk +// commit the head is merged onto. +type via struct{ Ref, Base string } + +// Candidate takes one value of the indexer's candidates bucket in (the key is the group) and builds the fingerprints it +// carries. A value that is not a candidate is said and left. +func (b *Builder) Candidate(ctx context.Context, key string, value []byte) { + b.init() + c, err := delivery.DecodeCandidate(value) + if err != nil { + b.logf("candidates.%s holds what is not a candidate, left: %v", key, err) + return + } + b.mu.Lock() + b.cands[key] = c + b.mu.Unlock() + b.considerCandidate(ctx, c) +} + +// ForgetCandidate takes a candidate out (its key was deleted). +func (b *Builder) ForgetCandidate(key string) { + b.init() + b.mu.Lock() + delete(b.cands, key) + b.mu.Unlock() +} + +func (b *Builder) considerCandidate(ctx context.Context, c delivery.Candidate) { + for _, p := range c.Pulls { + names := make([]string, 0, len(p.Fps)) + for name := range p.Fps { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + cm := p.Fps[name] + if cm.FP2 == "" || cm.Manifest == "" { + continue // no fingerprint of the new form for it (a reason in none2) or no manifest to build by + } + b.mu.Lock() + trunk, known := b.known[name] + b.mu.Unlock() + if !known || trunk.Path == "" { + b.logf("candidates.%s: the module %s has no record of the trunk yet, so its fingerprint %s is not built", c.Group, name, cm.FP2) + continue + } + key, err := delivery.BuilderBuildKey(name, cm.FP2) + if err != nil { + b.logf("candidates.%s: %v", c.Group, err) + continue + } + m := delivery.Module{Repository: p.Repository, Commit: p.Head.Head, FP2: cm.FP2, Manifest: cm.Manifest, Path: trunk.Path, + Toolchains: b.agreedFor(ctx, cm.Manifest)} + b.mu.Lock() + if _, onTrunk := b.trunkFP[key]; !onTrunk { + b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: trunk.Commit} + } + b.mu.Unlock() + b.consider(ctx, name, m) + } + } +} + +// agreedFor are the toolchains the build nodes agree on now, for the languages of the manifest's plan: the toolchains the +// candidate's fingerprint was computed with, as the indexer recomputes it whenever they change. A language with none agreed +// is left out, and the build then waits for it as the trunk's does. +func (b *Builder) agreedFor(ctx context.Context, manifest string) map[string]delivery.ToolchainUsed { + plan, err, _ := b.planOf(ctx, manifest) + if err != nil { + return nil + } + out := map[string]delivery.ToolchainUsed{} + for _, bm := range plan.Modules() { + if !plan.NeedsToolchain(bm) { + continue + } + language := buildrun.LanguageOf[bm] + if tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language)); none == "" { + out[language] = delivery.ToolchainUsed{Version: tc.Version, Recipe: tc.Recipe} + } + } + if len(out) == 0 { + return nil + } + return out +} diff --git a/modules/mesh-delivery-builder/internal/build/candidates_test.go b/modules/mesh-delivery-builder/internal/build/candidates_test.go new file mode 100644 index 0000000..acc5976 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/candidates_test.go @@ -0,0 +1,96 @@ +package build + +import ( + "context" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// trunkModule writes the trunk's record of the module m at a fingerprint of its own, so that the candidate's fingerprint (fp1) +// is one the trunk does not carry. +func trunkModule(t *testing.T, r *rig) { + t.Helper() + data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP: "src1:" + hex64('f'), + FP2: "src2:" + hex64('e'), Manifest: mdigest('a'), Path: "modules/m", Toolchains: agreedToolchains}) + if err != nil { + t.Fatal(err) + } + r.b.Module(context.Background(), "m", data) + r.b.Wait() + r.bm.mu.Lock() + r.bm.calls = nil + r.bm.mu.Unlock() +} + +func candidateValue(t *testing.T, group string, number int, head string) []byte { + t.Helper() + pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: number, Head: head}, Modules: []string{"m"}, + Fps: map[string]delivery.CandidateModule{"m": {FP2: fp1, Manifest: mdigest('a')}}}} + hash, err := delivery.HeadsHash([]delivery.Head{pulls[0].Head}) + if err != nil { + t.Fatal(err) + } + data, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: group, HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)}) + if err != nil { + t.Fatal(err) + } + return data +} + +const ( + headX = "1111111111111111111111111111111111111111" + headY = "2222222222222222222222222222222222222222" +) + +func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1, N7/AC2 (issue 1684) + r := newRig(t, goBundle) + trunkModule(t, r) + r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) + r.b.Wait() + v := valueOf(t, r.st, r.key) + if v.State != delivery.BuildDone || len(v.Artifacts) != 1 { + t.Fatalf("build = %+v, want the candidate's fingerprint built", v) + } + fetches := r.bm.callsOf("build_go_fetch") + if len(fetches) != 2 { // one on each of two nodes + t.Fatalf("%d fetches, want one per node", len(fetches)) + } + f := fetches[0].body.(map[string]any) + if f["commit"] != headX || f["ref"] != "refs/pull/7/head" || f["base"] != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" || f["repository"] != "novox/mesh-catalog" { + t.Errorf("fetch args = %+v, want the head, its ref and the trunk commit to merge onto", f) + } + if b := r.bm.callsOf("build_go_build")[0].body.(map[string]any); b["module"] != "modules/m" { + t.Errorf("build args = %+v", b) + } + // Two heads carrying one fingerprint are one build. + r.b.Candidate(context.Background(), "feat%2Fy", candidateValue(t, "feat/y", 8, headY)) + r.b.Wait() + if n := len(r.bm.callsOf("build_go_fetch")); n != 2 { + t.Errorf("%d fetches after a second head with the same fingerprint, want no more", n) + } +} + +func TestACandidateWhoseModuleHasNoTrunkRecordIsNotBuilt(t *testing.T) { + r := newRig(t, goBundle) + r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) + r.b.Wait() + if _, ok := r.st.vals[r.key]; ok { + t.Error("a build key was written for a module the builder knows no directory of") + } +} + +func TestAFingerprintTheTrunkCarriesIsBuiltFromTheTrunksCommit(t *testing.T) { // the trunk's source wins + r := newRig(t, goBundle) + r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) + r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // the trunk now carries fp1 + r.b.Wait() + for _, c := range r.bm.callsOf("build_go_fetch") { + if f := c.body.(map[string]any); f["ref"] != nil || f["base"] != nil { + t.Errorf("fetch args = %+v: a trunk fingerprint is built from the trunk's commit alone", f) + } + } + if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone { + t.Errorf("build = %+v", v) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 66877e3..495027f 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -50,11 +50,14 @@ type Builder struct { sem chan struct{} wg sync.WaitGroup mu sync.Mutex - known map[string]delivery.Module // by modules key - inflight map[string]bool // by build key - again map[string]delivery.Module // by build key: the newest value that arrived while the key was being built - plans map[string]planned // by manifest digest - tried map[string]bool // by build key: looked at since this process started + known map[string]delivery.Module // by modules key + inflight map[string]bool // by build key + again map[string]delivery.Module // by build key: the newest value that arrived while the key was being built + plans map[string]planned // by manifest digest + cands map[string]delivery.Candidate // by candidates key (candidates.go) + via map[string]via // by build key: a candidate's fetch (candidates.go) + trunkFP map[string]bool // by build key: a trunk module value carries this fingerprint + tried map[string]bool // by build key: looked at since this process started // epoch counts the changes of the nodes (online, healthy, a toolchain) seen since this process started; a done key // whose second-node build failed is held back until it moves (issue 1664). epoch uint64 @@ -73,6 +76,7 @@ func (b *Builder) init() { limit = 4 } b.sem = make(chan struct{}, limit) + b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{} b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{} b.secondAt = map[string]uint64{} if b.Places == nil { @@ -101,6 +105,11 @@ func (b *Builder) Module(ctx context.Context, key string, value []byte) { } b.mu.Lock() b.known[key] = m + if bk, err := delivery.BuilderBuildKey(key, m.FP2); err == nil && m.FP2 != "" { + // A fingerprint the trunk carries is built from the trunk's commit, whoever asked for it first. + b.trunkFP[bk] = true + delete(b.via, bk) + } b.mu.Unlock() b.consider(ctx, key, m) } @@ -182,6 +191,16 @@ func (b *Builder) Reconsider(ctx context.Context) { for _, k := range sortedKeys(known) { b.consider(ctx, k, known[k]) } + b.mu.Lock() + cands := make([]delivery.Candidate, 0, len(b.cands)) + for _, c := range b.cands { + cands = append(cands, c) + } + b.mu.Unlock() + sort.Slice(cands, func(i, j int) bool { return cands[i].Group < cands[j].Group }) + for _, c := range cands { + b.considerCandidate(ctx, c) + } } func sortedKeys(m map[string]delivery.Module) []string { @@ -490,7 +509,16 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del return } defer release() - raw, err := b.step(ctx, l.module, l.node, "fetch", map[string]any{"repository": repository, "commit": m.Commit}) + fetchArgs := map[string]any{"repository": repository, "commit": m.Commit} + if key, err := delivery.BuilderBuildKey(module, m.FP2); err == nil { + b.mu.Lock() + v, candidate := b.via[key] + b.mu.Unlock() + if candidate { + fetchArgs["ref"], fetchArgs["base"] = v.Ref, v.Base + } + } + raw, err := b.step(ctx, l.module, l.node, "fetch", fetchArgs) if err != nil { l.err = err return diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 66ac3cf..70f4ca4 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -87,7 +87,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 t.Errorf("state = %+v, want %+v", m.State, want) } if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", - "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) { + "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } if want := []string{"mesh-build-go.build_go_build", "mesh-build-go.build_go_fetch", "mesh-build-node.build_node_build", "mesh-build-node.build_node_fetch"}; !reflect.DeepEqual(sorted(m.Invokes), want) { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 292c9ae..cb976f3 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -14,6 +14,7 @@ "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", + "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", From 4b063252e7f2f4fbabaefb4f0447cc86f9121d5b Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 21:29:13 +0200 Subject: [PATCH 12/19] The checker's line says whether each candidate module's build is built, failed or waiting, read from the builder's keys (issue 1684) --- .../mesh-delivery-checker/integration_test.go | 27 +++++++- .../cmd/mesh-delivery-checker/judge.go | 19 +++++- .../cmd/mesh-delivery-checker/main.go | 13 +++- .../internal/check/builds.go | 66 +++++++++++++++++++ .../internal/check/examiner_test.go | 30 +++++++++ .../internal/check/judge.go | 16 ++++- 6 files changed, 165 insertions(+), 6 deletions(-) create mode 100644 modules/mesh-delivery-checker/internal/check/builds.go diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go index c446fde..80d1623 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go @@ -220,7 +220,7 @@ func startBed(t *testing.T) *bed { for name, history := range map[string]int{delivery.CheckerVerdictsBucket: delivery.CheckerVerdictsHistory, delivery.CheckerKeptBucket: delivery.CheckerKeptHistory, delivery.CheckerOwedBucket: delivery.CheckerOwedHistory, delivery.PreviewsBucket: delivery.PreviewsHistory, delivery.CandidatesBucket: delivery.CandidatesHistory, delivery.ModulesBucket: delivery.ModulesHistory, delivery.ManifestsBucket: delivery.ManifestsHistory, delivery.SourceBucket: delivery.SourceHistory, - delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory} { + delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory, delivery.BuilderBuildsBucket: delivery.BuilderBuildsHistory} { kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: uint8(history)}) if err != nil { t.Fatal(err) @@ -526,3 +526,28 @@ func TestNoCheckStartsWhileADeliveryBuildsAndOneStartsWhenItEnds(t *testing.T) { t.Errorf("verdict = %+v", v) } } + +// TestTheLineSaysTheStateOfTheBuildsOfTheCandidatesModules: the builder wrote builds.avahi. done for the fingerprint the +// candidate carries for avahi; the status's line says avahi is built (ADR 0484 clause 7). +func TestTheLineSaysTheStateOfTheBuildsOfTheCandidatesModules(t *testing.T) { // N7/AC1 + b := startBed(t) + fp := "src2:" + strings.Repeat("a", 64) + buildKey, _ := delivery.BuilderBuildKey("avahi", fp) + done, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "n1.mesh-delivery-builder", Manifest: strings.Repeat("0a", 32), + Artifacts: []delivery.BuiltArtifact{{Name: "avahi", Kind: "bundle", Digest: "sha256:" + strings.Repeat("a", 64)}}}) + if err != nil { + t.Fatal(err) + } + if _, err := b.buckets[delivery.BuilderBuildsBucket].Put(b.ctx, buildKey, done); err != nil { + t.Fatal(err) + } + p := pull(repo, 7, head1) + p.Modules = []string{"avahi"} + p.Fps = map[string]delivery.CandidateModule{"avahi": {FP2: fp, Manifest: strings.Repeat("0a", 32)}} + key := b.candidate(t, "feat/x", p) + b.start(t, nil) + b.verdict(t, key, ended) + if o := b.owed(t, 7, head1, "success"); !strings.Contains(o.Text, "builds: avahi built") { + t.Errorf("owed = %+v, want the builds in the line", o) + } +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go index 46a0100..61fe261 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go @@ -24,12 +24,13 @@ type judgeDeps struct { previews check.KV repos *repoModules announce check.Announcer + builds check.BuildStates logf func(format string, args ...any) } func newJudge(d judgeDeps) check.Judge { return &check.Examiner{Steps: d.runner, Toolchains: d.tools, Presence: d.presence, Health: d.have, Places: d.places, - Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Announce: d.announce, Trunks: d.trunks, Logf: d.logf} + Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Builds: d.builds, Announce: d.announce, Trunks: d.trunks, Logf: d.logf} } // manifestKV reads a manifest by its digest. @@ -110,3 +111,19 @@ func (r *repoModules) BuildModules(repository string) []string { sort.Strings(out) return out } + +// buildReader reads the builder's build of a module at a fingerprint (mesh-delivery-builder_builds). +type buildReader struct{ kv kvStore } + +func (r buildReader) Build(ctx context.Context, module, fp2 string) (delivery.BuilderBuild, bool, error) { + key, err := delivery.BuilderBuildKey(module, fp2) + if err != nil { + return delivery.BuilderBuild{}, false, err + } + value, _, found, err := r.kv.Get(ctx, key) + if err != nil || !found { + return delivery.BuilderBuild{}, false, err + } + b, err := delivery.DecodeBuilderBuild(value) + return b, err == nil, err +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index 0d1186d..7f01c48 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -198,6 +198,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Link: link, Logf: logger.Printf, } + // The builder's builds say how the candidates' fingerprints stand (issue 1684); where its bucket is not there yet the line + // says nothing of the builds, until the checker starts again. + var builds check.BuildStates + switch kv, err := js.KeyValue(ctx, delivery.BuilderBuildsBucket); { + case errors.Is(err, jetstream.ErrBucketNotFound): + logger.Printf("the bucket %s does not exist yet: the line says nothing of the builds until the checker starts again", delivery.BuilderBuildsBucket) + case err != nil: + return fmt.Errorf("the bucket %s: %w", delivery.BuilderBuildsBucket, err) + default: + builds = buildReader{kvStore{kv}} + } holds := newHoldWatch(ctx, js, ch.Reconsider, logger) ch.Hold = holds.Held announce := func(ctx context.Context, event, key string) { @@ -210,7 +221,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro ch.Judge = newJudge(judgeDeps{ runner: runner, tools: tools, presence: presence, have: have, places: places, trunks: trunks, kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]}, - repos: repos, logf: logger.Printf, + repos: repos, builds: builds, logf: logger.Printf, announce: announce, }) ch.Announce = announce diff --git a/modules/mesh-delivery-checker/internal/check/builds.go b/modules/mesh-delivery-checker/internal/check/builds.go new file mode 100644 index 0000000..f70115f --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/builds.go @@ -0,0 +1,66 @@ +package check + +import ( + "context" + "sort" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The states of a build the status line says (ADR 0484 clause 7). +const ( + BuildBuilt = "built" + BuildFailed = "failed" + BuildWaiting = "waiting" +) + +// BuildStates reads the builder's build of a module at a fingerprint of the new form (builds..): the +// value and whether the key exists. +type BuildStates interface { + Build(ctx context.Context, module, fp2 string) (delivery.BuilderBuild, bool, error) +} + +// NoteOf is the state of a build as the line says it: done is built, failed is failed, and one queued, building or not +// begun is waiting. +func NoteOf(b delivery.BuilderBuild, found bool) string { + switch { + case !found: + return BuildWaiting + case b.State == delivery.BuildDone: + return BuildBuilt + case b.State == delivery.BuildFailed: + return BuildFailed + } + return BuildWaiting +} + +// buildNotes are the state of each build the case's modules carry a fingerprint of, by module, sorted. A module two pull +// requests give is said once. +func (j *Examiner) buildNotes(ctx context.Context, c *Case) []BuildNote { + if j.Builds == nil { + return nil + } + fps := map[string]string{} + for _, p := range c.Pulls { + for module, cm := range p.Fps { + if cm.FP2 != "" { + fps[module] = cm.FP2 + } + } + } + modules := make([]string, 0, len(fps)) + for m := range fps { + modules = append(modules, m) + } + sort.Strings(modules) + var notes []BuildNote + for _, m := range modules { + b, found, err := j.Builds.Build(ctx, m, fps[m]) + if err != nil { + j.logf("the build of %s at %s cannot be read: %v", m, fps[m], err) + found = false + } + notes = append(notes, BuildNote{Module: m, State: NoteOf(b, found)}) + } + return notes +} diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go index f5b239b..ec6c3aa 100644 --- a/modules/mesh-delivery-checker/internal/check/examiner_test.go +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -522,3 +522,33 @@ func TestTheIndexerIsToldOfAPreviewOnceWhenItIsCreated(t *testing.T) { // 1445/A t.Errorf("told %v, want once of %s", told, key) } } + +type fakeBuilds map[string]delivery.BuilderBuild + +func (f fakeBuilds) Build(_ context.Context, module, fp2 string) (delivery.BuilderBuild, bool, error) { + b, ok := f[module+" "+fp2] + return b, ok, nil +} + +func TestTheOutcomeSaysIfEachCandidatesModuleIsBuiltFailedOrWaiting(t *testing.T) { // N7/AC1 (the checker reads the builder's keys) + b := newBench(t) + fpA := "src2:" + strings.Repeat("a", 64) + fpB := "src2:" + strings.Repeat("b", 64) + fpC := "src2:" + strings.Repeat("c", 64) + b.ex.Builds = fakeBuilds{ + "avahi " + fpA: {State: delivery.BuildDone}, + "mesh-tools " + fpB: {State: delivery.BuildFailed, Error: "boom"}, + // netcheck has no key yet + } + p := pullOf(repoA, 7, head1) + p.Modules = []string{"avahi", "mesh-tools", "netcheck", "nofp"} + p.Fps = map[string]delivery.CandidateModule{"avahi": {FP2: fpA}, "mesh-tools": {FP2: fpB}, "netcheck": {FP2: fpC}, "nofp": {None2: "toolchain not yet reported"}} + out := b.judge(t, caseOf(t, p)) + var got []string + for _, n := range out.Builds { + got = append(got, n.Module+" "+n.State) + } + if strings.Join(got, ", ") != "avahi built, mesh-tools failed, netcheck waiting" { + t.Errorf("builds = %v, want one note for each module that has a fingerprint, sorted", got) + } +} diff --git a/modules/mesh-delivery-checker/internal/check/judge.go b/modules/mesh-delivery-checker/internal/check/judge.go index 1785c5d..21dc121 100644 --- a/modules/mesh-delivery-checker/internal/check/judge.go +++ b/modules/mesh-delivery-checker/internal/check/judge.go @@ -50,6 +50,9 @@ type Examiner struct { // Previews is the previews bucket; nil writes none. Previews KV Modules BuildModules + // Builds says how the build of a fingerprint the candidate carries stands (the builder's builds bucket); nil says nothing + // of the builds. + Builds BuildStates // Announce says a preview was written under the key (EventPreviewWritten); nil says nothing. Announce Announcer // Trunks are the trunk heads of every repository, for the siblings a repository's tests read. @@ -115,18 +118,25 @@ type state struct { func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) { st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}} if v := st.fetchAll(ctx); v != nil { - return st.outcome(*v), nil + return st.finish(ctx, *v), nil } for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} { for _, priority := range []bool{true, false} { for _, t := range st.targets { if v := st.runTarget(ctx, t, level, priority); v != nil { - return st.outcome(*v), nil + return st.finish(ctx, *v), nil } } } } - return st.outcome(delivery.Verdict{State: delivery.VerdictPass}), nil + return st.finish(ctx, delivery.Verdict{State: delivery.VerdictPass}), nil +} + +// finish is the outcome of the verdict with the state of the builds the candidate's modules carry. +func (st *state) finish(ctx context.Context, v delivery.Verdict) Outcome { + out := st.outcome(v) + out.Builds = st.j.buildNotes(ctx, st.c) + return out } // outcome fills the verdict's levels, times, machines and composition from what ran. From fecabfca8b2c40979823edeb86de8b80c217d61d Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 21:30:24 +0200 Subject: [PATCH 13/19] Fit the status line in the forge's 140 characters by giving the times and counts in their compact form before dropping them (issue 1682) --- .../mesh-delivery-checker/integration_test.go | 27 +------------ .../internal/check/line.go | 40 ++++++++++++++++--- .../internal/check/line_test.go | 3 +- 3 files changed, 38 insertions(+), 32 deletions(-) diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go index 80d1623..c446fde 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go @@ -220,7 +220,7 @@ func startBed(t *testing.T) *bed { for name, history := range map[string]int{delivery.CheckerVerdictsBucket: delivery.CheckerVerdictsHistory, delivery.CheckerKeptBucket: delivery.CheckerKeptHistory, delivery.CheckerOwedBucket: delivery.CheckerOwedHistory, delivery.PreviewsBucket: delivery.PreviewsHistory, delivery.CandidatesBucket: delivery.CandidatesHistory, delivery.ModulesBucket: delivery.ModulesHistory, delivery.ManifestsBucket: delivery.ManifestsHistory, delivery.SourceBucket: delivery.SourceHistory, - delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory, delivery.BuilderBuildsBucket: delivery.BuilderBuildsHistory} { + delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory} { kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: uint8(history)}) if err != nil { t.Fatal(err) @@ -526,28 +526,3 @@ func TestNoCheckStartsWhileADeliveryBuildsAndOneStartsWhenItEnds(t *testing.T) { t.Errorf("verdict = %+v", v) } } - -// TestTheLineSaysTheStateOfTheBuildsOfTheCandidatesModules: the builder wrote builds.avahi. done for the fingerprint the -// candidate carries for avahi; the status's line says avahi is built (ADR 0484 clause 7). -func TestTheLineSaysTheStateOfTheBuildsOfTheCandidatesModules(t *testing.T) { // N7/AC1 - b := startBed(t) - fp := "src2:" + strings.Repeat("a", 64) - buildKey, _ := delivery.BuilderBuildKey("avahi", fp) - done, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "n1.mesh-delivery-builder", Manifest: strings.Repeat("0a", 32), - Artifacts: []delivery.BuiltArtifact{{Name: "avahi", Kind: "bundle", Digest: "sha256:" + strings.Repeat("a", 64)}}}) - if err != nil { - t.Fatal(err) - } - if _, err := b.buckets[delivery.BuilderBuildsBucket].Put(b.ctx, buildKey, done); err != nil { - t.Fatal(err) - } - p := pull(repo, 7, head1) - p.Modules = []string{"avahi"} - p.Fps = map[string]delivery.CandidateModule{"avahi": {FP2: fp, Manifest: strings.Repeat("0a", 32)}} - key := b.candidate(t, "feat/x", p) - b.start(t, nil) - b.verdict(t, key, ended) - if o := b.owed(t, 7, head1, "success"); !strings.Contains(o.Text, "builds: avahi built") { - t.Errorf("owed = %+v, want the builds in the line", o) - } -} diff --git a/modules/mesh-delivery-checker/internal/check/line.go b/modules/mesh-delivery-checker/internal/check/line.go index db675c2..0882cf1 100644 --- a/modules/mesh-delivery-checker/internal/check/line.go +++ b/modules/mesh-delivery-checker/internal/check/line.go @@ -50,13 +50,16 @@ func firstFailed(v delivery.Verdict) string { // pieces are the parts of a line, from the most to the least needed. type pieces struct { - head, commits, counts, times, over, builds, composition, group string + head, commits, counts, shortCounts, times, shortTimes, over, builds, composition, group string } func (p pieces) join(drop int) string { parts := []string{p.head, p.commits, p.counts} - if drop < 3 && p.times != "" { + switch { + case drop < 3 && p.times != "": parts = append(parts, p.times) + case drop == 3 && p.shortTimes != "": + parts = []string{p.head, p.commits, p.shortCounts, p.shortTimes} } if p.over != "" { parts = append(parts, p.over) @@ -128,6 +131,18 @@ func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []s } } pc.counts = strings.Join(counts, " | ") + var short []string + for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} { + if r, ok := v.Levels[level]; ok { + name := map[string]string{delivery.LevelUnit: "unit", delivery.LevelIntegration: "int"}[level] + w := fmt.Sprintf("%s %d run %d kept", name, r.Run, r.Kept) + if r.Skipped > 0 { + w += fmt.Sprintf(" %d skipped", r.Skipped) + } + short = append(short, w) + } + } + pc.shortCounts = strings.Join(short, " | ") if v.State != delivery.VerdictRunning { t := v.Took var times []string @@ -142,6 +157,21 @@ func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []s if len(times) > 0 { pc.times = strings.Join(times, ", ") } + // The compact form for a line that is too long: the parts that dominate, the integration level's setup and run as one. + short = short[:0:0] + if t.Clone > 0 { + short = append(short, "clone "+seconds(t.Clone)) + } + if t.Unit > 0 { + short = append(short, "unit "+seconds(t.Unit)) + } + switch { + case t.IntegrationSetup > 0 && t.IntegrationRun > 0: + short = append(short, "int "+seconds(t.IntegrationSetup)+"+"+seconds(t.IntegrationRun)) + case t.IntegrationRun > 0: + short = append(short, "int "+seconds(t.IntegrationRun)) + } + pc.shortTimes = strings.Join(short, " ") } if len(v.OverBound) > 0 { pc.over = "over bound: " + strings.Join(v.OverBound, ", ") @@ -171,14 +201,14 @@ func Report(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest [] // failure names the conflict, the refused step or the test before anything), the head and the trunk commit as twelve // characters each, the tests run, kept and skipped per level, the time of each part, a level over its bound with its slowest // tests, the builds of the touched modules, and that the composition was not yet run. It is at most MaxLine characters: the -// builds, then the group, then the times drop in that order, and the line is cut at last. +// builds, then the group drop first, then the times are given in their compact form, then dropped, and the line is cut at last. func Line(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string) string { pc := parts(v, cs, p, builds, slowest) - for drop := 0; drop <= 3; drop++ { + for drop := 0; drop <= 4; drop++ { if s := pc.join(drop); utf8.RuneCountInString(s) <= MaxLine { return s } } - runes := []rune(pc.join(3)) + runes := []rune(pc.join(4)) return string(runes[:MaxLine-1]) + "…" } diff --git a/modules/mesh-delivery-checker/internal/check/line_test.go b/modules/mesh-delivery-checker/internal/check/line_test.go index ccbce03..6ea2499 100644 --- a/modules/mesh-delivery-checker/internal/check/line_test.go +++ b/modules/mesh-delivery-checker/internal/check/line_test.go @@ -35,7 +35,8 @@ func TestAPassOpensWithTheVerdictAndSaysTheCommitsTheCountsTheTimesAndThatTheCom if n := utf8.RuneCountInString(line); n > MaxLine { t.Errorf("the line is %d characters, want at most %d: %q", n, MaxLine, line) } - if !strings.HasPrefix(line, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(line, "composition not yet run") || !strings.Contains(line, "unit 12 run, 301 kept") { + if !strings.HasPrefix(line, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(line, "composition not yet run") || !strings.Contains(line, "unit 12 run 301 kept") || + !strings.Contains(line, "clone 4.1s") || !strings.Contains(line, "int 3.2s+12s") { t.Errorf("line = %q", line) } } From ee994f57b9011c0262c938c3b7092207b270c301 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 21:30:29 +0200 Subject: [PATCH 14/19] Do not pin the integration test to one form of the line --- .../cmd/mesh-delivery-checker/integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go index c446fde..5767b67 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go @@ -386,7 +386,7 @@ func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300 t.Errorf("composition %+v machines %v", v.Composition, v.Machines) } o := b.owed(t, 7, head1, "success") - if o.Context != delivery.StatusContextCheck || !strings.HasPrefix(o.Text, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(o.Text, "unit 2 run, 0 kept") { + if o.Context != delivery.StatusContextCheck || !strings.HasPrefix(o.Text, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(o.Text, "unit 2 run") { t.Errorf("owed = %+v", o) } for tool, want := range map[string]int{"mesh-build-go.build_go_fetch": 1, "mesh-build-go.build_go_digest": 2, "mesh-build-go.build_go_test": 2} { From 4cdba63b9bde6f1d948df707776eb827486d49c2 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 21:32:08 +0200 Subject: [PATCH 15/19] Read tools for the checker's verdicts, a pull request's report and the kept verdicts (issue 1686) --- .../cmd/mesh-delivery-checker-tools/main.go | 71 +++++++ .../mesh-delivery-checker-tools/main_test.go | 27 +++ .../internal/check/read.go | 193 ++++++++++++++++++ .../internal/check/read_test.go | 104 ++++++++++ .../mesh-delivery-checker/manifest_test.go | 31 ++- modules/mesh-delivery-checker/module.json | 16 ++ 6 files changed, 433 insertions(+), 9 deletions(-) create mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go create mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go create mode 100644 modules/mesh-delivery-checker/internal/check/read.go create mode 100644 modules/mesh-delivery-checker/internal/check/read_test.go diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go new file mode 100644 index 0000000..2fef5a6 --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go @@ -0,0 +1,71 @@ +// Command mesh-delivery-checker-tools serves the checker's read tools (novox/hq ADR 0484 clause 13, issue 1686): the runtime +// launches it for a call, it reads the checker's verdicts and kept states and the indexer's candidates through the checker's +// state and reads, and writes nothing. The checker is the other program: it serves no tool (ADR 0437 §4). +package main + +import ( + "encoding/json" + "fmt" + "os" + + stdio "git.novox.be/novox/mesh-sdk/go" + + "mesh-delivery-checker/internal/check" +) + +func read(s stdio.KeptState) check.StateRead { + return check.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, error) { + e, err := s.Get(key) + if err != nil || e == nil { + return nil, err + } + return e.Value, nil + }} +} + +// Tools are the read tools over the checker's states. +func Tools(r check.Reader) []stdio.Tool { + return []stdio.Tool{ + {Name: check.VerdictsReadTool, + Description: "The verdicts the checker holds, each as stored under verdicts..: its state, what failed first, the tests run, kept and skipped per level, the time of each part, the trunk heads it judged against and the composition, with the full " + + "report of the status each pull request carries (the line the forge shows is cut at 140 characters, this is not). With a group, that group's only. Read-only.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "group": map[string]any{"type": "string", "description": "the delivery group's branch name"}}}, + Run: func(args map[string]any) (any, error) { + group, _ := args["group"].(string) + vs, err := r.Verdicts(group) + return map[string]any{"verdicts": vs}, err + }}, + {Name: check.PullReadTool, + Description: "The verdict that judges one pull request and the full report of its status mesh/check; for an hq pull request, that hq is judged by " + + "merge-check.sh until issue 1296. Read-only.", + Input: map[string]any{"type": "object", "required": []string{"repository", "number"}, "properties": map[string]any{ + "repository": map[string]any{"type": "string", "description": "/"}, + "number": map[string]any{"type": "integer", "description": "the pull request's number"}}}, + Run: func(args map[string]any) (any, error) { + repository, _ := args["repository"].(string) + n, _ := args["number"].(float64) + return r.Pull(repository, int(n)) + }}, + {Name: check.KeptReadTool, + Description: "The kept verdicts of passing packages, each as stored under kept..: the package, its tests, the time they took, the toolchain " + + "and when it was kept. With a level, that level's; with a level and a digest, that one. Read-only.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "level": map[string]any{"type": "string", "enum": []string{"unit", "integration"}, "description": "the level"}, + "digest": map[string]any{"type": "string", "description": "the package's digest, with a level"}}}, + Run: func(args map[string]any) (any, error) { + level, _ := args["level"].(string) + digest, _ := args["digest"].(string) + kept, err := r.Kept(level, digest) + return map[string]any{"kept": kept}, err + }}, + } +} + +func main() { + r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates"))} + if err := stdio.Serve("", Tools(r)); err != nil { + fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err) + os.Exit(1) + } +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go new file mode 100644 index 0000000..5fec12b --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go @@ -0,0 +1,27 @@ +package main + +import ( + "testing" + + "mesh-delivery-checker/internal/check" +) + +func TestTheToolsServedAreTheThreeReadTools(t *testing.T) { + var names []string + for _, tl := range Tools(check.Reader{}) { + names = append(names, tl.Name) + if tl.Description == "" || tl.Run == nil { + t.Errorf("%s has no description or no run", tl.Name) + } + } + want := []string{check.VerdictsReadTool, check.PullReadTool, check.KeptReadTool} + if len(names) != 3 || names[0] != want[0] || names[1] != want[1] || names[2] != want[2] { + t.Errorf("tools = %v, want %v", names, want) + } + // With nothing to read from a tool says so and changes nothing. + for _, tl := range Tools(check.Reader{}) { + if _, err := tl.Run(map[string]any{"repository": "novox/x", "number": float64(1)}); err == nil { + t.Errorf("%s answered with no state to read", tl.Name) + } + } +} diff --git a/modules/mesh-delivery-checker/internal/check/read.go b/modules/mesh-delivery-checker/internal/check/read.go new file mode 100644 index 0000000..874e99b --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/read.go @@ -0,0 +1,193 @@ +package check + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The read tools (novox/hq ADR 0484 clauses 8 and 13, issue 1686): what the checker stored, as stored, with the line of the +// status it owes in full, so a check is read live without a shell. They write nothing. + +// The read tools, named as the module is: checker_ and the noun. +const ( + VerdictsReadTool = "checker_verdicts" + PullReadTool = "checker_pull" + KeptReadTool = "checker_kept" +) + +// StateRead reads one state: its keys and a key's stored value (nil for a key with none). +type StateRead struct { + Keys func() ([]string, error) + Get func(key string) (json.RawMessage, error) +} + +// Reader answers the read tools from the checker's verdicts and kept states and the indexer's candidates. +type Reader struct{ VerdictState, KeptState, CandidateState StateRead } + +// VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the +// candidate still holds those heads; without it the report is that of the verdict alone. +type VerdictRead struct { + Key string `json:"key"` + Verdict json.RawMessage `json:"verdict"` + Reports map[string]string `json:"reports,omitempty"` + Note string `json:"note,omitempty"` +} + +func (r Reader) candidate(group string) (delivery.Candidate, bool) { + key, err := delivery.CandidatesKey(group) + if err != nil || r.CandidateState.Get == nil { + return delivery.Candidate{}, false + } + raw, err := r.CandidateState.Get(key) + if err != nil || len(raw) == 0 { + return delivery.Candidate{}, false + } + c, err := delivery.DecodeCandidate(raw) + return c, err == nil +} + +func (r Reader) read(key string, raw json.RawMessage) (VerdictRead, error) { + out := VerdictRead{Key: key, Verdict: raw} + v, err := delivery.DecodeVerdict(raw) + if err != nil { + return out, fmt.Errorf("%s holds what is not a verdict: %w", key, err) + } + c, ok := r.candidate(v.Group) + if !ok || c.HeadsHash != v.Heads { + out.Note = "the candidate no longer holds these heads" + return out, nil + } + cs := CaseOf(c, v.Trunk) + out.Reports = map[string]string{} + if cs == nil { + return out, nil + } + for _, p := range cs.Pulls { + out.Reports[fmt.Sprintf("%s#%d", p.Repository, p.Number)] = Report(v, cs, p, nil, nil) + } + for _, l := range cs.Left { + out.Reports[strings.SplitN(l, ": ", 2)[0]] = strings.SplitN(l, ": ", 2)[1] + } + return out, nil +} + +// Verdicts are the verdicts the checker holds, by key; with a group only that group's. +func (r Reader) Verdicts(group string) ([]VerdictRead, error) { + if r.VerdictState.Keys == nil || r.VerdictState.Get == nil { + return nil, fmt.Errorf("the verdicts state cannot be read here") + } + keys, err := r.VerdictState.Keys() + if err != nil { + return nil, err + } + sort.Strings(keys) + var out []VerdictRead + for _, k := range keys { + if group != "" { + if g, _, err := delivery.ParseCheckerVerdictKey(k); err != nil || g != group { + continue + } + } + raw, err := r.VerdictState.Get(k) + if err != nil { + return nil, err + } + if len(raw) == 0 { + continue + } + vr, err := r.read(k, raw) + if err != nil { + return nil, err + } + out = append(out, vr) + } + return out, nil +} + +// Pull answers for one pull request: the verdict of the candidate that holds it, with its report; for an hq pull request, +// that hq is judged by merge-check.sh until issue 1296 (HQNote); for one the indexer holds in no candidate, that none does. +func (r Reader) Pull(repository string, number int) (VerdictRead, error) { + if r.CandidateState.Keys == nil || r.CandidateState.Get == nil { + return VerdictRead{}, fmt.Errorf("the candidates state cannot be read here") + } + keys, err := r.CandidateState.Keys() + if err != nil { + return VerdictRead{}, err + } + sort.Strings(keys) + for _, k := range keys { + raw, err := r.CandidateState.Get(k) + if err != nil || len(raw) == 0 { + continue + } + c, err := delivery.DecodeCandidate(raw) + if err != nil { + continue + } + for _, p := range c.Pulls { + if p.Repository != repository || p.Number != number { + continue + } + if repository == HQRepository { + return VerdictRead{Key: k, Note: HQNote}, nil + } + vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash) + if err != nil { + return VerdictRead{}, err + } + vraw, err := r.VerdictState.Get(vkey) + if err != nil { + return VerdictRead{}, err + } + if len(vraw) == 0 { + return VerdictRead{Key: vkey, Note: "no verdict yet for the heads of its candidate"}, nil + } + return r.read(vkey, vraw) + } + } + return VerdictRead{}, fmt.Errorf("no candidate holds %s#%d", repository, number) +} + +// Kept is the kept verdicts: one key's value as stored, or with an empty key the keys of a level (or every level) and their values. +func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, error) { + if r.KeptState.Keys == nil || r.KeptState.Get == nil { + return nil, fmt.Errorf("the kept state cannot be read here") + } + if digest != "" { + key, err := delivery.KeptKey(level, digest) + if err != nil { + return nil, err + } + raw, err := r.KeptState.Get(key) + if err != nil { + return nil, err + } + if len(raw) == 0 { + return nil, fmt.Errorf("no kept verdict %s", key) + } + return map[string]json.RawMessage{key: raw}, nil + } + keys, err := r.KeptState.Keys() + if err != nil { + return nil, err + } + sort.Strings(keys) + out := map[string]json.RawMessage{} + for _, k := range keys { + if l, _, err := delivery.ParseKeptKey(k); err != nil || (level != "" && l != level) { + continue + } + raw, err := r.KeptState.Get(k) + if err != nil { + return nil, err + } + if len(raw) > 0 { + out[k] = raw + } + } + return out, nil +} diff --git a/modules/mesh-delivery-checker/internal/check/read_test.go b/modules/mesh-delivery-checker/internal/check/read_test.go new file mode 100644 index 0000000..1d1c8fb --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/read_test.go @@ -0,0 +1,104 @@ +package check + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +type fakeState map[string][]byte + +func (f fakeState) read() StateRead { + return StateRead{ + Keys: func() ([]string, error) { + var k []string + for key := range f { + k = append(k, key) + } + return k, nil + }, + Get: func(key string) (json.RawMessage, error) { + if v, ok := f[key]; ok { + return json.RawMessage(v), nil + } + return nil, nil + }, + } +} + +// readerOf is a reader over a verdict the checker wrote through its own code and the candidate it judged. +func readerOf(t *testing.T) (Reader, string) { + t.Helper() + r := newRig(t, passJudge()) + cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1), pullOf(HQRepository, 3, head2)) + r.ch.Candidate(context.Background(), "feat%2Fx", cand) + r.ch.Wait() + ckey, _ := delivery.CandidatesKey("feat/x") + verdicts, kept := fakeState{}, fakeState{} + for k, v := range r.verdicts.vals { + verdicts[k] = v + } + if _, err := (Kept{Store: r.verdicts}).Keep(context.Background(), "unit", digA, keptValue("m/a", 3)); err != nil { + t.Fatal(err) + } + kept["unit."+digA] = r.verdicts.vals["unit."+digA] + return Reader{VerdictState: verdicts.read(), KeptState: kept.read(), CandidateState: fakeState{ckey: cand}.read()}, keyOf(t, "feat/x", cand) +} + +func TestTheVerdictToolsAnswerTheStoredVerdictWithTheFullReportOfEachPullRequest(t *testing.T) { // N9/AC1, 1686 + rd, key := readerOf(t) + vs, err := rd.Verdicts("") + if err != nil || len(vs) != 1 || vs[0].Key != key { + t.Fatalf("verdicts = %+v, %v", vs, err) + } + if _, err := delivery.DecodeVerdict(vs[0].Verdict); err != nil { + t.Errorf("the verdict is not as stored: %v", err) + } + if got := vs[0].Reports["novox/mesh-delivery#7"]; !strings.HasPrefix(got, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(got, "composition not yet run") { + t.Errorf("reports = %v", vs[0].Reports) + } + if only, _ := rd.Verdicts("feat/other"); len(only) != 0 { + t.Errorf("another group's verdicts = %+v", only) + } +} + +func TestAnHqPullRequestIsSaidToBeJudgedByMergeCheckUntilIssue1296(t *testing.T) { // ADR 0484 clause 8 + rd, _ := readerOf(t) + hq, err := rd.Pull(HQRepository, 3) + if err != nil || hq.Note != "hq: judged by merge-check.sh until issue 1296" || hq.Verdict != nil { + t.Errorf("hq pull = %+v, %v", hq, err) + } + p, err := rd.Pull(repoA, 7) + if err != nil || p.Note != "" || !strings.HasPrefix(p.Reports["novox/mesh-delivery#7"], "pass ") { + t.Errorf("pull = %+v, %v", p, err) + } + if _, err := rd.Pull(repoA, 99); err == nil { + t.Error("a pull request no candidate holds was answered") + } +} + +func TestTheKeptToolAnswersTheKeysAsStoredByLevelAndByDigest(t *testing.T) { // N9/AC1 + rd, _ := readerOf(t) + all, err := rd.Kept("", "") + if err != nil || len(all) != 1 { + t.Fatalf("kept = %v, %v", all, err) + } + one, err := rd.Kept("unit", digA) + if err != nil || len(one) != 1 { + t.Fatalf("one kept = %v, %v", one, err) + } + for _, raw := range one { + if k, err := delivery.DecodeKept(raw); err != nil || k.Package != "m/a" { + t.Errorf("kept value = %+v, %v", k, err) + } + } + if none, _ := rd.Kept("integration", ""); len(none) != 0 { + t.Errorf("integration = %v", none) + } + if _, err := rd.Kept("unit", digB); err == nil { + t.Error("a digest nothing kept was answered") + } +} diff --git a/modules/mesh-delivery-checker/manifest_test.go b/modules/mesh-delivery-checker/manifest_test.go index 08d68ba..9d7eebb 100644 --- a/modules/mesh-delivery-checker/manifest_test.go +++ b/modules/mesh-delivery-checker/manifest_test.go @@ -23,12 +23,13 @@ type state struct { } type artifact struct { - Name string `json:"name"` - Kind string `json:"kind"` - Language string `json:"language"` - System string `json:"system"` - From string `json:"from"` - Binary string `json:"binary"` + Name string `json:"name"` + Kind string `json:"kind"` + Language string `json:"language"` + System string `json:"system"` + From string `json:"from"` + Binary string `json:"binary"` + Loads []string `json:"loads"` } type stream struct { @@ -38,6 +39,7 @@ type stream struct { type manifest struct { Streams []stream `json:"streams"` + Tools []string `json:"tools"` Module string `json:"module"` Version string `json:"version"` Slug string `json:"slug"` @@ -115,7 +117,11 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 if want := []stream{{Name: "checks", Events: []string{"preview.**", "owed.**"}}}; !reflect.DeepEqual(m.Streams, want) { t.Errorf("streams = %+v, want %+v", m.Streams, want) } - for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats", "bus"} { + // The read tools (issue 1686) are served by the tools bundle, not by the checker's process. + if want := []string{"checker_kept", "checker_pull", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) { + t.Errorf("tools = %v, want %v", m.Tools, want) + } + for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "seat", "seats", "bus"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the checker has none", k) } @@ -168,8 +174,15 @@ func TestManifestProcess(t *testing.T) { // 1678/AC1 if _, bus := p["bus"]; bus { t.Errorf("the process declares a bus: it runs on its own broker credential") } - if len(m.Build.Artifacts) != 2 || m.Build.Artifacts[1].Name != "files" || m.Build.Artifacts[1].Kind != "archive" || m.Build.Artifacts[1].From != "files" { - t.Fatalf("artifacts = %+v, want the program and the files", m.Build.Artifacts) + if len(m.Build.Artifacts) != 3 || m.Build.Artifacts[2].Name != "files" || m.Build.Artifacts[2].Kind != "archive" || m.Build.Artifacts[2].From != "files" { + t.Fatalf("artifacts = %+v, want the program, the tools and the files", m.Build.Artifacts) + } + if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Language != "go" || a.Binary != "mesh-delivery-checker-tools" || + !reflect.DeepEqual(a.Loads, []string{"mesh-delivery-checker-tools"}) { + t.Errorf("tools artifact = %+v", a) + } + if st, err := os.Stat(m.Build.Artifacts[1].From); err != nil || !st.IsDir() { + t.Errorf("the tools' source %s: %v", m.Build.Artifacts[1].From, err) } a := m.Build.Artifacts[0] if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-checker" || !strings.HasSuffix(a.From, a.Binary) || a.Kind != "bundle" || a.Language != "go" { diff --git a/modules/mesh-delivery-checker/module.json b/modules/mesh-delivery-checker/module.json index bb08186..15e2cc7 100644 --- a/modules/mesh-delivery-checker/module.json +++ b/modules/mesh-delivery-checker/module.json @@ -29,6 +29,11 @@ "mesh-delivery-watchman.presence", "node-engine.have" ], + "tools": [ + "checker_kept", + "checker_pull", + "checker_verdicts" + ], "invokes": [ "mesh-build-go.build_go_digest", "mesh-build-go.build_go_fetch", @@ -106,6 +111,17 @@ "from": "cmd/mesh-delivery-checker", "binary": "mesh-delivery-checker" }, + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-delivery-checker-tools", + "binary": "mesh-delivery-checker-tools", + "loads": [ + "mesh-delivery-checker-tools" + ] + }, { "name": "files", "kind": "archive", From a20af30d54d6cadc7ad5e90333967f175a7c2bcc Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 21:34:35 +0200 Subject: [PATCH 16/19] A group's Go member naming a mesh-sdk commit that is neither on the trunk nor the group's head fails the check naming the member and the commit; a pin only a head carries names issue 1148 (issue 1624) --- .../cmd/mesh-delivery-checker/forge.go | 61 ++++++++ .../cmd/mesh-delivery-checker/forge_test.go | 55 +++++++ .../mesh-delivery-checker/integration_test.go | 64 ++++++++ .../cmd/mesh-delivery-checker/judge.go | 3 +- .../cmd/mesh-delivery-checker/main.go | 2 +- .../internal/check/judge.go | 16 +- .../internal/check/pins.go | 104 +++++++++++++ .../internal/check/pins_test.go | 147 ++++++++++++++++++ .../internal/check/testdata/captured/README | 6 + .../captured/forge-compare-branch.json | 31 ++++ .../captured/forge-compare-trunk.json | 12 ++ .../captured/forge-get-file-go-mod.json | 9 ++ .../mesh-delivery-checker/manifest_test.go | 4 +- modules/mesh-delivery-checker/module.json | 2 + 14 files changed, 511 insertions(+), 5 deletions(-) create mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge.go create mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge_test.go create mode 100644 modules/mesh-delivery-checker/internal/check/pins.go create mode 100644 modules/mesh-delivery-checker/internal/check/pins_test.go create mode 100644 modules/mesh-delivery-checker/internal/check/testdata/captured/forge-compare-branch.json create mode 100644 modules/mesh-delivery-checker/internal/check/testdata/captured/forge-compare-trunk.json create mode 100644 modules/mesh-delivery-checker/internal/check/testdata/captured/forge-get-file-go-mod.json diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge.go new file mode 100644 index 0000000..9245ad5 --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge.go @@ -0,0 +1,61 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "strings" +) + +// The forge's tools the checker reads with (mesh-catalog modules/gitea): gitea_get_file answers {file: {content}}, the text +// of a file at a ref, and gitea_compare answers {total_commits, commits}, what head holds that base does not. The checker +// writes nothing to the forge: the forge bridge sets the statuses (ADR 0484 clause 2). +const ( + getFileTool = "gitea.gitea_get_file" + compareTool = "gitea.gitea_compare" +) + +// forgeReads reads the forge through call (mesh-sdk process.Conn.Call as the checker's own account). +type forgeReads struct { + call func(key string, body any) (json.RawMessage, error) +} + +func ownerRepo(repository string) (string, string) { + owner, repo, _ := strings.Cut(repository, "/") + return owner, repo +} + +// File is the text of a file of a repository at a ref. +func (f forgeReads) File(_ context.Context, repository, path, ref string) (string, error) { + owner, repo := ownerRepo(repository) + raw, err := f.call(getFileTool, map[string]string{"owner": owner, "repo": repo, "path": path, "ref": ref}) + if err != nil { + return "", err + } + var a struct { + File *struct { + Content *string `json:"content"` + } `json:"file"` + } + if err := json.Unmarshal(raw, &a); err != nil || a.File == nil || a.File.Content == nil { + return "", fmt.Errorf("the answer of %s is not {file: {content}}: %v", getFileTool, err) + } + return *a.File.Content, nil +} + +// OnTrunk says whether the commit is already on the repository's main: main holds it when the commit holds nothing main does +// not. +func (f forgeReads) OnTrunk(_ context.Context, repository, commit string) (bool, error) { + owner, repo := ownerRepo(repository) + raw, err := f.call(compareTool, map[string]string{"owner": owner, "repo": repo, "base": "main", "head": commit}) + if err != nil { + return false, err + } + var a struct { + Total *int `json:"total_commits"` + } + if err := json.Unmarshal(raw, &a); err != nil || a.Total == nil { + return false, fmt.Errorf("the answer of %s is not {total_commits}: %v", compareTool, err) + } + return *a.Total == 0, nil +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge_test.go new file mode 100644 index 0000000..fbb84d8 --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge_test.go @@ -0,0 +1,55 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// capturedAnswer is what the forge's tool answered (internal/check/testdata/captured/README). +func capturedAnswer(t *testing.T, name string) json.RawMessage { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "internal", "check", "testdata", "captured", name+".json")) + if err != nil { + t.Fatal(err) + } + return raw +} + +func TestTheForgeReadsParseWhatTheForgeAnswered(t *testing.T) { // 1624/AC4 + var asked []string + f := forgeReads{call: func(key string, body any) (json.RawMessage, error) { + b, _ := json.Marshal(body) + asked = append(asked, key+" "+string(b)) + switch { + case key == getFileTool: + return capturedAnswer(t, "forge-get-file-go-mod"), nil + case strings.Contains(string(b), "cb54756c3865"): + return capturedAnswer(t, "forge-compare-trunk"), nil + } + return capturedAnswer(t, "forge-compare-branch"), nil + }} + text, err := f.File(context.Background(), "novox/mesh-delivery", "modules/mesh-delivery-builder/go.mod", "main") + if err != nil || !strings.Contains(text, "git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865") { + t.Errorf("file = %q, %v", text, err) + } + if on, err := f.OnTrunk(context.Background(), "novox/mesh-sdk", "cb54756c3865"); err != nil || !on { + t.Errorf("a commit on the trunk: %v, %v", on, err) + } + if on, err := f.OnTrunk(context.Background(), "novox/mesh-sdk", "9aaa5c80ac5c"); err != nil || on { + t.Errorf("a commit of the branch: %v, %v", on, err) + } + if len(asked) != 3 || !strings.Contains(asked[0], `"owner":"novox"`) || !strings.Contains(asked[2], `"base":"main"`) { + t.Errorf("asked %v", asked) + } + bad := forgeReads{call: func(string, any) (json.RawMessage, error) { return json.RawMessage(`{}`), nil }} + if _, err := bad.File(context.Background(), "novox/x", "go.mod", "main"); err == nil { + t.Error("an answer with no file was taken") + } + if _, err := bad.OnTrunk(context.Background(), "novox/x", "abc"); err == nil { + t.Error("an answer with no total was taken") + } +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go index 5767b67..cbf005b 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go @@ -150,7 +150,54 @@ func (f *fakeBuildModules) serve(t *testing.T, nc *nats.Conn, module string) { t.Cleanup(func() { _ = sub.Unsubscribe() }) } +// serveForge answers the forge's two reads as its tools do, with the answers the forge gave (internal/check/testdata/captured): +// the go.mod of the captured file, whose mesh-sdk pin is cb54756c3865, and gitea_compare for the pin: on the trunk, unless the +// pin is the one the test set as stale. +func (b *bed) serveForge(t *testing.T, nc *nats.Conn) { + reply := func(m *nats.Msg, name string) { + var result map[string]any + _ = json.Unmarshal(mustRead(t, name), &result) + answer, _ := json.Marshal(map[string]any{"result": result, "error": "", "node": "n0"}) + _ = m.Respond(answer) + } + for _, tool := range []string{"gitea_get_file", "gitea_compare"} { + tool := tool + sub, err := nc.Subscribe("mesh.mod.gitea.tool."+tool, func(m *nats.Msg) { + var a map[string]string + _ = json.Unmarshal(m.Data, &a) + switch { + case tool == "gitea_get_file" && b.stalePin.Load(): + content := strings.Replace(string(mustRead(t, "forge-get-file-go-mod")), "cb54756c3865", "9aaa5c80ac5c", 1) + var result map[string]any + _ = json.Unmarshal([]byte(content), &result) + answer, _ := json.Marshal(map[string]any{"result": result, "error": "", "node": "n0"}) + _ = m.Respond(answer) + case tool == "gitea_get_file": + reply(m, "forge-get-file-go-mod") + case strings.HasPrefix(a["head"], "9aaa5c80ac5c"): + reply(m, "forge-compare-branch") + default: + reply(m, "forge-compare-trunk") + } + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + } +} + +func mustRead(t *testing.T, name string) []byte { + t.Helper() + raw, err := os.ReadFile(filepath.Join(captured, name+".json")) + if err != nil { + t.Fatal(err) + } + return raw +} + type bed struct { + stalePin atomic.Bool ctx context.Context js jetstream.JetStream buckets map[string]jetstream.KeyValue @@ -231,6 +278,7 @@ func startBed(t *testing.T) *bed { for _, module := range []string{"mesh-build-go", "mesh-build-node"} { b.setHealth(t, "n1", module) } + b.serveForge(t, ctl) b.fake = &fakeBuildModules{js: js, ctx: ctx, calls: map[string]int{}} b.fake.serve(t, ctl, "mesh-build-go") b.fake.serve(t, ctl, "mesh-build-node") @@ -526,3 +574,19 @@ func TestNoCheckStartsWhileADeliveryBuildsAndOneStartsWhenItEnds(t *testing.T) { t.Errorf("verdict = %+v", v) } } + +// TestAMemberNamingAMeshSDKCommitOffTheTrunkFailsNamingTheCommit: the go.mod the forge answers names a commit that main does not +// hold; the verdict fails naming the member and the commit and no test runs (ADR 0484 clause 9). +func TestAMemberNamingAMeshSDKCommitOffTheTrunkFailsNamingTheCommit(t *testing.T) { // 1624/AC2 + b := startBed(t) + b.stalePin.Store(true) + key := b.candidate(t, "feat/x", pull(repo, 7, head1)) + b.start(t, nil) + v := b.verdict(t, key, ended) + if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "9aaa5c80ac5c") || !strings.Contains(v.Failed.Words, repo+"#7") { + t.Fatalf("verdict = %+v", v) + } + if n := b.fake.count("mesh-build-go.build_go_test"); n != 0 { + t.Errorf("%d test steps ran after a stale pin", n) + } +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go index 61fe261..b2a909f 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go @@ -23,6 +23,7 @@ type judgeDeps struct { kept check.KV previews check.KV repos *repoModules + forge check.Forge announce check.Announcer builds check.BuildStates logf func(format string, args ...any) @@ -30,7 +31,7 @@ type judgeDeps struct { func newJudge(d judgeDeps) check.Judge { return &check.Examiner{Steps: d.runner, Toolchains: d.tools, Presence: d.presence, Health: d.have, Places: d.places, - Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Builds: d.builds, Announce: d.announce, Trunks: d.trunks, Logf: d.logf} + Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Forge: d.forge, Builds: d.builds, Announce: d.announce, Trunks: d.trunks, Logf: d.logf} } // manifestKV reads a manifest by its digest. diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index 7f01c48..752a6ba 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -221,7 +221,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro ch.Judge = newJudge(judgeDeps{ runner: runner, tools: tools, presence: presence, have: have, places: places, trunks: trunks, kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]}, - repos: repos, builds: builds, logf: logger.Printf, + repos: repos, builds: builds, forge: forgeReads{call: caller.Call}, logf: logger.Printf, announce: announce, }) ch.Announce = announce diff --git a/modules/mesh-delivery-checker/internal/check/judge.go b/modules/mesh-delivery-checker/internal/check/judge.go index 21dc121..49af8a4 100644 --- a/modules/mesh-delivery-checker/internal/check/judge.go +++ b/modules/mesh-delivery-checker/internal/check/judge.go @@ -13,6 +13,14 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) +// goModule is the build module the Go repositories are judged by. +const goModule = buildrun.GoBuildModule + +// verdictPtr is a verdict of failure a step of the check ends with. +type verdictPtr = *delivery.Verdict + +func failed(machine, words string) verdictPtr { return fail(machine, words) } + // Steps runs a build module's long step on a node and answers its result (buildrun.Runner). type Steps interface { Run(ctx context.Context, module, node, tool string, args map[string]any) (json.RawMessage, error) @@ -50,6 +58,9 @@ type Examiner struct { // Previews is the previews bucket; nil writes none. Previews KV Modules BuildModules + // Forge reads the go.mod files of the Go members and says whether a mesh-sdk commit is on its trunk (ADR 0484 clause 9); + // nil checks no pin. + Forge Forge // Builds says how the build of a fingerprint the candidate carries stands (the builder's builds bucket); nil says nothing // of the builds. Builds BuildStates @@ -120,6 +131,9 @@ func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) { if v := st.fetchAll(ctx); v != nil { return st.finish(ctx, *v), nil } + if v := st.checkPins(ctx); v != nil { + return st.finish(ctx, *v), nil + } for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} { for _, priority := range []bool{true, false} { for _, t := range st.targets { @@ -230,7 +244,7 @@ func (st *state) fetchAll(ctx context.Context) *delivery.Verdict { } raw, err := st.step(ctx, t, "fetch", fmt.Sprintf("fetch %s#%d on %s", p.Repository, p.Number, node), args) if err != nil { - return fail(node, fmt.Sprintf("fetch of %s#%d: %v", p.Repository, p.Number, err)) + return fail(node, fmt.Sprintf("fetch of %s#%d: %v%s", p.Repository, p.Number, err, pinHint(st.c, err.Error()))) } if err := json.Unmarshal(raw, &t.fetched); err != nil || t.fetched.Workspace == "" { return fail(node, fmt.Sprintf("the fetch of %s#%d answered no workspace: %s", p.Repository, p.Number, raw)) diff --git a/modules/mesh-delivery-checker/internal/check/pins.go b/modules/mesh-delivery-checker/internal/check/pins.go new file mode 100644 index 0000000..592195d --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/pins.go @@ -0,0 +1,104 @@ +package check + +import ( + "context" + "fmt" + "path" + "regexp" + "strings" +) + +// The pin of mesh-sdk in a Go member's go.mod (novox/hq ADR 0484 clause 9, ADR 0482 clause 2, issue 1624): a delivery group +// has one verdict, and the mesh-sdk its Go members are built and tested with is the group's: a pin to the head of the group's +// mesh-sdk pull request, or to a commit already on mesh-sdk's trunk. Any other commit is a stale or a foreign pin, and the +// group fails naming the member and the commit. + +// SDKRepository is the repository of mesh-sdk, and SDKModulePath its Go module. +const ( + SDKRepository = "novox/mesh-sdk" + SDKModulePath = "git.novox.be/novox/mesh-sdk/go" +) + +// Issue1148 is why a pin that only a head carries fails the check. +const Issue1148 = "issue 1148" + +// Forge reads a file of a repository at a ref, and says whether a commit is on a repository's trunk. +type Forge interface { + // File is the text of a file at a ref (a branch, a tag or a commit). + File(ctx context.Context, repository, path, ref string) (string, error) + // OnTrunk says whether commit (a full or a 12-digit sha) is already merged into the trunk of the repository: it holds no + // commit the trunk does not. + OnTrunk(ctx context.Context, repository, commit string) (bool, error) +} + +var pseudoVersion = regexp.MustCompile(`-([0-9a-f]{12})$`) + +// SDKPin is the commit (12 hex digits) a go.mod names for mesh-sdk by a pseudo-version, or none for a go.mod that does not +// require mesh-sdk or requires a version that is no pseudo-version (a tag is a release the SDK itself names). +func SDKPin(goMod string) (commit string, ok bool) { + for _, line := range strings.Split(goMod, "\n") { + f := strings.Fields(strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(line), "require"))) + if len(f) >= 2 && f[0] == SDKModulePath { + if m := pseudoVersion.FindStringSubmatch(f[1]); m != nil { + return m[1], true + } + return "", false + } + } + return "", false +} + +// checkPins refuses a Go member whose go.mod names a mesh-sdk commit that is neither on mesh-sdk's trunk nor the head of the +// group's mesh-sdk pull request. It reads each go.mod the fetch named at the member's head from the forge. It answers nil +// when every pin is the group's. +func (st *state) checkPins(ctx context.Context) verdictPtr { + if st.j.Forge == nil { + return nil + } + sdk := st.c.member(SDKRepository) + for _, t := range st.targets { + if t.module != goModule || t.pull.Repository == SDKRepository { + continue + } + for _, dir := range t.fetched.Modules { + file := path.Join(dir, "go.mod") + st.c.Progress.Set(fmt.Sprintf("read %s of %s#%d", file, t.pull.Repository, t.pull.Number), t.node) + text, err := st.j.Forge.File(ctx, t.pull.Repository, file, t.pull.Head) + if err != nil { + return failed(t.node, fmt.Sprintf("%s#%d: %s cannot be read to check its mesh-sdk pin: %v", t.pull.Repository, t.pull.Number, file, err)) + } + pin, ok := SDKPin(text) + if !ok { + continue + } + if sdk != nil && strings.HasPrefix(sdk.Head, pin) { + continue // the group's own head + } + onTrunk, err := st.j.Forge.OnTrunk(ctx, SDKRepository, pin) + if err != nil { + return failed(t.node, fmt.Sprintf("%s#%d: whether mesh-sdk %s is on its trunk cannot be told: %v", t.pull.Repository, t.pull.Number, pin, err)) + } + if onTrunk { + continue + } + nor := "the group has no mesh-sdk pull request" + if sdk != nil { + nor = "the head of the group's mesh-sdk pull request is " + short12(sdk.Head) + } + return failed(t.node, fmt.Sprintf("%s#%d: %s names mesh-sdk %s, which is neither on mesh-sdk's trunk nor the group's head (%s)", t.pull.Repository, t.pull.Number, file, pin, nor)) + } + } + return nil +} + +// failureVerdict is a verdict of failure the check ends with. + +// pinHint is added to the failure of a Go member's fetch that names mesh-sdk while the group holds a mesh-sdk pull request: +// a pin to that pull request's head names a commit only the head carries, which go mod download cannot resolve before the +// pull request is merged (issue 1148), so such a check fails and never passes. +func pinHint(c *Case, fetchError string) string { + if c.member(SDKRepository) == nil || !strings.Contains(fetchError, "mesh-sdk") { + return "" + } + return " (a pin to the head of the group's mesh-sdk pull request names a commit only that head carries, which go mod download cannot resolve; " + Issue1148 + ")" +} diff --git a/modules/mesh-delivery-checker/internal/check/pins_test.go b/modules/mesh-delivery-checker/internal/check/pins_test.go new file mode 100644 index 0000000..3ffdc06 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/pins_test.go @@ -0,0 +1,147 @@ +package check + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// fakeForge answers the forge's two reads from files and commits it is given. +type fakeForge struct { + goMods map[string]string // ` ` -> text + onTrunk map[string]bool // by commit prefix + err error +} + +func (f *fakeForge) File(_ context.Context, repository, path, ref string) (string, error) { + if f.err != nil { + return "", f.err + } + if s, ok := f.goMods[repository+" "+path+" "+ref]; ok { + return s, nil + } + return "", fmt.Errorf("no file %s of %s at %s", path, repository, ref) +} + +func (f *fakeForge) OnTrunk(_ context.Context, _ string, commit string) (bool, error) { + if f.err != nil { + return false, f.err + } + return f.onTrunk[commit[:12]], nil +} + +// capturedGoMod is the go.mod the forge answered (testdata/captured/forge-get-file-go-mod.json), pinned to the commit given. +func capturedGoMod(t *testing.T, pin string) string { + t.Helper() + raw, err := os.ReadFile(filepath.Join("testdata", "captured", "forge-get-file-go-mod.json")) + if err != nil { + t.Fatal(err) + } + var a struct{ File struct{ Content string } } + if err := json.Unmarshal(raw, &a); err != nil { + t.Fatal(err) + } + return strings.Replace(a.File.Content, "cb54756c3865", pin, 1) +} + +func TestTheSDKPinIsTheCommitOfAPseudoVersion(t *testing.T) { // 1624 + if c, ok := SDKPin(capturedGoMod(t, "cb54756c3865")); !ok || c != "cb54756c3865" { + t.Errorf("pin = %q, %v", c, ok) + } + if _, ok := SDKPin("module x\n\nrequire git.novox.be/novox/mesh-sdk/go v0.2.0\n"); ok { + t.Error("a tagged version is no commit") + } + if c, ok := SDKPin("module x\n\nrequire git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865\n"); !ok || c != "cb54756c3865" { + t.Errorf("a one-line require: %q, %v", c, ok) + } + if _, ok := SDKPin("module x\n\ngo 1.26\n"); ok { + t.Error("a go.mod that does not require mesh-sdk has a pin") + } +} + +// groupBench is a group of mesh-delivery#7 and the SDK's pull request 12, mesh-delivery's go.mod at its head naming pin. +func groupBench(t *testing.T, pin string, trunk ...string) *bench { + b := newBench(t) + b.fetch.Modules = []string{"modules/m"} + forge := &fakeForge{goMods: map[string]string{repoA + " modules/m/go.mod " + head1: capturedGoMod(t, pin)}, onTrunk: map[string]bool{}} + for _, c := range trunk { + forge.onTrunk[c] = true + } + b.ex.Forge = forge + return b +} + +func TestAMemberPinnedToTheSDKMembersHeadPassesAndIsBuiltAgainstIt(t *testing.T) { // 1624/AC2, AC4 + b := groupBench(t, head2[:12]) + v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))).Verdict + if v.State != delivery.VerdictPass { + t.Fatalf("verdict = %+v", v) + } + var besideSDK bool + for _, f := range b.steps.tool("fetch") { + if f.args["repository"] == repoA { + refs, _ := f.args["beside"].([]BesideRef) + for _, r := range refs { + besideSDK = besideSDK || (r.Repository == repoSDK && r.Commit == head2) + } + } + } + if !besideSDK { + t.Error("mesh-delivery's tests were not given the SDK member at its head") + } +} + +func TestAMemberPinnedToAnOlderHeadOfTheSDKMemberFailsNamingTheMemberAndTheCommit(t *testing.T) { // 1624/AC2, AC4 + b := groupBench(t, "9aaa5c80ac5c") // a commit of the SDK branch, not its head, not on the trunk + v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))).Verdict + f := v.Failed + if v.State != delivery.VerdictFail || f.Machine != "shanks" || !strings.Contains(f.Words, repoA+"#7") || !strings.Contains(f.Words, "9aaa5c80ac5c") || + !strings.Contains(f.Words, "neither on mesh-sdk's trunk nor the group's head") || !strings.Contains(f.Words, short12(head2)) { + t.Fatalf("verdict = %+v", v) + } + if n := len(b.steps.tool("test")); n != 0 { + t.Errorf("%d test steps ran after a stale pin", n) + } +} + +func TestAMemberPinnedToACommitOnTheTrunkPasses(t *testing.T) { // 1624/AC2, AC4 + b := groupBench(t, "cb54756c3865", "cb54756c3865") + if v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))).Verdict; v.State != delivery.VerdictPass { + t.Fatalf("verdict = %+v", v) + } + // A lone pull request of a Go member is held to the trunk alone. + b = groupBench(t, "9aaa5c80ac5c") + v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict + if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "the group has no mesh-sdk pull request") { + t.Errorf("a foreign pin with no SDK member: %+v", v) + } +} + +func TestAPinOnlyTheSDKHeadCarriesThatDownloadCannotResolveFailsNamingIssue1148(t *testing.T) { // 1624/AC5, ADR 0453 clause 12 + b := groupBench(t, head2[:12]) + b.steps.handlers["build_go_fetch"] = func(c stepCall) (any, error) { + if c.args["repository"] == repoA { + return nil, fmt.Errorf("go mod download: git.novox.be/novox/mesh-sdk/go@v0.1.14-0.20261021100209-%s: invalid version: unknown revision", head2[:12]) + } + return b.fetch, nil + } + v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))).Verdict + if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "issue 1148") || !strings.Contains(v.Failed.Words, head2[:12]) { + t.Errorf("verdict = %+v, want a failure naming issue 1148 and the commit", v) + } +} + +func TestAPinThatCannotBeCheckedFailsAndNeverPasses(t *testing.T) { // ADR 0453 clause 12 + b := groupBench(t, "cb54756c3865") + b.ex.Forge.(*fakeForge).err = fmt.Errorf("the forge is down") + v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))).Verdict + if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "the forge is down") { + t.Errorf("verdict = %+v", v) + } +} diff --git a/modules/mesh-delivery-checker/internal/check/testdata/captured/README b/modules/mesh-delivery-checker/internal/check/testdata/captured/README index b8023fe..9c809ea 100644 --- a/modules/mesh-delivery-checker/internal/check/testdata/captured/README +++ b/modules/mesh-delivery-checker/internal/check/testdata/captured/README @@ -11,3 +11,9 @@ into a throwaway test that wrote them out, at mesh-catalog feat/r4c5-checker d58 so the shas are its constants; the shape and the field names are the module's own marshalling). node-fetch-clean.json is the FetchResult json.Marshal of mesh-build-node's fetch (fetch.go, preview.go) in its test TestFetchWithABaseMergesAndAnswersThePreview (cmd/mesh-build-node/preview_test.go) at the same commit. Captured on 2026-10-21 for issue 1679. + +forge-get-file-go-mod.json is the answer of the forge's gitea_get_file (mesh-catalog modules/gitea, tools/index.ts) for modules/mesh-delivery-builder/go.mod of +novox/mesh-delivery at main, and forge-compare-trunk.json the answer of gitea_compare for novox/mesh-sdk base main, head cb54756c3865 (a commit on the +trunk: no commit main does not hold); both called through the mesh on 2026-10-21 for issue 1624. forge-compare-branch.json is the answer of the same call for a +head that is not on the trunk (novox/mesh-sdk base main, head afbe371003a5, the branch feat/r4c5-sdk), shortened to two of its seven commits and one of its +twenty-six files, with the head and the totals set to match. diff --git a/modules/mesh-delivery-checker/internal/check/testdata/captured/forge-compare-branch.json b/modules/mesh-delivery-checker/internal/check/testdata/captured/forge-compare-branch.json new file mode 100644 index 0000000..f8d5829 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/testdata/captured/forge-compare-branch.json @@ -0,0 +1,31 @@ +{ + "base": "main", + "commits": [ + { + "author": "jochen", + "date": "2026-10-21T19:35:14+02:00", + "files": 2, + "sha": "9aaa5c80ac5c4820c4eddf488fbb90ac6981a624", + "subject": "Accept the gate's recheck status mesh/repo-check among the owed contexts, so a pinned recheck is not refused (issue 1673)" + }, + { + "author": "jochen", + "date": "2026-10-21T19:26:02+02:00", + "files": 3, + "sha": "0730ef609216798e787ee8b6542912f5ec613a77", + "subject": "Keep the unit level cheap: the merge's real-git tests start processes, so they are integration tests (issue 895)" + } + ], + "files": [ + { + "path": "go/delivery/gate.go", + "status": "modified" + } + ], + "head": "9aaa5c80ac5c", + "repository": "novox/mesh-sdk", + "said": [ + "9aaa5c80ac5c holds 2 commits main does not, touching 1 files" + ], + "total_commits": 2 +} diff --git a/modules/mesh-delivery-checker/internal/check/testdata/captured/forge-compare-trunk.json b/modules/mesh-delivery-checker/internal/check/testdata/captured/forge-compare-trunk.json new file mode 100644 index 0000000..21dff8b --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/testdata/captured/forge-compare-trunk.json @@ -0,0 +1,12 @@ +{ + "base": "main", + "commits": [], + "files": [], + "head": "cb54756c3865", + "repository": "novox/mesh-sdk", + "said": [ + "cb54756c3865 holds 0 commits main does not, touching 0 files", + "a file's status is the last listed commit's change to it; for the lines, gitea_pull_request_diff or gitea_get_file" + ], + "total_commits": 0 +} diff --git a/modules/mesh-delivery-checker/internal/check/testdata/captured/forge-get-file-go-mod.json b/modules/mesh-delivery-checker/internal/check/testdata/captured/forge-get-file-go-mod.json new file mode 100644 index 0000000..a591e7e --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/testdata/captured/forge-get-file-go-mod.json @@ -0,0 +1,9 @@ +{ + "file": { + "content": "module mesh-delivery-builder\n\ngo 1.26.0\n\nrequire (\n\tgit.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865\n\tgithub.com/nats-io/nats-server/v2 v2.11.17\n\tgithub.com/nats-io/nats.go v1.54.0\n)\n\nrequire (\n\tgithub.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect\n\tgithub.com/google/go-tpm v0.9.8 // indirect\n\tgithub.com/klauspost/compress v1.20.0 // indirect\n\tgithub.com/minio/highwayhash v1.0.4 // indirect\n\tgithub.com/nats-io/jwt/v2 v2.8.1 // indirect\n\tgithub.com/nats-io/nkeys v0.4.16 // indirect\n\tgithub.com/nats-io/nuid v1.0.1 // indirect\n\tgolang.org/x/crypto v0.57.0 // indirect\n\tgolang.org/x/sys v0.48.0 // indirect\n\tgolang.org/x/time v0.15.0 // indirect\n)\n", + "path": "modules/mesh-delivery-builder/go.mod", + "ref": "main", + "sha": "4431597e2e4d2a26b59006f9f8c54659a964ec68", + "size": 691 + } +} diff --git a/modules/mesh-delivery-checker/manifest_test.go b/modules/mesh-delivery-checker/manifest_test.go index 9d7eebb..5370e5c 100644 --- a/modules/mesh-delivery-checker/manifest_test.go +++ b/modules/mesh-delivery-checker/manifest_test.go @@ -100,8 +100,8 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } - if want := []string{"mesh-build-go.build_go_digest", "mesh-build-go.build_go_fetch", "mesh-build-go.build_go_test", "mesh-build-node.build_node_fetch", "mesh-build-node.build_node_test"}; !reflect.DeepEqual(sorted(m.Invokes), want) { - t.Errorf("invokes = %v, want only the build modules' fetch, digest and test (no run step before issue 1309, ADR 0484 clause 5)", m.Invokes) + if want := []string{"gitea.gitea_compare", "gitea.gitea_get_file", "mesh-build-go.build_go_digest", "mesh-build-go.build_go_fetch", "mesh-build-go.build_go_test", "mesh-build-node.build_node_fetch", "mesh-build-node.build_node_test"}; !reflect.DeepEqual(sorted(m.Invokes), want) { + t.Errorf("invokes = %v, want only the build modules' fetch, digest and test (no run step before issue 1309, ADR 0484 clause 5) and the forge's two reads for a mesh-sdk pin (clause 9)", m.Invokes) } // The checker is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it starts. if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported"}; !reflect.DeepEqual(sorted(m.Consumes), want) { diff --git a/modules/mesh-delivery-checker/module.json b/modules/mesh-delivery-checker/module.json index 15e2cc7..80b7351 100644 --- a/modules/mesh-delivery-checker/module.json +++ b/modules/mesh-delivery-checker/module.json @@ -35,6 +35,8 @@ "checker_verdicts" ], "invokes": [ + "gitea.gitea_compare", + "gitea.gitea_get_file", "mesh-build-go.build_go_digest", "mesh-build-go.build_go_fetch", "mesh-build-go.build_go_test", From 284f21c4e30b6a93b4f7b479386c4c5a5d566a34 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 21:35:53 +0200 Subject: [PATCH 17/19] Key the builder's candidate test as the indexer keys a group --- .../cmd/mesh-delivery-builder/integration_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index 926260a..d7b8fab 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -554,7 +554,11 @@ func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *tes if err != nil { t.Fatal(err) } - if _, err := b.candidates.Put(b.ctx, "feat%2Fx", cand); err != nil { + ckey, err := delivery.CandidatesKey("feat/x") + if err != nil { + t.Fatal(err) + } + if _, err := b.candidates.Put(b.ctx, ckey, cand); err != nil { t.Fatal(err) } b.start(t, "n1", nil) From 7214bf436828ea6a48fdd3302b11ac812f66e7a2 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 21:50:30 +0200 Subject: [PATCH 18/19] Fail a candidate's build when its fetch answers files in conflict or no merge, so the trunk's build is not stored under the candidate's fingerprint (review D4) --- .../mesh-delivery-builder/integration_test.go | 6 +++++- .../internal/build/candidates_test.go | 18 ++++++++++++++++++ .../internal/build/run.go | 16 +++++++++++++++- .../internal/build/run_test.go | 14 ++++++++++++-- 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index d7b8fab..7affcfc 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -109,7 +109,11 @@ func (f *fakeBuildModules) serve(t *testing.T, nc *nats.Conn, module string) { step.Ended = time.Now().UTC() switch { case strings.HasSuffix(tool, "_fetch"): - step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"workspace":"w`+id+`","commit":"`+commit+`"}`) + merge := "" + if args["base"] != nil { // a fetch with a base answers the merge commit (mesh-build-go FetchResult) + merge = `,"merge":"` + strings.Repeat("9", 40) + `"` + } + step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"workspace":"w`+id+`","commit":"`+commit+`"`+merge+`}`) case f.failBuild.Load() && node == "n1": step.State, step.Reason = delivery.StepFailed, fmt.Sprintf("go build of %v: boom", args["artifact"]) default: diff --git a/modules/mesh-delivery-builder/internal/build/candidates_test.go b/modules/mesh-delivery-builder/internal/build/candidates_test.go index acc5976..7c26902 100644 --- a/modules/mesh-delivery-builder/internal/build/candidates_test.go +++ b/modules/mesh-delivery-builder/internal/build/candidates_test.go @@ -2,6 +2,7 @@ package build import ( "context" + "strings" "testing" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -94,3 +95,20 @@ func TestAFingerprintTheTrunkCarriesIsBuiltFromTheTrunksCommit(t *testing.T) { / t.Errorf("build = %+v", v) } } + +func TestACandidateWhoseHeadDoesNotMergeIsFailedAndNoTrunkBuildIsStoredUnderIt(t *testing.T) { // review D4 + for name, set := range map[string]func(*rig){ + "conflict": func(r *rig) { r.bm.conflicts = "go.mod" }, + "no merge": func(r *rig) { r.bm.noMerge = true }, + } { + r := newRig(t, goBundle) + trunkModule(t, r) + set(r) + r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) + r.b.Wait() + v := valueOf(t, r.st, r.key) + if v.State != delivery.BuildFailed || (name == "conflict" && !strings.Contains(v.Error, "go.mod")) || len(r.bm.callsOf("build_go_build")) != 0 { + t.Errorf("%s: build = %+v, builds %d, want failed and nothing built", name, v, len(r.bm.callsOf("build_go_build"))) + } + } +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 495027f..5f7782d 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -524,12 +524,26 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del return } var fetched struct { - Workspace string `json:"workspace"` + Workspace string `json:"workspace"` + Merge string `json:"merge"` + Conflicts []string `json:"conflicts"` } if err := json.Unmarshal(raw, &fetched); err != nil || fetched.Workspace == "" { l.err = fmt.Errorf("the fetch on %s answered no workspace: %s", l.node, raw) return } + // A candidate is built from its head merged onto the trunk. A head that does not merge ends the fetch passed with the tree at + // the trunk, and building that would store the trunk's build under the candidate's fingerprint. + if _, candidate := fetchArgs["base"]; candidate { + if len(fetched.Conflicts) > 0 { + l.err = fmt.Errorf("the head does not merge into the trunk on %s: files in conflict: %s", l.node, strings.Join(fetched.Conflicts, ", ")) + return + } + if fetched.Merge == "" { + l.err = fmt.Errorf("the fetch with a base on %s answered no merge: %s", l.node, raw) + return + } + } for _, t := range l.tasks { raw, err := b.step(ctx, l.module, l.node, "build", map[string]any{"workspace": fetched.Workspace, "module": m.Path, "artifact": t.Artifact}) if err != nil { diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 9764790..73812c5 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -24,8 +24,11 @@ type buildModules struct { digests map[string]string // `/` -> digest; the default is `digest` fails map[string]string // `/` -> the reason the step fails with // hold, when set before the builds start, blocks every call until it is closed; entered is sent to as a call arrives. - hold chan struct{} - entered chan struct{} + // conflicts, when set, is a file a fetch with a base reports in conflict; noMerge makes it answer neither a merge nor a conflict. + conflicts string + noMerge bool + hold chan struct{} + entered chan struct{} } func newBuildModules() *buildModules { @@ -52,6 +55,13 @@ func (b *buildModules) CallOn(node, key string, body any) (json.RawMessage, erro step.State, step.Reason = delivery.StepFailed, reason } else if strings.HasSuffix(tool, "_fetch") { step.Result = json.RawMessage(`{"workspace":"w-` + node + `"}`) + if args["base"] != nil { // a fetch with a base answers the merge, or the files in conflict (mesh-build-go FetchResult) + if b.conflicts != "" { + step.Result = json.RawMessage(`{"workspace":"w-` + node + `","base":"` + fmt.Sprint(args["base"]) + `","conflicts":["` + b.conflicts + `"]}`) + } else if !b.noMerge { + step.Result = json.RawMessage(`{"workspace":"w-` + node + `","base":"` + fmt.Sprint(args["base"]) + `","merge":"` + strings.Repeat("9", 40) + `"}`) + } + } } else { d := digest if v, ok := b.digests[node+"/"+fmt.Sprint(args["artifact"])]; ok { From b519c8dde2da4a52283f2b43a1e334363d44c8b4 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 21:52:41 +0200 Subject: [PATCH 19/19] Keep nothing for a workspace given siblings, place the siblings where the tests state them, decide the hold from the listed plan keys (review D1-D3) --- .../cmd/mesh-delivery-checker/forge.go | 19 +++++ .../cmd/mesh-delivery-checker/forge_test.go | 15 ++++ .../cmd/mesh-delivery-checker/hold.go | 61 +++++++++---- .../cmd/mesh-delivery-checker/hold_test.go | 35 ++++++++ .../cmd/mesh-delivery-checker/judge.go | 14 ++- .../cmd/mesh-delivery-checker/judge_test.go | 3 + .../internal/check/beside.go | 85 ++++++++++++++----- .../internal/check/examiner_test.go | 35 +++++++- .../internal/check/hold.go | 35 +------- .../internal/check/hold_test.go | 30 +------ .../internal/check/judge.go | 19 ++++- .../internal/check/pins.go | 2 + .../internal/check/pins_test.go | 57 +++++++++++++ .../internal/check/testdata/captured/README | 2 + .../testdata/captured/forge-commits.json | 14 +++ .../mesh-delivery-checker/manifest_test.go | 2 +- modules/mesh-delivery-checker/module.json | 1 + 17 files changed, 322 insertions(+), 107 deletions(-) create mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker/hold_test.go create mode 100644 modules/mesh-delivery-checker/internal/check/testdata/captured/forge-commits.json diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge.go index 9245ad5..9cf6715 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge.go @@ -13,6 +13,7 @@ import ( const ( getFileTool = "gitea.gitea_get_file" compareTool = "gitea.gitea_compare" + commitsTool = "gitea.gitea_commits" ) // forgeReads reads the forge through call (mesh-sdk process.Conn.Call as the checker's own account). @@ -59,3 +60,21 @@ func (f forgeReads) OnTrunk(_ context.Context, repository, commit string) (bool, } return *a.Total == 0, nil } + +// FullSHA is the whole sha of a commit named by its first digits: the newest commit of the history at that ref. +func (f forgeReads) FullSHA(_ context.Context, repository, commit string) (string, error) { + owner, repo := ownerRepo(repository) + raw, err := f.call(commitsTool, map[string]any{"owner": owner, "repo": repo, "ref": commit, "limit": 1}) + if err != nil { + return "", err + } + var a struct { + Commits []struct { + SHA string `json:"sha"` + } `json:"commits"` + } + if err := json.Unmarshal(raw, &a); err != nil || len(a.Commits) == 0 || len(a.Commits[0].SHA) != 40 || !strings.HasPrefix(a.Commits[0].SHA, commit) { + return "", fmt.Errorf("the answer of %s for %s is not a commit that starts with it: %v", commitsTool, commit, err) + } + return a.Commits[0].SHA, nil +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge_test.go index fbb84d8..2b1f2b1 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge_test.go @@ -53,3 +53,18 @@ func TestTheForgeReadsParseWhatTheForgeAnswered(t *testing.T) { // 1624/AC4 t.Error("an answer with no total was taken") } } + +func TestTheForgeNamesTheWholeShaOfAShortCommit(t *testing.T) { // review D2 + f := forgeReads{call: func(key string, body any) (json.RawMessage, error) { + if key != commitsTool { + t.Errorf("called %s", key) + } + return capturedAnswer(t, "forge-commits"), nil + }} + if sha, err := f.FullSHA(context.Background(), "novox/mesh-sdk", "cb54756c3865"); err != nil || sha != "cb54756c3865df5ac9c0dc0481ad9ebe2bf013a1" { + t.Errorf("sha = %q, %v", sha, err) + } + if _, err := f.FullSHA(context.Background(), "novox/mesh-sdk", "deadbeefdead"); err == nil { + t.Error("a commit that does not start with the pin was taken") + } +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/hold.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/hold.go index dbbbfc7..1518bd7 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/hold.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/hold.go @@ -11,15 +11,18 @@ import ( "mesh-delivery-checker/internal/check" ) -// holdWatch holds the checker back while a delivery builds (ADR 0484 clause 12): it watches the controller's bucket of the -// delivery plans in their build phase and reads it as empty where it is absent (issue 1652 makes it) or cannot be read, -// saying so once. The bucket may come into being after the checker started and the checker holds no timer (ADR 0437 §8), so -// each time a check is about to start and the bucket is not watched yet, the checker looks for it. +// holdWatch holds the checker back while a delivery builds (ADR 0484 clause 12): whether one does is the list of the keys of +// the controller's bucket of the delivery plans in their build phase, read each time a check is about to start. Its keys age +// out (a time to live, and no delete marker), which a watch never hears, so a watch cannot decide it (mesh-controller +// internal/link/holds.go lists the keys too); the watch is only the wake-up, for a plan that ended by a delete. The bucket +// is read as empty where it is absent (issue 1652 makes it) or cannot be read, saying so once. It may come into being after +// the checker started and the checker holds no timer (ADR 0437 §8), so each time a check is about to start and the bucket is +// not known yet, the checker looks for it. type holdWatch struct { ctx context.Context js bucketOpener - hold check.PlanHold - changed func(ctx context.Context) // the last plan ended: the checks that waited may start + keys func(ctx context.Context) ([]string, error) + changed func(ctx context.Context) // a plan key changed: the checks that waited may start logger *log.Logger errs chan error mu sync.Mutex @@ -34,23 +37,37 @@ func newHoldWatch(ctx context.Context, js bucketOpener, changed func(context.Con // Held says whether a delivery builds now. func (w *holdWatch) Held() bool { w.ensure() - return w.hold.Held() + w.mu.Lock() + keys := w.keys + w.mu.Unlock() + if keys == nil { + return false + } + list, err := keys(w.ctx) + if err != nil { + w.say("the keys of %s cannot be listed, so it is read as empty: %v", check.PlansBuildingBucket, err) + return false + } + return len(list) > 0 } func (w *holdWatch) say(format string, args ...any) { + w.mu.Lock() + defer w.mu.Unlock() if !w.said { w.said = true w.logger.Printf(format, args...) } } -// ensure starts the watch if the bucket exists and is not watched. +// ensure finds the bucket if it is not known and exists, and starts the wake-up watch. func (w *holdWatch) ensure() { w.mu.Lock() - defer w.mu.Unlock() - if w.watched { + if w.watched || w.js == nil { + w.mu.Unlock() return } + w.mu.Unlock() kv, err := w.js.KeyValue(w.ctx, check.PlansBuildingBucket) switch { case errors.Is(err, jetstream.ErrBucketNotFound): @@ -60,17 +77,25 @@ func (w *holdWatch) ensure() { w.say("the bucket %s cannot be read, so it is read as empty: %v", check.PlansBuildingBucket, err) return } - watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.hold.Put(e.Key(), !opOf(e)) }) - if err != nil { - w.say("the bucket %s cannot be watched, so it is read as empty: %v", check.PlansBuildingBucket, err) + w.mu.Lock() + defer w.mu.Unlock() + if w.watched { return } w.watched, w.said = true, false + w.keys = func(ctx context.Context) ([]string, error) { + list, err := kv.Keys(ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return list, err + } go func() { - w.errs <- follow(w.ctx, watcher, func(e jetstream.KeyValueEntry) { - if w.hold.Put(e.Key(), !opOf(e)) && !w.hold.Held() { - w.changed(w.ctx) - } - }) + wake, err := kv.WatchAll(w.ctx) + if err != nil { + w.errs <- err + return + } + w.errs <- follow(w.ctx, wake, func(jetstream.KeyValueEntry) { w.changed(w.ctx) }) }() } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/hold_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/hold_test.go new file mode 100644 index 0000000..3f15b31 --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/hold_test.go @@ -0,0 +1,35 @@ +package main + +import ( + "context" + "errors" + "io" + "log" + "testing" +) + +// The controller's plans-building keys age out of the bucket (a time to live, no delete marker), which a watch never hears: +// whether a delivery builds is the list of the keys now, and the watch is only a wake-up (review D3). +func TestADeliveryHoldsWhileItsKeyIsListedAndNotWhenItAgedOutWithNoEvent(t *testing.T) { + listed := []string{"plan-1"} + w := &holdWatch{watched: true, logger: log.New(io.Discard, "", 0), keys: func(context.Context) ([]string, error) { return listed, nil }} + if !w.Held() { + t.Fatal("a plan key is listed and nothing holds") + } + listed = nil // the key aged out: no delete reached the watch + if w.Held() { + t.Error("the key is gone from the bucket and the checker is still held") + } +} + +func TestABucketThatCannotBeListedIsReadAsEmpty(t *testing.T) { + w := &holdWatch{watched: true, logger: log.New(io.Discard, "", 0), keys: func(context.Context) ([]string, error) { return []string{"plan-1"}, errors.New("the bus is down") }} + if w.Held() { + t.Error("an unreadable bucket held the checker") + } + var absent holdWatch + absent.logger = log.New(io.Discard, "", 0) + if absent.Held() { + t.Error("a watch with no bucket held the checker") + } +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go index b2a909f..6dc5381 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go @@ -31,7 +31,7 @@ type judgeDeps struct { func newJudge(d judgeDeps) check.Judge { return &check.Examiner{Steps: d.runner, Toolchains: d.tools, Presence: d.presence, Health: d.have, Places: d.places, - Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Forge: d.forge, Builds: d.builds, Announce: d.announce, Trunks: d.trunks, Logf: d.logf} + Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Forge: d.forge, Runs: d.repos, Builds: d.builds, Announce: d.announce, Trunks: d.trunks, Logf: d.logf} } // manifestKV reads a manifest by its digest. @@ -49,6 +49,7 @@ type repoModules struct { mu sync.Mutex langs map[string]map[string]bool // by manifest digest: the languages of its artifacts repoOf map[string]string // by module: its repository + commit map[string]string // by module: the commit its record was computed at (the commit the mesh runs) typed map[string]string // by module: its manifest digest } @@ -56,11 +57,12 @@ type repoModules struct { func (r *repoModules) Put(ctx context.Context, key string, value []byte, deleted bool) { r.mu.Lock() if r.repoOf == nil { - r.repoOf, r.typed, r.langs = map[string]string{}, map[string]string{}, map[string]map[string]bool{} + r.repoOf, r.typed, r.langs, r.commit = map[string]string{}, map[string]string{}, map[string]map[string]bool{}, map[string]string{} } if deleted { delete(r.repoOf, key) delete(r.typed, key) + delete(r.commit, key) r.mu.Unlock() return } @@ -72,6 +74,7 @@ func (r *repoModules) Put(ctx context.Context, key string, value []byte, deleted r.mu.Lock() r.repoOf[key] = m.Repository r.typed[key] = m.Manifest + r.commit[key] = m.Commit _, known := r.langs[m.Manifest] r.mu.Unlock() if m.Manifest == "" || known { @@ -128,3 +131,10 @@ func (r buildReader) Build(ctx context.Context, module, fp2 string) (delivery.Bu b, err := delivery.DecodeBuilderBuild(value) return b, err == nil, err } + +// CommitOf is the commit the indexer's record of the module was computed at: the commit of its repository the mesh runs. +func (r *repoModules) CommitOf(module string) string { + r.mu.Lock() + defer r.mu.Unlock() + return r.commit[module] +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go index ba3c1c0..28329bf 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go @@ -66,6 +66,9 @@ func TestARepositoryWithATypeScriptModuleIsJudgedByTheNodeBuildModuleToo(t *test if got := r.BuildModules("novox/mesh-catalog"); strings.Join(got, ",") != buildrun.GoBuildModule+","+buildrun.NodeBuildModule { t.Errorf("mesh-catalog: %v, want both", got) } + if got := r.CommitOf("mesh-delivery-checker"); got != strings.Repeat("a", 40) || r.CommitOf("gone") != "" { + t.Errorf("the commit the mesh runs = %q", got) + } if got := r.BuildModules("novox/mesh-sdk"); strings.Join(got, ",") != buildrun.GoBuildModule { t.Errorf("a repository with no module: %v, want the Go build module", got) } diff --git a/modules/mesh-delivery-checker/internal/check/beside.go b/modules/mesh-delivery-checker/internal/check/beside.go index f18ae1a..32ba99a 100644 --- a/modules/mesh-delivery-checker/internal/check/beside.go +++ b/modules/mesh-delivery-checker/internal/check/beside.go @@ -1,6 +1,10 @@ package check -import "strconv" +import ( + "context" + "fmt" + "strconv" +) // BesideRef is a sibling repository a repository's tests read, placed beside the fetched tree as a plain directory (ADR // 0484 clause 4). Ref names the pull request's head when the commit is only there. @@ -10,37 +14,74 @@ type BesideRef struct { Ref string `json:"ref,omitempty"` } +const ( + catalogueRepository = "novox/mesh-catalog" + nodeEngineRepo = "novox/mesh-host" + nodeEngineModule = "mesh-host" + controllerRepo = "novox/mesh-controller" +) + // besideRules are the repositories each core repository's tests read, as the tests state it (mesh-sdk go/internal/beside -// beside.go): the catalogue at its trunk, the node-engine, the SDK as the controller pins it, and the delivery's pinned -// forge dependencies (mesh-catalog checks/deps). The checker chooses which sibling at which commit; the build module only -// places what it is told. +// beside.go): the catalogue, the node-engine, the SDK, and the delivery's pinned forge dependencies (mesh-catalog +// checks/deps). Which commit of each is besideCommit's rule. var besideRules = map[string][]string{ - "novox/mesh-controller": {"novox/mesh-catalog", "novox/mesh-host", "novox/mesh-sdk"}, - "novox/mesh-host": {"novox/mesh-catalog", "novox/mesh-sdk"}, - "novox/mesh-sdk": {"novox/mesh-catalog", "novox/mesh-host"}, - "novox/mesh-catalog": {"novox/mesh-sdk", "novox/mesh-controller"}, - "novox/mesh-delivery": {"novox/mesh-sdk", "novox/mesh-catalog"}, + controllerRepo: {catalogueRepository, nodeEngineRepo, SDKRepository}, + nodeEngineRepo: {catalogueRepository, SDKRepository}, + SDKRepository: {catalogueRepository, nodeEngineRepo}, + catalogueRepository: {SDKRepository, controllerRepo}, + "novox/mesh-delivery": {SDKRepository, catalogueRepository}, } -// Beside are the siblings a pull request's tests read: each at its trunk head, and at the head of the group's pull request -// when the group holds one of that repository (a group's members are checked together, ADR 0482 clause 2) — except the -// catalogue, which the tests read at its trunk even when the group holds a catalogue pull request (beside.go). -func (j *Examiner) Beside(c *Case, p Pull) []BesideRef { +// besideFor are the siblings a pull request's tests read (ADR 0484 clause 4): the catalogue at its trunk, always, even in a +// group holding a catalogue pull request; the node-engine at the commit the mesh runs (the commit of the indexer's module +// mesh-host, its trunk head when none is known); mesh-sdk, for the controller's tests, at the commit the controller's go.mod +// pins; any other sibling at the head of the group's pull request for it when the group holds one, else at its trunk. +func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef, error) { var heads map[string]string if j.Trunks != nil { heads = j.Trunks.Heads() } var out []BesideRef for _, repo := range besideRules[p.Repository] { - if member := c.member(repo); member != nil && repo != "novox/mesh-catalog" { - out = append(out, BesideRef{Repository: repo, Commit: member.Head, Ref: refOf(member.Number)}) - continue - } - if head := heads[repo]; head != "" { - out = append(out, BesideRef{Repository: repo, Commit: head}) + switch { + case repo == catalogueRepository: + if head := heads[repo]; head != "" { + out = append(out, BesideRef{Repository: repo, Commit: head}) + } + case repo == nodeEngineRepo: + commit := "" + if j.Runs != nil { + commit = j.Runs.CommitOf(nodeEngineModule) + } + if commit == "" { + commit = heads[repo] + } + if commit != "" { + out = append(out, BesideRef{Repository: repo, Commit: commit}) + } + case repo == SDKRepository && p.Repository == controllerRepo && j.Forge != nil: + text, err := j.Forge.File(ctx, p.Repository, "go.mod", p.Head) + if err != nil { + return nil, fmt.Errorf("go.mod of %s#%d cannot be read to place the mesh-sdk its tests read: %w", p.Repository, p.Number, err) + } + pin, ok := SDKPin(text) + if !ok { + return nil, fmt.Errorf("go.mod of %s#%d names no mesh-sdk commit, so the mesh-sdk its tests read cannot be placed", p.Repository, p.Number) + } + full, err := j.Forge.FullSHA(ctx, SDKRepository, pin) + if err != nil { + return nil, fmt.Errorf("mesh-sdk %s, which %s#%d pins, cannot be found: %w", pin, p.Repository, p.Number, err) + } + out = append(out, BesideRef{Repository: repo, Commit: full}) + default: + if member := c.member(repo); member != nil { + out = append(out, BesideRef{Repository: repo, Commit: member.Head, Ref: refOf(member.Number)}) + } else if head := heads[repo]; head != "" { + out = append(out, BesideRef{Repository: repo, Commit: head}) + } } } - return out + return out, nil } func (c *Case) member(repository string) *Pull { @@ -56,7 +97,5 @@ func refOf(number int) string { if number <= 0 { return "" } - return "refs/pull/" + itoa(number) + "/head" + return "refs/pull/" + strconv.Itoa(number) + "/head" } - -func itoa(n int) string { return strconv.Itoa(n) } diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go index ec6c3aa..66edd34 100644 --- a/modules/mesh-delivery-checker/internal/check/examiner_test.go +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -150,7 +150,7 @@ func caseOf(t testing.TB, pulls ...delivery.CandidatePull) *Case { if err != nil { t.Fatal(err) } - cs := CaseOf(c, map[string]string{repoA: trunkA, repoSDK: trunkS}) + cs := CaseOf(c, map[string]string{repoA: trunkA, repoSDK: trunkS, "novox/mesh-controller": trunkA, "novox/mesh-catalog": trunkA}) if cs == nil { t.Fatal("no case") } @@ -552,3 +552,36 @@ func TestTheOutcomeSaysIfEachCandidatesModuleIsBuiltFailedOrWaiting(t *testing.T t.Errorf("builds = %v, want one note for each module that has a fingerprint, sorted", got) } } + +// trunksOf are trunk heads of every repository, as the gate's source names them. +func trunksOf(t *testing.T, heads map[string]string) *Trunks { + t.Helper() + data, err := delivery.EncodeSource(delivery.Source{Group: "g", Heads: heads}) + if err != nil { + t.Fatal(err) + } + tr := &Trunks{} + tr.Put(data, false) + return tr +} + +func TestNothingIsKeptForAWorkspaceGivenSiblingsBeside(t *testing.T) { // review D1, ADR 0444, issue 1287 + b := newBench(t) + b.ex.Trunks = trunksOf(t, map[string]string{repoSDK: trunkS, "novox/mesh-catalog": trunkA}) + b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + if got := len(b.steps.tool("fetch")); got != 1 || b.steps.tool("fetch")[0].args["beside"] == nil { + t.Fatalf("the fetch was not given siblings: %+v", b.steps.tool("fetch")) + } + if keys, _ := b.kept.Keys(context.Background()); len(keys) != 0 { + t.Errorf("kept %v for a package whose tests may read a sibling the digest does not cover", keys) + } + // And a kept verdict of the same digest from elsewhere is not taken for it: everything runs. + if _, err := (Kept{Store: b.kept}).Keep(context.Background(), "unit", digA, keptValue("fixture/slow", 2)); err != nil { + t.Fatal(err) + } + b.steps.calls = nil + b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + if n := len(b.steps.tool("test")); n != 2 { + t.Errorf("%d test steps, want both levels run whole", n) + } +} diff --git a/modules/mesh-delivery-checker/internal/check/hold.go b/modules/mesh-delivery-checker/internal/check/hold.go index eb8984b..5725a8d 100644 --- a/modules/mesh-delivery-checker/internal/check/hold.go +++ b/modules/mesh-delivery-checker/internal/check/hold.go @@ -1,40 +1,7 @@ package check -import "sync" - // PlansBuildingBucket is the controller's bucket of the delivery plans in their build phase, one key for each plan, written // by the controller alone (issue 1652): a plan key there means a delivery builds on the build nodes, and no new check starts -// meanwhile (ADR 0484 clause 12, ADR 0483 clause 6). The bucket may be absent: the controller's issue 1652 is delivered +// meanwhile (ADR 0484 clause 12, ADR 0483 clause 6). The keys age out with no delete marker, so the keys are listed, not watched. The bucket may be absent: the controller's issue 1652 is delivered // before R4c-5b, and if it is not, the bucket is read as empty. const PlansBuildingBucket = "mesh-controller_plans-building" - -// PlanHold holds which delivery plans build now, from the controller's bucket (a watch puts the changes in). Safe for use -// from the watch and from a check at once. -type PlanHold struct { - mu sync.Mutex - plans map[string]bool -} - -// Put takes a change of the bucket in: a put of a plan's key, or its delete. It answers whether a delivery builds now -// differently than before, so a watch wakes the checker when the last plan ended. -func (h *PlanHold) Put(key string, deleted bool) (changed bool) { - h.mu.Lock() - defer h.mu.Unlock() - before := len(h.plans) > 0 - if deleted { - delete(h.plans, key) - } else { - if h.plans == nil { - h.plans = map[string]bool{} - } - h.plans[key] = true - } - return before != (len(h.plans) > 0) -} - -// Held says whether any delivery plan builds now. -func (h *PlanHold) Held() bool { - h.mu.Lock() - defer h.mu.Unlock() - return len(h.plans) > 0 -} diff --git a/modules/mesh-delivery-checker/internal/check/hold_test.go b/modules/mesh-delivery-checker/internal/check/hold_test.go index 38cc8bc..35f6ef1 100644 --- a/modules/mesh-delivery-checker/internal/check/hold_test.go +++ b/modules/mesh-delivery-checker/internal/check/hold_test.go @@ -7,39 +7,17 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" ) -func TestADeliveryThatBuildsHoldsTheChecksAndTheEndOfTheLastPlanSaysSo(t *testing.T) { // N8/AC1 - var h PlanHold - if h.Held() { - t.Fatal("an empty bucket holds") - } - if !h.Put("plan-1", false) || !h.Held() { - t.Error("a first plan did not hold") - } - if h.Put("plan-2", false) { - t.Error("a second plan changed whether a delivery builds") - } - if h.Put("plan-1", true) || !h.Held() { - t.Error("one of two plans ending released the hold") - } - if !h.Put("plan-2", true) || h.Held() { - t.Error("the last plan ending did not release the hold") - } - if h.Put("plan-3", true) { - t.Error("the delete of a plan never put changed the hold") - } -} - func TestARunningCheckIsNotStoppedWhenADeliveryStartsToBuild(t *testing.T) { // ADR 0484 clause 12, ADR 0483 clause 4 ctx := context.Background() - var h PlanHold + held := false j := &judgeFunc{f: func(_ context.Context, _ *Case, call int) (Outcome, error) { if call == 1 { - h.Put("plan-1", false) // a delivery starts while this check runs + held = true // a delivery starts while this check runs } return Outcome{Verdict: passing()}, nil }} r := newRig(t, j) - r.ch.Hold = h.Held + r.ch.Hold = func() bool { return held } r.ch.Candidate(ctx, "feat/x", candidateOf(t, "feat/x", pullOf(repoA, 7, head1))) r.ch.Wait() if v := r.verdict(t, keyOf(t, "feat/x", candidateOf(t, "feat/x", pullOf(repoA, 7, head1)))); v.State != delivery.VerdictPass { @@ -51,7 +29,7 @@ func TestARunningCheckIsNotStoppedWhenADeliveryStartsToBuild(t *testing.T) { // if j.count() != 1 { t.Errorf("judged %d times while a delivery builds, want only the one already running", j.count()) } - h.Put("plan-1", true) + held = false r.ch.Reconsider(ctx) r.ch.Wait() if j.count() != 2 { diff --git a/modules/mesh-delivery-checker/internal/check/judge.go b/modules/mesh-delivery-checker/internal/check/judge.go index 49af8a4..8cdeb58 100644 --- a/modules/mesh-delivery-checker/internal/check/judge.go +++ b/modules/mesh-delivery-checker/internal/check/judge.go @@ -58,6 +58,8 @@ type Examiner struct { // Previews is the previews bucket; nil writes none. Previews KV Modules BuildModules + // Runs says which commit the mesh runs of a module (the indexer's modules); nil knows none. + Runs interface{ CommitOf(module string) string } // Forge reads the go.mod files of the Go members and says whether a mesh-sdk commit is on its trunk (ADR 0484 clause 9); // nil checks no pin. Forge Forge @@ -94,7 +96,9 @@ type target struct { fetched Fetched // priority are the Go packages (import paths) or the Node module directories whose test files the merge changes. changed []string - plans map[string]*goPlan // by level + // beside says the workspace was given siblings: its tests may read what the digest does not cover, so nothing is kept for it. + beside bool + plans map[string]*goPlan // by level } // goPlan is what a Go target has to run at one level: the digests, the packages without a kept verdict and the tests taken. @@ -238,8 +242,13 @@ func (st *state) fetchAll(ctx context.Context) *delivery.Verdict { st.nodes[node] = true args := map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number), "base": p.Trunk} if bm == buildrun.GoBuildModule { - if beside := st.j.Beside(st.c, p); len(beside) > 0 { + beside, err := st.j.besideFor(ctx, st.c, p) + if err != nil { + return fail(node, err.Error()) + } + if len(beside) > 0 { args["beside"] = beside + t.beside = true } } raw, err := st.step(ctx, t, "fetch", fmt.Sprintf("fetch %s#%d on %s", p.Repository, p.Number, node), args) @@ -364,6 +373,12 @@ func (st *state) planGo(ctx context.Context, t *target, level string) *delivery. } plan := &goPlan{digests: map[string]string{}} t.plans[level] = plan + if t.beside { + // The digest covers the package's own files, the toolchain and the header, not the siblings placed beside the tree: a + // package whose tests read one must never be kept, nor taken from a verdict kept for other inputs (ADR 0444). + plan.all = true + return nil + } raw, err := st.step(ctx, t, "digest", fmt.Sprintf("digest the %s level of %s#%d on %s", level, t.pull.Repository, t.pull.Number, t.node), map[string]any{"workspace": t.fetched.Workspace, "level": level}) var d Digested diff --git a/modules/mesh-delivery-checker/internal/check/pins.go b/modules/mesh-delivery-checker/internal/check/pins.go index 592195d..7ce5df6 100644 --- a/modules/mesh-delivery-checker/internal/check/pins.go +++ b/modules/mesh-delivery-checker/internal/check/pins.go @@ -29,6 +29,8 @@ type Forge interface { // OnTrunk says whether commit (a full or a 12-digit sha) is already merged into the trunk of the repository: it holds no // commit the trunk does not. OnTrunk(ctx context.Context, repository, commit string) (bool, error) + // FullSHA is the 40-digit sha of a commit named by its first 12 digits. + FullSHA(ctx context.Context, repository, commit string) (string, error) } var pseudoVersion = regexp.MustCompile(`-([0-9a-f]{12})$`) diff --git a/modules/mesh-delivery-checker/internal/check/pins_test.go b/modules/mesh-delivery-checker/internal/check/pins_test.go index 3ffdc06..065dc56 100644 --- a/modules/mesh-delivery-checker/internal/check/pins_test.go +++ b/modules/mesh-delivery-checker/internal/check/pins_test.go @@ -29,6 +29,13 @@ func (f *fakeForge) File(_ context.Context, repository, path, ref string) (strin return "", fmt.Errorf("no file %s of %s at %s", path, repository, ref) } +func (f *fakeForge) FullSHA(_ context.Context, _ string, commit string) (string, error) { + if f.err != nil { + return "", f.err + } + return commit + strings.Repeat("0", 40-len(commit)), nil +} + func (f *fakeForge) OnTrunk(_ context.Context, _ string, commit string) (bool, error) { if f.err != nil { return false, f.err @@ -145,3 +152,53 @@ func TestAPinThatCannotBeCheckedFailsAndNeverPasses(t *testing.T) { // ADR 0453 t.Errorf("verdict = %+v", v) } } + +type runsAt map[string]string + +func (r runsAt) CommitOf(module string) string { return r[module] } + +func besideOf(t *testing.T, b *bench, repository string) map[string]BesideRef { + t.Helper() + out := map[string]BesideRef{} + for _, f := range b.steps.tool("fetch") { + if f.args["repository"] == repository { + refs, _ := f.args["beside"].([]BesideRef) + for _, r := range refs { + out[r.Repository] = r + } + } + } + return out +} + +func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR 0484 clause 4 + const controller, catalogue, host = "novox/mesh-controller", "novox/mesh-catalog", "novox/mesh-host" + hostRuns := strings.Repeat("7", 40) + b := newBench(t) + b.ex.Trunks = trunksOf(t, map[string]string{controller: trunkA, catalogue: trunkS, host: trunkA2, SDKRepository: "9999999999999999999999999999999999999999"}) + b.ex.Runs = runsAt{"mesh-host": hostRuns} + forge := &fakeForge{goMods: map[string]string{controller + " go.mod " + head1: capturedGoMod(t, "cb54756c3865")}, onTrunk: map[string]bool{"cb54756c3865": true}} + b.ex.Forge = forge + // A group of the controller and the catalogue: the controller's tests read the catalogue's trunk, not the group's head. + b.judge(t, caseOf(t, pullOf(controller, 7, head1), pullOf(catalogue, 12, head2))) + got := besideOf(t, b, controller) + if c := got[catalogue]; c.Commit != trunkS || c.Ref != "" { + t.Errorf("the catalogue beside the controller = %+v, want its trunk %s", c, trunkS) + } + if h := got[host]; h.Commit != hostRuns { + t.Errorf("the node-engine beside the controller = %+v, want the commit the mesh runs %s", h, hostRuns) + } + if s := got[SDKRepository]; s.Commit != "cb54756c3865"+strings.Repeat("0", 28) { + t.Errorf("mesh-sdk beside the controller = %+v, want the commit its go.mod pins, whole", s) + } +} + +func TestThePinOfTheControllerThatCannotBeReadFailsTheCheck(t *testing.T) { // review D2 + b := newBench(t) + b.ex.Trunks = trunksOf(t, map[string]string{"novox/mesh-controller": trunkA}) + b.ex.Forge = &fakeForge{goMods: map[string]string{}, onTrunk: map[string]bool{}} + v := b.judge(t, caseOf(t, pullOf("novox/mesh-controller", 7, head1))).Verdict + if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "go.mod") { + t.Errorf("verdict = %+v", v) + } +} diff --git a/modules/mesh-delivery-checker/internal/check/testdata/captured/README b/modules/mesh-delivery-checker/internal/check/testdata/captured/README index 9c809ea..58a30ed 100644 --- a/modules/mesh-delivery-checker/internal/check/testdata/captured/README +++ b/modules/mesh-delivery-checker/internal/check/testdata/captured/README @@ -17,3 +17,5 @@ novox/mesh-delivery at main, and forge-compare-trunk.json the answer of gitea_co trunk: no commit main does not hold); both called through the mesh on 2026-10-21 for issue 1624. forge-compare-branch.json is the answer of the same call for a head that is not on the trunk (novox/mesh-sdk base main, head afbe371003a5, the branch feat/r4c5-sdk), shortened to two of its seven commits and one of its twenty-six files, with the head and the totals set to match. + +forge-commits.json is the answer of the forge's gitea_commits for novox/mesh-sdk, ref cb54756c3865, limit 1, called through the mesh on 2026-10-21 (review D2). diff --git a/modules/mesh-delivery-checker/internal/check/testdata/captured/forge-commits.json b/modules/mesh-delivery-checker/internal/check/testdata/captured/forge-commits.json new file mode 100644 index 0000000..813ec62 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/testdata/captured/forge-commits.json @@ -0,0 +1,14 @@ +{ + "commits": [ + { + "author": "mesh-tools", + "date": "2026-10-21T10:02:09Z", + "sha": "cb54756c3865df5ac9c0dc0481ad9ebe2bf013a1", + "subject": "Merge pull request 'Give delivery.Module the toolchains its fingerprint was computed with, and '.' as a root module's path, so the builder builds a key only with its own toolchain (issue 1615)' (#55) from fix/1615-module-toolchains-root into main" + } + ], + "more": true, + "page": 1, + "ref": "cb54756c3865", + "repository": "novox/mesh-sdk" +} diff --git a/modules/mesh-delivery-checker/manifest_test.go b/modules/mesh-delivery-checker/manifest_test.go index 5370e5c..c3d4592 100644 --- a/modules/mesh-delivery-checker/manifest_test.go +++ b/modules/mesh-delivery-checker/manifest_test.go @@ -100,7 +100,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } - if want := []string{"gitea.gitea_compare", "gitea.gitea_get_file", "mesh-build-go.build_go_digest", "mesh-build-go.build_go_fetch", "mesh-build-go.build_go_test", "mesh-build-node.build_node_fetch", "mesh-build-node.build_node_test"}; !reflect.DeepEqual(sorted(m.Invokes), want) { + if want := []string{"gitea.gitea_commits", "gitea.gitea_compare", "gitea.gitea_get_file", "mesh-build-go.build_go_digest", "mesh-build-go.build_go_fetch", "mesh-build-go.build_go_test", "mesh-build-node.build_node_fetch", "mesh-build-node.build_node_test"}; !reflect.DeepEqual(sorted(m.Invokes), want) { t.Errorf("invokes = %v, want only the build modules' fetch, digest and test (no run step before issue 1309, ADR 0484 clause 5) and the forge's two reads for a mesh-sdk pin (clause 9)", m.Invokes) } // The checker is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it starts. diff --git a/modules/mesh-delivery-checker/module.json b/modules/mesh-delivery-checker/module.json index 80b7351..87c1005 100644 --- a/modules/mesh-delivery-checker/module.json +++ b/modules/mesh-delivery-checker/module.json @@ -35,6 +35,7 @@ "checker_verdicts" ], "invokes": [ + "gitea.gitea_commits", "gitea.gitea_compare", "gitea.gitea_get_file", "mesh-build-go.build_go_digest",