From f555a05cfb39dc467f51510ff9036611e2d40806 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 11:36:32 +0200 Subject: [PATCH] Build a key only with the toolchain its fingerprint names, accept '.' as a root module's path, and choose only online nodes (issues 1615, 1607 review) --- .../mesh-delivery-builder/integration_test.go | 75 +++++++++++-- .../cmd/mesh-delivery-builder/main.go | 25 +++++ .../internal/build/choose.go | 24 +++- .../internal/build/choose_test.go | 24 +++- .../internal/build/presence.go | 53 +++++++++ .../internal/build/presence_test.go | 43 ++++++++ .../internal/build/run.go | 41 ++++++- .../internal/build/run_test.go | 104 +++++++++++++++--- .../mesh-delivery-builder/manifest_test.go | 2 +- modules/mesh-delivery-builder/module.json | 3 +- 10 files changed, 356 insertions(+), 38 deletions(-) create mode 100644 modules/mesh-delivery-builder/internal/build/presence.go create mode 100644 modules/mesh-delivery-builder/internal/build/presence_test.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index eeaa473..2131046 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -193,6 +193,13 @@ func startBed(t *testing.T) *bed { b.modules = mk(delivery.ModulesBucket, delivery.ModulesHistory) mk(delivery.ManifestsBucket, delivery.ManifestsHistory) b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory) + presence := mk(delivery.PresenceBucket, delivery.PresenceHistory) + for _, node := range []string{"n1", "n2"} { + data, _ := delivery.EncodePresence(delivery.Presence{Online: true}) + if _, err := presence.Put(ctx, node, data); err != nil { + t.Fatal(err) + } + } b.fake = &fakeBuildModules{js: js, ctx: ctx, calls: map[string]int{}} b.fake.serve(t, ctl, "mesh-build-go") b.fake.serve(t, ctl, "mesh-build-node") @@ -221,6 +228,15 @@ func (b *bed) report(t *testing.T, module, node, language, version, recipe strin // module writes modules. as the indexer does for a module with a new-form fingerprint, and its manifest by digest. func (b *bed) module(t *testing.T, name, fixture, path string) string { + return b.moduleWith(t, name, fixture, path, map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}}) +} + +// moduleWith is module with the toolchains its fingerprint was computed with. +func (b *bed) moduleWith(t *testing.T, name, fixture, path string, toolchains map[string]delivery.ToolchainUsed) string { + return b.moduleCommit(t, name, fixture, path, toolchains, commit) +} + +func (b *bed) moduleCommit(t *testing.T, name, fixture, path string, toolchains map[string]delivery.ToolchainUsed, at string) string { t.Helper() raw, err := os.ReadFile(filepath.Join("testdata", fixture)) if err != nil { @@ -238,8 +254,8 @@ func (b *bed) module(t *testing.T, name, fixture, path string) string { if _, err := manifests.Put(b.ctx, digest, mv); err != nil { t.Fatal(err) } - data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: commit, FP: "src1:" + strings.Repeat("f", 64), FP2: fp, - Manifest: digest, Path: path}) + data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: at, FP: "src1:" + strings.Repeat("f", 64), FP2: fp, + Manifest: digest, Path: path, Toolchains: toolchains}) if err != nil { t.Fatal(err) } @@ -334,10 +350,10 @@ func TestASecondBuilderInstanceDoesNotBuildTheSameSourceAgain(t *testing.T) { // } } -// TestAFailedKeyIsBuiltAgainAsANewRevisionWhenAToolchainChanges: a build step fails on one node, so the key is failed -// naming it; a toolchain report of a node then wakes the builder, which builds the key again as a new revision of the same -// key, now done. -func TestAFailedKeyIsBuiltAgainAsANewRevisionWhenAToolchainChanges(t *testing.T) { // 1607/AC6, AC7 +// TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFingerprint: a build step fails on one node, so the +// key is failed naming it; the indexer writing modules. again with the same fingerprint wakes the builder, which builds +// the key again as a new revision of the same key, now done. +func TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFingerprint(t *testing.T) { // 1607/AC6, AC7 b := startBed(t) b.fake.failBuild.Store(true) b.module(t, "ca-trust", "catalogue-a67f0196-ca-trust.module.json", "modules/ca-trust") @@ -352,9 +368,8 @@ func TestAFailedKeyIsBuiltAgainAsANewRevisionWhenAToolchainChanges(t *testing.T) t.Fatal(err) } b.fake.failBuild.Store(false) - // The build module's recipe moved on both nodes: the toolchain's entries changed, twice (between the two the nodes disagree). - b.report(t, "mesh-build-go", "n1", "go", "go1.27.1", "2") - b.report(t, "mesh-build-go", "n2", "go", "go1.27.1", "2") + // The indexer writes the module again (here at a later commit), with the same fingerprint. + b.moduleAt(t, "ca-trust", "catalogue-a67f0196-ca-trust.module.json", "modules/ca-trust", "go1.27.1", "89abcdef0123456789abcdef0123456789abcdef") b.await(t, "ca-trust", isState(delivery.BuildDone)) done, err := b.builds.Get(b.ctx, key) if err != nil { @@ -364,3 +379,45 @@ func TestAFailedKeyIsBuiltAgainAsANewRevisionWhenAToolchainChanges(t *testing.T) t.Errorf("revision %d, was %d: the key was not written as a new revision", done.Revision(), failed.Revision()) } } + +// TestAToolchainChangeDoesNotBuildTheOldFingerprint: the fingerprint names go1.27.1, the nodes then agree on go1.27.2 (a +// report of both nodes), and the builder writes no key for the old fingerprint although it was woken by the report; a +// module value naming go1.27.2 (the new fingerprint, computed by the indexer) is built. +func TestAToolchainChangeDoesNotBuildTheOldFingerprint(t *testing.T) { // 1615/AC4 + b := startBed(t) + b.report(t, "mesh-build-go", "n1", "go", "go1.27.2", "1") + b.report(t, "mesh-build-go", "n2", "go", "go1.27.2", "1") + b.module(t, "ca-trust", "catalogue-a67f0196-ca-trust.module.json", "modules/ca-trust") // names go1.27.1 + b.start(t, "n1", nil) + // Another module, whose fingerprint names the new toolchain, is built; by then the old one has been looked at. + b.moduleAt(t, "ca-trust2", "catalogue-a67f0196-ca-trust.module.json", "modules/ca-trust", "go1.27.2", commit) + b.await(t, "ca-trust2", isState(delivery.BuildDone)) + if _, err := b.builds.Get(b.ctx, mustKey(t, "ca-trust")); err == nil { + t.Error("the old fingerprint, naming go1.27.1, was built with go1.27.2") + } + if n := b.fake.count("mesh-build-go.build_go_build"); n != 2 { + t.Errorf("build_go_build was called %d times, want 2 (the new fingerprint on two nodes only)", n) + } +} + +// TestARootModuleIsBuilt: a module whose path is "." is given to the build tools as it is. +func TestARootModuleIsBuilt(t *testing.T) { // 1615/AC4 + b := startBed(t) + b.module(t, "ca-trust", "catalogue-a67f0196-ca-trust.module.json", delivery.RootPath) + b.start(t, "n1", nil) + b.await(t, "ca-trust", isState(delivery.BuildDone)) +} + +func mustKey(t *testing.T, module string) string { + k, err := delivery.BuilderBuildKey(module, fp) + if err != nil { + t.Fatal(err) + } + return k +} + +// moduleAt is module with the go toolchain version the fingerprint names and the commit it was computed at. +func (b *bed) moduleAt(t *testing.T, name, fixture, path, goVersion, at string) { + t.Helper() + b.moduleCommit(t, name, fixture, path, map[string]delivery.ToolchainUsed{"go": {Version: goVersion, Recipe: "1"}}, at) +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index c85cfe4..c3d70df 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -159,10 +159,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } } tools := &build.ToolchainSet{} + presence := &build.PresenceSet{} b := &build.Builder{ Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module}, Runner: build.Runner{Caller: caller, Steps: stepsWaiter{js}, Bound: bound}, Toolchains: tools, + Presence: presence, Manifests: manifestReader{manifests}, Nodes: nodesPerFingerprint, Logf: logger.Printf, @@ -189,6 +191,27 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } defer cc.Stop() } + // The watchman's presence says which nodes are asleep. Where its bucket is not there yet no node is online, so nothing + // builds; the builder says so and is started again with the watchman. + presenceErrs := make(chan error, 1) + switch kv, err := js.KeyValue(ctx, delivery.PresenceBucket); { + case errors.Is(err, jetstream.ErrBucketNotFound): + logger.Printf("the bucket %s does not exist yet: no node counts as online until the builder starts again", delivery.PresenceBucket) + case err != nil: + return fmt.Errorf("the bucket %s: %w", delivery.PresenceBucket, err) + default: + pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) }) + if err != nil { + return err + } + go func() { + presenceErrs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) { + if presence.Put(e.Key(), e.Value(), !opOf(e)) { + b.Reconsider(ctx) // a node came online or went: a build that waited for one may go on + } + }) + }() + } each := func(e jetstream.KeyValueEntry) { if opOf(e) { b.Module(ctx, e.Key(), e.Value()) @@ -211,5 +234,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err case err := <-watch.errs: return err + case err := <-presenceErrs: + return err } } diff --git a/modules/mesh-delivery-builder/internal/build/choose.go b/modules/mesh-delivery-builder/internal/build/choose.go index c4cf30d..88e09d3 100644 --- a/modules/mesh-delivery-builder/internal/build/choose.go +++ b/modules/mesh-delivery-builder/internal/build/choose.go @@ -3,28 +3,40 @@ package build import ( "fmt" "sort" + "strings" "git.novox.be/novox/mesh-sdk/go/delivery" ) // ChooseNodes picks up to want build nodes for a language (ADR 0478 clause 3): the nodes whose toolchains key reports the // version the nodes agree on, the ones with the fewest steps the builder has running there first, a tie broken by the -// node's name. It answers the toolchain they agree on. When nothing is reported, or the nodes report different versions -// (so the indexer agreed none), it fails naming the language and why. The build module's health is not read here: a node -// whose module refuses on its toolchain refuses the step, and the build fails naming it. -func ChooseNodes(language string, entries []delivery.NodeToolchain, running map[string]int, want int) ([]string, delivery.Toolchain2, error) { +// node's name. A node qualifies only when online is true of it (the watchman's presence of the node; nil means every +// node is). It answers the toolchain all the reporting nodes agree on, online or not, as the indexer computes it. When +// nothing is reported, or the nodes report different versions (so the indexer agreed none), it fails naming the language +// and why; when no reporting node is online it fails naming the nodes. A node whose build module refuses on its toolchain +// refuses the step, and the build fails naming it. +func ChooseNodes(language string, entries []delivery.NodeToolchain, running map[string]int, online func(node string) bool, want int) ([]string, delivery.Toolchain2, error) { tc, none := delivery.AgreedToolchain(entries) if none != "" { return nil, delivery.Toolchain2{}, fmt.Errorf("no %s build node qualifies: %s", language, none) } nodes := make([]string, 0, len(entries)) + var reporting []string seen := map[string]bool{} for _, e := range entries { - if !seen[e.Node] { - seen[e.Node] = true + if seen[e.Node] { + continue + } + seen[e.Node] = true + reporting = append(reporting, e.Node) + if online == nil || online(e.Node) { nodes = append(nodes, e.Node) } } + if len(nodes) == 0 { + sort.Strings(reporting) + return nil, tc, fmt.Errorf("no %s build node qualifies: %s report the toolchain, and none is online", language, strings.Join(reporting, ", ")) + } sort.Slice(nodes, func(i, j int) bool { if running[nodes[i]] != running[nodes[j]] { return running[nodes[i]] < running[nodes[j]] diff --git a/modules/mesh-delivery-builder/internal/build/choose_test.go b/modules/mesh-delivery-builder/internal/build/choose_test.go index c085dcd..7057f3f 100644 --- a/modules/mesh-delivery-builder/internal/build/choose_test.go +++ b/modules/mesh-delivery-builder/internal/build/choose_test.go @@ -17,7 +17,7 @@ func tcs(language, version string, nodes ...string) []delivery.NodeToolchain { func TestChooseTakesTheAgreedNodesWithFewestRunningStepsTiesByName(t *testing.T) { // 1607/AC4, ADR 0478 clause 3 entries := tcs("go", "go1.27.1", "novox", "ace", "g14", "shanks") - nodes, tc, err := ChooseNodes("go", entries, map[string]int{"novox": 3, "ace": 1, "g14": 1, "shanks": 0}, 2) + nodes, tc, err := ChooseNodes("go", entries, map[string]int{"novox": 3, "ace": 1, "g14": 1, "shanks": 0}, nil, 2) if err != nil { t.Fatal(err) } @@ -30,19 +30,19 @@ func TestChooseTakesTheAgreedNodesWithFewestRunningStepsTiesByName(t *testing.T) } func TestChooseWithOneNodeAnswersOneNode(t *testing.T) { // 1607/AC4 - nodes, _, err := ChooseNodes("go", tcs("go", "go1.27.1", "novox"), nil, 2) + nodes, _, err := ChooseNodes("go", tcs("go", "go1.27.1", "novox"), nil, nil, 2) if err != nil || len(nodes) != 1 || nodes[0] != "novox" { t.Errorf("nodes = %v, err %v, want [novox]", nodes, err) } } func TestChooseFailsNamingTheLanguageAndTheVersionsWhenNoNodeQualifies(t *testing.T) { // 1607/AC4 - _, _, err := ChooseNodes("typescript", nil, nil, 2) + _, _, err := ChooseNodes("typescript", nil, nil, nil, 2) if err == nil || !strings.Contains(err.Error(), "typescript") || !strings.Contains(err.Error(), delivery.ToolchainNoReport) { t.Errorf("no report: error = %v, want it to name typescript and %q", err, delivery.ToolchainNoReport) } entries := append(tcs("go", "go1.27.1", "novox"), tcs("go", "go1.26.0", "ace")...) - _, _, err = ChooseNodes("go", entries, nil, 2) + _, _, err = ChooseNodes("go", entries, nil, nil, 2) if err == nil || !strings.Contains(err.Error(), "go1.27.1") || !strings.Contains(err.Error(), "go1.26.0") || !strings.Contains(err.Error(), "go") { t.Errorf("disagreeing nodes: error = %v, want both versions named", err) } @@ -61,3 +61,19 @@ func TestAgreeNeedsEqualDigestsAndNamesBothNodesWhenTheyDiffer(t *testing.T) { / t.Errorf("one node: %q, %v", d, err) } } + +func TestChooseSkipsNodesThatAreNotOnlineAndFailsNamingThemWhenNoneIs(t *testing.T) { // 1607 review 3, ADR 0478 clause 3 + entries := tcs("go", "go1.27.1", "ace", "g14", "novox") + online := func(n string) bool { return n != "g14" } // a sleeping laptop + nodes, tc, err := ChooseNodes("go", entries, map[string]int{"novox": 5}, online, 2) + if err != nil || len(nodes) != 2 || nodes[0] != "ace" || nodes[1] != "novox" || tc.Version != "go1.27.1" { + t.Errorf("nodes = %v, toolchain %+v, err %v, want [ace novox] with g14 left out although it runs nothing", nodes, tc, err) + } + _, tc, err = ChooseNodes("go", entries, nil, func(string) bool { return false }, 2) + if err == nil || !strings.Contains(err.Error(), "online") || !strings.Contains(err.Error(), "g14") || !strings.Contains(err.Error(), "go") { + t.Errorf("none online: error = %v, want it to name the nodes that report and that none is online", err) + } + if tc.Version != "go1.27.1" { + t.Errorf("none online: the agreed toolchain %+v is still answered", tc) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/presence.go b/modules/mesh-delivery-builder/internal/build/presence.go new file mode 100644 index 0000000..b1131aa --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/presence.go @@ -0,0 +1,53 @@ +package build + +import ( + "sync" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// PresenceSet holds which nodes are online, as the watchman writes it (mesh-delivery-watchman_presence, key ``, the +// node-engine's connection): a node asleep or off the bus is offline and its build module cannot answer. A node the +// watchman has no key of is not online. Safe for use from the watch and from a build at once. +type PresenceSet struct { + mu sync.Mutex + online map[string]bool +} + +// Put takes a change of the presence bucket in and answers whether a node's online state changed. A part's key and a +// value that is not a presence are left out. +func (s *PresenceSet) Put(key string, value []byte, deleted bool) bool { + node, part, _, err := delivery.ParsePresenceKey(key) + if err != nil || part != "" { + return false + } + on := false + if !deleted { + p, err := delivery.DecodePresence(value) + if err != nil { + return false + } + on = p.Online + } + s.mu.Lock() + defer s.mu.Unlock() + if s.online[node] == on { + return false + } + if s.online == nil { + s.online = map[string]bool{} + } + if on { + s.online[node] = true + } else { + delete(s.online, node) + } + return true +} + +// Online is whether the watchman last saw the node online. +func (s *PresenceSet) Online(node string) bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.online[node] +} diff --git a/modules/mesh-delivery-builder/internal/build/presence_test.go b/modules/mesh-delivery-builder/internal/build/presence_test.go new file mode 100644 index 0000000..c1f3de7 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/presence_test.go @@ -0,0 +1,43 @@ +package build + +import ( + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +func presenceValue(t *testing.T, online bool) []byte { + t.Helper() + b, err := delivery.EncodePresence(delivery.Presence{Online: online}) + if err != nil { + t.Fatal(err) + } + return b +} + +func TestPresenceSetKnowsOnlyNodesTheWatchmanSawOnline(t *testing.T) { // 1607 review 3 + var s PresenceSet + if s.Online("ace") { + t.Error("a node the watchman never wrote is online") + } + if !s.Put("ace", presenceValue(t, true), false) || !s.Online("ace") { + t.Error("ace online was not taken in") + } + if s.Put("ace", presenceValue(t, true), false) { + t.Error("the same presence again counted as a change") + } + if !s.Put("ace", presenceValue(t, false), false) || s.Online("ace") { + t.Error("ace going offline was not taken in") + } + s.Put("g14", presenceValue(t, true), false) + if !s.Put("g14", nil, true) || s.Online("g14") { + t.Error("a deleted key is still online") + } + // A part's instance (part..) is not a node. + if s.Put("part.mesh-delivery-builder.novox", presenceValue(t, true), false) || s.Online("part") { + t.Error("a part's key was taken for a node") + } + if s.Put("novox", []byte("junk"), false) { + t.Error("junk counted") + } +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index b927a9a..4fad468 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -28,7 +28,9 @@ type Builder struct { Claimer Claimer Runner Runner Toolchains ToolchainEntries - Manifests Manifests + // Presence says which nodes are online (the watchman's); a node that is not is not chosen. + Presence interface{ Online(node string) bool } + Manifests Manifests // Nodes is how many build nodes build each fingerprint: two until release R4e (ADR 0478 clause 4). Nodes int // Limit is how many builds run at once; four when 0. @@ -182,6 +184,14 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { if read && planErr == nil && len(plan.Tasks) == 0 { return // nothing in the manifest for a builder to build } + if planErr == nil { + if why := b.stale(m, plan); why != "" { + // The fingerprint names a toolchain the build nodes no longer agree on: a new fingerprint is coming, and a build + // now would be of this source with another toolchain (ADR 0477 clause 2). Nothing is written. + b.logf("%s at %s is not built: %s", module, m.FP2, why) + return + } + } cl, err := b.Claimer.Claim(ctx, module, m.FP2, m.Manifest) if err != nil { b.logf("%s: %v", module, err) @@ -194,6 +204,10 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { if ctx.Err() != nil { return } + if result.State == "" { + b.logf("%s: left building, the toolchains moved under it", cl.Key) + return + } result.By, result.Manifest = b.Claimer.By, m.Manifest if err := b.Claimer.Finish(ctx, cl, result); err != nil { b.logf("%s: the build could not be written: %v", cl.Key, err) @@ -204,6 +218,23 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { } } +// stale is why the module's fingerprint is not built now, or empty when it is: the toolchain each language of the plan +// has agreed (novox/hq ADR 0465) must be the one the fingerprint was computed with (Module.Toolchains). +func (b *Builder) stale(m delivery.Module, plan Plan) string { + for _, bm := range plan.Modules() { + language := LanguageOf[bm] + tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language)) + if none != "" { + return fmt.Sprintf("the %s toolchain is not agreed: %s", language, none) + } + want, ok := m.Toolchains[language] + if !ok || want.Version != tc.Version || want.Recipe != tc.Recipe { + return fmt.Sprintf("the fingerprint names %s %s (recipe %s) and the nodes agree on %s (recipe %s)", language, want.Version, want.Recipe, tc.Version, tc.Recipe) + } + } + return "" +} + func failed(format string, args ...any) delivery.BuilderBuild { return delivery.BuilderBuild{State: delivery.BuildFailed, Error: fmt.Sprintf(format, args...)} } @@ -221,6 +252,7 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p case planErr != nil: return failed("%s", planErr) case m.Path == "": + // Empty is not known (a root module's path is "."): the indexer writes it again (issue 1615). return failed("the module's key names no directory (path), so the build tools cannot be told where %s lives", module) } repository := delivery.RepositoryName(m.Repository) @@ -229,10 +261,13 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p var steps []string for _, bm := range plan.Modules() { language := LanguageOf[bm] - nodes, tc, err := ChooseNodes(language, b.Toolchains.Entries(language), b.runningOn(bm), b.Nodes) + nodes, tc, err := ChooseNodes(language, b.Toolchains.Entries(language), b.runningOn(bm), b.online, b.Nodes) if err != nil { return failed("%s", err) } + if want := m.Toolchains[language]; want.Version != tc.Version || want.Recipe != tc.Recipe { + return delivery.BuilderBuild{} // the agreed toolchain moved since try looked: nothing is written + } used[language] = delivery.ToolchainUsed{Version: tc.Version, Recipe: tc.Recipe} if len(nodes) == 1 { steps = append(steps, fmt.Sprintf("%s on one node, %s", bm, nodes[0])) @@ -283,6 +318,8 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p return delivery.BuilderBuild{State: delivery.BuildDone, Step: "built " + strings.Join(steps, "; "), Artifacts: artifacts, Toolchains: used} } +func (b *Builder) online(node string) bool { return b.Presence == nil || b.Presence.Online(node) } + func (b *Builder) runningOn(buildModule string) map[string]int { b.mu.Lock() defer b.mu.Unlock() diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 23427b0..3a00ba8 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -95,20 +95,22 @@ type rig struct { st *memStore bm *buildModules tools *ToolchainSet + pres *PresenceSet mf *fakeManifests key string } func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig { t.Helper() - r := &rig{st: newMem(), bm: newBuildModules(), tools: &ToolchainSet{}, mf: &fakeManifests{byDigest: map[string]catalogue.Manifest{mdigest('a'): {Module: "m", Build: &catalogue.Build{Artifacts: artifacts}}}}} + r := &rig{st: newMem(), bm: newBuildModules(), tools: &ToolchainSet{}, pres: &PresenceSet{}, mf: &fakeManifests{byDigest: map[string]catalogue.Manifest{mdigest('a'): {Module: "m", Build: &catalogue.Build{Artifacts: artifacts}}}}} for _, n := range []string{"ace", "novox"} { r.tools.Put(GoBuildModule, n+".go", tcValue(t, "go", "go1.27.1"), false) r.tools.Put(NodeBuildModule, n+".typescript", tcValue(t, "typescript", "tsc5"), false) + r.pres.Put(n, presenceValue(t, true), false) } r.b = &Builder{ Claimer: Claimer{Store: r.st, By: "novox.builder"}, Runner: Runner{Caller: r.bm, Steps: r.bm.steps, Bound: time.Second}, - Toolchains: r.tools, Manifests: r.mf, Nodes: 2, Logf: t.Logf, + Toolchains: r.tools, Presence: r.pres, Manifests: r.mf, Nodes: 2, Logf: t.Logf, } var err error if r.key, err = delivery.BuilderBuildKey("m", fp1); err != nil { @@ -117,9 +119,14 @@ func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig { return r } -func moduleValue(t *testing.T, path string) []byte { +// agreedToolchains are what the rig's nodes report, and so what a fingerprint computed now names. +var agreedToolchains = map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}} + +func moduleValue(t *testing.T, path string) []byte { return moduleValueWith(t, path, agreedToolchains) } + +func moduleValueWith(t *testing.T, path string, toolchains map[string]delivery.ToolchainUsed) []byte { t.Helper() - data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP: "src1:" + hex64('f'), FP2: fp1, Manifest: mdigest('a'), Path: path}) + data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP: "src1:" + hex64('f'), FP2: fp1, Manifest: mdigest('a'), Path: path, Toolchains: toolchains}) if err != nil { t.Fatal(err) } @@ -244,16 +251,6 @@ func TestAModuleKeyWithNoPathFailsNamingThatReason(t *testing.T) { // 1607/AC8, } } -func TestNoQualifyingNodeFailsTheBuildNamingTheLanguage(t *testing.T) { // 1607/AC4 - r := newRig(t, tsBundle) - r.tools.Put(NodeBuildModule, "ace.typescript", nil, true) - r.tools.Put(NodeBuildModule, "novox.typescript", nil, true) - v := r.consider(t, "modules/m") - if v.State != delivery.BuildFailed || !strings.Contains(v.Error, "typescript") || !strings.Contains(v.Error, delivery.ToolchainNoReport) { - t.Errorf("build = %+v, want failed naming typescript and that no toolchain is reported", v) - } -} - func TestAnUnreadableManifestFailsTheBuild(t *testing.T) { r := newRig(t, goBundle) delete(r.mf.byDigest, mdigest('a')) @@ -291,7 +288,7 @@ func TestAKeyTheBuilderLeftBuildingIsBuiltAtStartAndDoneKeysAreNotBuiltTwice(t * func TestTwoInstancesBuildOneFingerprintOnce(t *testing.T) { // 1607/AC7 r := newRig(t, goBundle) - other := &Builder{Claimer: Claimer{Store: r.st, By: "ace.builder"}, Runner: r.b.Runner, Toolchains: r.tools, Manifests: r.mf, Nodes: 2, Logf: t.Logf} + other := &Builder{Claimer: Claimer{Store: r.st, By: "ace.builder"}, Runner: r.b.Runner, Toolchains: r.tools, Presence: r.pres, Manifests: r.mf, Nodes: 2, Logf: t.Logf} value := moduleValue(t, "modules/m") var wg sync.WaitGroup for _, b := range []*Builder{r.b, other} { @@ -320,3 +317,80 @@ func TestAModulesValueThatArrivesWhileItsKeyIsBeingBuiltIsLookedAtAgainWhenItEnd t.Errorf("build = %+v, want the failed key built again as a new revision after the first build ended", v) } } + +func TestAKeyIsBuiltOnlyWithTheToolchainItsFingerprintNamesAndNothingIsWrittenOtherwise(t *testing.T) { // 1615/AC4 + r := newRig(t, goBundle) + old := map[string]delivery.ToolchainUsed{"go": {Version: "go1.26.0", Recipe: "1"}} + r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", old)) + r.b.Wait() + if len(r.st.vals) != 0 || len(r.bm.calls) != 0 { + t.Errorf("keys %v and %d calls for a fingerprint of go1.26.0 while the nodes agree on go1.27.1, want neither", r.st.vals, len(r.bm.calls)) + } + // A recipe that differs is a different toolchain too, and a value that names none (written before) is not built. + r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "0"}})) + r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", nil)) + r.b.Wait() + if len(r.st.vals) != 0 || len(r.bm.calls) != 0 { + t.Errorf("keys %v and %d calls for a recipe that differs or a value that names no toolchain", r.st.vals, len(r.bm.calls)) + } + // The toolchain moves under a key that is not built yet: the old fingerprint stays unbuilt, the new one builds. + r.tools.Put(GoBuildModule, "ace.go", tcValue(t, "go", "go1.27.2"), false) + r.tools.Put(GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.2"), false) + r.b.Reconsider(context.Background()) + r.b.Wait() + if len(r.st.vals) != 0 { + t.Errorf("keys %v: the old fingerprint was built with the new toolchain", r.st.vals) + } + r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.2", Recipe: "1"}})) + r.b.Wait() + if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || v.Toolchains["go"].Version != "go1.27.2" { + t.Errorf("build = %+v, want done with go1.27.2", v) + } +} + +func TestNodesTheyDisagreeOnLeaveTheKeyUnwritten(t *testing.T) { // 1615/AC4 + r := newRig(t, goBundle) + r.tools.Put(GoBuildModule, "ace.go", tcValue(t, "go", "go1.26.0"), false) + r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) + r.b.Wait() + if len(r.st.vals) != 0 || len(r.bm.calls) != 0 { + t.Errorf("keys %v and %d calls while the nodes disagree", r.st.vals, len(r.bm.calls)) + } +} + +func TestARootModulesPathIsDotAndIsGivenToTheBuildTools(t *testing.T) { // 1615/AC4 + r := newRig(t, goBundle) + if v := r.consider(t, delivery.RootPath); v.State != delivery.BuildDone { + t.Fatalf("build = %+v, want done for a root module", v) + } + if args := r.bm.callsOf("build_go_build")[0].body.(map[string]any); args["module"] != "." { + t.Errorf("build args = %+v, want module \".\"", args) + } +} + +func TestANodeThatIsNotOnlineIsNotChosenAndComingBackWakesTheBuild(t *testing.T) { // 1607 review 3 + r := newRig(t, goBundle) + r.tools.Put(GoBuildModule, "g14.go", tcValue(t, "go", "go1.27.1"), false) // a laptop that reports its toolchain and sleeps + v := r.consider(t, "modules/m") + if v.State != delivery.BuildDone { + t.Fatalf("build = %+v", v) + } + for _, c := range r.bm.calls { + if c.node == "g14" { + t.Errorf("a call went to g14, which is not online: %+v", c) + } + } + // With no node online the key fails naming why; a node coming online builds it as a new revision. + r2 := newRig(t, goBundle) + r2.pres.Put("ace", nil, true) + r2.pres.Put("novox", nil, true) + if v := r2.consider(t, "modules/m"); v.State != delivery.BuildFailed || !strings.Contains(v.Error, "online") { + t.Fatalf("build = %+v, want failed because no node is online", v) + } + r2.pres.Put("ace", presenceValue(t, true), false) + r2.b.Reconsider(context.Background()) + r2.b.Wait() + if v := valueOf(t, r2.st, r2.key); v.State != delivery.BuildDone { + t.Errorf("build = %+v, want done once a node is online", v) + } +} diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 5d224e7..d957467 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -81,7 +81,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 t.Errorf("state = %+v, want %+v", m.State, want) } if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", - "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) { + "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } if want := []string{"mesh-build-go.build_go_build", "mesh-build-go.build_go_fetch", "mesh-build-node.build_node_build", "mesh-build-node.build_node_fetch"}; !reflect.DeepEqual(sorted(m.Invokes), want) { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 9d27205..1a902a2 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -15,7 +15,8 @@ "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-delivery-indexer.manifests", - "mesh-delivery-indexer.modules" + "mesh-delivery-indexer.modules", + "mesh-delivery-watchman.presence" ], "invokes": [ "mesh-build-go.build_go_build",