From 8545e7fc9fc0c97a1fb71d731c39028ca2adc1dc Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 1 Nov 2026 15:22:04 +0100 Subject: [PATCH 1/5] Cite ADR 0584 where the builder waits on a toolchain the nodes do not agree on (issue 1578) Comment only; the majority rule itself is in mesh-sdk's AgreedToolchain. Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- modules/mesh-delivery-builder/internal/build/run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index fd8fc69..4578824 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -415,7 +415,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { if planErr == nil && !nothing { if why := b.stale(m, plan); why != "" { // The fingerprint names a toolchain the build nodes no longer agree on: a new fingerprint is coming, and a build - // now would be of this source with another toolchain (ADR 0477 clause 2). Nothing is written. + // now would be of this source with another toolchain (ADR 0584 clause 1, which changes ADR 0477 clause 2). Nothing is written. b.logf("%s at %s is not built: %s", module, m.FP2, why) return } From 9e407f8f76c1f5c7f348336fdbf3b32d9798047b Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 1 Nov 2026 15:26:55 +0100 Subject: [PATCH 2/5] Build with the toolchain a majority of build nodes report; say which nodes sit out (ADR 0584, issue 1578) One build node on another version refused every build of the language. The builder and the checker now follow mesh-sdk's majority agreement; the key's step names a node that sits out ("ace sits out: typescript 26.12.1, agreed 26.13.0"), and a build ask a minority node declined is put again for a node on the agreed version. Check steps compare no toolchain, so clause 2 already holds for them. Pins builder, indexer and checker to mesh-sdk 71be15f. Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- modules/mesh-delivery-builder/go.mod | 2 +- modules/mesh-delivery-builder/go.sum | 4 +-- .../internal/build/askagain.go | 5 +++ .../internal/build/run.go | 31 ++++++++++++++----- .../internal/build/run_test.go | 8 +++++ modules/mesh-delivery-checker/go.mod | 2 +- modules/mesh-delivery-checker/go.sum | 4 +-- modules/mesh-delivery-indexer/go.mod | 2 +- modules/mesh-delivery-indexer/go.sum | 4 +-- 9 files changed, 46 insertions(+), 16 deletions(-) diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index 9b9cb54..f1d1a30 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-builder go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101142413-71be15f53066 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-builder/go.sum b/modules/mesh-delivery-builder/go.sum index 37e06a8..c368184 100644 --- a/modules/mesh-delivery-builder/go.sum +++ b/modules/mesh-delivery-builder/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027 h1:a33dsavoaqllh40eVICbhxi6Kero1EGSjt1V+2fa2Hk= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101142413-71be15f53066 h1:rpYDAiJ+245xHvmYdta/hE3gSGEoebglr4ujdOoFH/8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101142413-71be15f53066/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-builder/internal/build/askagain.go b/modules/mesh-delivery-builder/internal/build/askagain.go index 1b03da9..23a0974 100644 --- a/modules/mesh-delivery-builder/internal/build/askagain.go +++ b/modules/mesh-delivery-builder/internal/build/askagain.go @@ -317,6 +317,11 @@ func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-ch defer stop() } step, err := b.Asks.Wait(wait, ask) + if d, declined := delivery.DeclinedOf(step); err == nil && declined { + // A build node on another toolchain than the agreed one sat out (ADR 0584 clause 2): the ask is put again, for a node + // on the agreed one to take. The most askings stays the bound, and the build then fails with these reasons in it. + return step, strings.TrimSpace(step.Node + " declined: " + step.Reason + " (lacks " + strings.Join(d.Lacks, ", ") + ")"), errAgain + } if err == nil && buildrun.IsLost(step) { // The build seat gave the ask up (issue 734): it can no longer be answered, so it is asked again now, with the // words of why, not at the check bound. diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 4578824..f4c2357 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -506,10 +506,11 @@ func (b *Builder) stale(m delivery.Module, plan Plan) string { continue // an archive is packed with no toolchain } language := buildrun.LanguageOf[bm] - tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language)) - if none != "" { - return fmt.Sprintf("the %s toolchain is not agreed: %s", language, none) + ag := delivery.ToolchainAgreement(b.Toolchains.Entries(language)) + if ag.Wait != "" { + return fmt.Sprintf("the %s toolchain is not agreed: %s", language, ag.Wait) } + tc := ag.Toolchain want, ok := m.Toolchains[language] if !ok || want.Version != tc.Version || want.Recipe != tc.Recipe { return fmt.Sprintf("the fingerprint names %s %s (recipe %s) and the nodes agree on %s (recipe %s)", language, want.Version, want.Recipe, tc.Version, tc.Recipe) @@ -543,7 +544,8 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p repository := delivery.RepositoryName(m.Repository) var lanes []*lane used := map[string]delivery.ToolchainUsed{} - var kept []string // build modules whose artifacts stay as the prior done key has them + var sitting []string // the nodes that sit out of a language's build, said in the key's step (ADR 0584 clause 2) + var kept []string // build modules whose artifacts stay as the prior done key has them for _, bm := range plan.Modules() { if prior != nil && len(only) > 0 && !only[bm] { kept = append(kept, bm) @@ -553,10 +555,12 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p if bm != ImageBuildModule && plan.NeedsToolchain(bm) { // The ask names the toolchain the fingerprint was computed with; a node building with another refuses it. language := buildrun.LanguageOf[bm] - tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language)) - if none != "" { - return failed("the %s toolchain is not agreed: %s", language, none) + ag := delivery.ToolchainAgreement(b.Toolchains.Entries(language)) + if ag.Wait != "" { + return failed("the %s toolchain is not agreed: %s", language, ag.Wait) } + tc := ag.Toolchain + sitting = append(sitting, SittingOutWords(language, ag)...) if want := m.Toolchains[language]; want.Version != tc.Version || want.Recipe != tc.Recipe { return delivery.BuilderBuild{} // the agreed toolchain moved since try looked: nothing is written } @@ -653,6 +657,9 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p } } step := "built " + strings.Join(steps, "; ") + if len(sitting) > 0 { + step += " (" + strings.Join(sitting, "; ") + ")" + } if len(used) == 0 { used = nil } @@ -942,3 +949,13 @@ func LogEnded(module string, m delivery.Module, ref, by string, at time.Time, re e.Made = append([]delivery.BuiltArtifact(nil), result.Artifacts...) return e } + +// SittingOutWords says, per node that is not on the agreed toolchain of a language, that it sits out and which version +// it has against the agreed one: "ace sits out: typescript 26.12.1, agreed 26.13.0" (novox/hq ADR 0584 clause 2). +func SittingOutWords(language string, a delivery.Agreement) []string { + var out []string + for _, s := range a.SittingOut { + out = append(out, fmt.Sprintf("%s sits out: %s %s, agreed %s", s.Node, language, s.Version, a.Toolchain.Version)) + } + return out +} diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index c05901d..2eeb07d 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -223,6 +223,14 @@ var ( func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7, ADR 0478 clauses 1, 4 r := newRig(t, goBundle, archiveA, tsBundle) + // ADR 0584: a third node agrees with novox and ace's typescript is the odd one; the build is done and its step says ace sits out. + r.tools.Put(buildrun.NodeBuildModule, "g14.typescript", tcValue(t, "typescript", "tsc5"), false) + r.tools.Put(buildrun.NodeBuildModule, "ace.typescript", tcValue(t, "typescript", "tsc4"), false) + defer func() { + if st := r.st.vals[r.key]; !strings.Contains(string(st), "ace sits out: typescript tsc4, agreed tsc5") { + t.Errorf("the key does not say ace sits out: %s", st) + } + }() var logged []delivery.BuildLogEntry var lmu sync.Mutex r.b.Node, r.b.Log = "novox", func(e delivery.BuildLogEntry) { lmu.Lock(); logged = append(logged, e); lmu.Unlock() } diff --git a/modules/mesh-delivery-checker/go.mod b/modules/mesh-delivery-checker/go.mod index ce26061..487fb8d 100644 --- a/modules/mesh-delivery-checker/go.mod +++ b/modules/mesh-delivery-checker/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-checker go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101142413-71be15f53066 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-checker/go.sum b/modules/mesh-delivery-checker/go.sum index 5fc11b2..c368184 100644 --- a/modules/mesh-delivery-checker/go.sum +++ b/modules/mesh-delivery-checker/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101142413-71be15f53066 h1:rpYDAiJ+245xHvmYdta/hE3gSGEoebglr4ujdOoFH/8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101142413-71be15f53066/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index 10c900e..5c3e040 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-indexer go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101142413-71be15f53066 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-indexer/go.sum b/modules/mesh-delivery-indexer/go.sum index 5fc11b2..c368184 100644 --- a/modules/mesh-delivery-indexer/go.sum +++ b/modules/mesh-delivery-indexer/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101142413-71be15f53066 h1:rpYDAiJ+245xHvmYdta/hE3gSGEoebglr4ujdOoFH/8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101142413-71be15f53066/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= From 5085b681e9338265a31831a8e7db7d0a8dc1c4ef Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 1 Nov 2026 15:36:23 +0100 Subject: [PATCH 3/5] Count no decline as an asking; bound declines by time and name the nodes (ADR 0584, issue 1578) A build node off the agreed toolchain declines the ask and, idle, takes the ask put again at once, so three declines used the three askings and failed the build. A decline now uses no asking: the ask is put again after a short pause, and past a bound from the first decline the key fails naming the declining nodes and their reasons. A person reads which node sits out in the key's step text, which the view's builder_builds answers as stored. Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- .../internal/build/askagain.go | 76 ++++++++++++++++++- .../internal/build/run.go | 3 + .../internal/build/run_test.go | 19 +++++ 3 files changed, 94 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-builder/internal/build/askagain.go b/modules/mesh-delivery-builder/internal/build/askagain.go index 23a0974..a7d6cd6 100644 --- a/modules/mesh-delivery-builder/internal/build/askagain.go +++ b/modules/mesh-delivery-builder/internal/build/askagain.go @@ -6,6 +6,7 @@ import ( "fmt" "regexp" "slices" + "sort" "strconv" "strings" "time" @@ -177,6 +178,42 @@ func (b *Builder) drop(ctx context.Context, ask delivery.StepAsk, old string) { } } +// DefaultDeclineBound is how long one ask may go on being declined, from its first decline, before its key fails (the +// checker's DefaultDeclineBound, issue 890); DefaultDeclinePause is how long the builder waits before it puts a declined +// ask again, so a lone node off the agreed toolchain cannot spin on it. +const ( + DefaultDeclineBound = 30 * time.Minute + DefaultDeclinePause = 5 * time.Second +) + +func (b *Builder) declineBound() time.Duration { + if b.DeclineBound > 0 { + return b.DeclineBound + } + return DefaultDeclineBound +} + +func (b *Builder) declinePause() time.Duration { + if b.DeclinePause != 0 { + return max(b.DeclinePause, 0) + } + return DefaultDeclinePause +} + +// declinedWords names the nodes that declined an ask and why, in the order of their names. +func declinedWords(by map[string]string) string { + nodes := make([]string, 0, len(by)) + for n := range by { + nodes = append(nodes, n) + } + sort.Strings(nodes) + var parts []string + for _, n := range nodes { + parts = append(parts, n+": "+by[n]) + } + return strings.Join(parts, "; ") +} + func (b *Builder) askings() int { if b.Askings > 0 { return b.Askings @@ -255,6 +292,8 @@ func (b *Builder) unlisten(key, language string, c chan string) { var ( errCheck = errors.New("the check bound passed") errAgain = errors.New("asked again") + // errDeclined: a build node on another toolchain than the agreed one sat out of the ask (ADR 0584 clause 2). + errDeclined = errors.New("declined") ) type byHand struct{ why string } @@ -268,11 +307,40 @@ func (b *Builder) answer(ctx context.Context, l *lane, ask delivery.StepAsk, ask again := b.listen(key, ask.Language) defer b.unlisten(key, ask.Language, again) asking := askingOf(asked.step, l.module) + var firstDecline time.Time + declines := map[string]string{} // node -> why it declined, the last for { step, why, err := b.waitOnce(ctx, ask, again) switch { case errors.Is(err, errCheck): continue + case errors.Is(err, errDeclined): + // Not an asking: a node that sits out would use them all while it idles and pulls the ask again at once. + // Bounded by time from the first decline, then failed loudly naming the nodes and their reasons (as the + // checker's decline loop is, issue 890). + now := time.Now() + if firstDecline.IsZero() { + firstDecline = now + } + declines[step.Node] = why + if bound := b.declineBound(); now.Sub(firstDecline) > bound { + return step, ask, fmt.Errorf("the ask %s of %s was declined for %s and no build node took it: %s", ask.ID, l.module, bound.Round(time.Second), declinedWords(declines)) + } + select { + case <-ctx.Done(): + return step, ask, ctx.Err() + case <-time.After(b.declinePause()): + } + old := ask.ID + ask.ID, ask.At = buildstep.NewID(), time.Now() + if err := asked.note(ctx, ask.Language, ask.ID, fmt.Sprintf("put %s again after %s declined it", l.module, step.Node)); err != nil { + return step, ask, fmt.Errorf("the ask of %s put again after a decline could not be written to the key: %w", l.module, err) + } + if err := b.Asks.Put(ctx, ask); err != nil { + return step, ask, err + } + b.drop(ctx, ask, old) + b.logf("%s: the ask %s of %s was declined on %s and is put again as %s: %s", key, old, l.module, step.Node, ask.ID, why) case errors.Is(err, errAgain): if asking >= b.askings() { return step, ask, fmt.Errorf("the ask %s could not be answered and was asked %d times, the most it is: %s", ask.ID, asking, why) @@ -317,10 +385,10 @@ func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-ch defer stop() } step, err := b.Asks.Wait(wait, ask) - if d, declined := delivery.DeclinedOf(step); err == nil && declined { - // A build node on another toolchain than the agreed one sat out (ADR 0584 clause 2): the ask is put again, for a node - // on the agreed one to take. The most askings stays the bound, and the build then fails with these reasons in it. - return step, strings.TrimSpace(step.Node + " declined: " + step.Reason + " (lacks " + strings.Join(d.Lacks, ", ") + ")"), errAgain + if _, declined := delivery.DeclinedOf(step); err == nil && declined { + // A build node off the agreed toolchain sat out (ADR 0584 clause 2). It is no asking: answer() puts the ask again + // after a pause, within DeclineBound. + return step, step.Reason, errDeclined } if err == nil && buildrun.IsLost(step) { // The build seat gave the ask up (issue 734): it can no longer be answered, so it is asked again now, with the diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index f4c2357..c7b812b 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -67,6 +67,9 @@ type Builder struct { // Askings is how many times one build module's part of a build is asked, the first included, before its key is failed // with why; DefaultAskings when 0. Askings int + // DeclineBound and DeclinePause bound and pace the asks build nodes off the agreed toolchain declined (ADR 0584 clause 2): + // they are no askings. DefaultDeclineBound and DefaultDeclinePause when 0; a negative DeclinePause puts it again at once. + DeclineBound, DeclinePause time.Duration // Said is told each ask asked again (the builder's event build.asked-again). Nil tells nobody. Said func(AskedAgain) // Log is told each build's start and end, the entries of the builder's log (novox/hq ADR 0571 §1). Nil tells nobody. diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 2eeb07d..6950dc3 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -738,6 +738,25 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks())) } }) + t.Run("declined by a node off the agreed toolchain: put again, no asking used, the bound ends it loudly", func(t *testing.T) { // ADR 0584 clause 2 + r := newRig(t, goBundle) + r.asks.Answer = nil + r.b.DeclinePause, r.b.DeclineBound = -1, 150*time.Millisecond + r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) + declined, _ := json.Marshal(delivery.StepDeclined{Declined: true, Lacks: []string{"the agreed toolchain go1.27.1 (recipe 1)"}}) + waitForAsks(t, r, 1) + for ended, deadline := 0, time.Now().Add(5*time.Second); valueOf(t, r.st, r.key).State == delivery.BuildBuilding && time.Now().Before(deadline); time.Sleep(5 * time.Millisecond) { + if asks := r.asks.Asks(); len(asks) > ended { // each ask put again is declined in its turn + r.asks.End(delivery.Step{ID: asks[ended].ID, Node: "ace", State: delivery.StepRefused, Reason: "this node builds with go1.26.0", Result: declined}) + ended++ + } + } + r.b.Wait() + v := valueOf(t, r.st, r.key) + if n := len(r.asks.Asks()); v.State != delivery.BuildFailed || n <= DefaultAskings || !strings.Contains(v.Error, "declined for") || !strings.Contains(v.Error, "ace: this node builds with go1.26.0") { + t.Fatalf("the key = %+v after %d asks, want it failed after more than %d asks, naming ace and its reason", v, n, DefaultAskings) + } + }) for _, c := range []struct { name string askings int From a6c225eb45b6c98ae65f6d655da5160a664b8e53 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 1 Nov 2026 15:36:52 +0100 Subject: [PATCH 4/5] =?UTF-8?q?Pause=20before=20a=20declined=20ask=20is=20?= =?UTF-8?q?put=20again=20with=20a=20context's=20bound,=20not=20a=20timer?= =?UTF-8?q?=20(ADR=200437=20=C2=A78,=20issue=201578)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The builder's manifest test refuses time.After. The previous commit's pause used it. Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- modules/mesh-delivery-builder/internal/build/askagain.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/mesh-delivery-builder/internal/build/askagain.go b/modules/mesh-delivery-builder/internal/build/askagain.go index a7d6cd6..0a7e2a9 100644 --- a/modules/mesh-delivery-builder/internal/build/askagain.go +++ b/modules/mesh-delivery-builder/internal/build/askagain.go @@ -326,10 +326,12 @@ func (b *Builder) answer(ctx context.Context, l *lane, ask delivery.StepAsk, ask if bound := b.declineBound(); now.Sub(firstDecline) > bound { return step, ask, fmt.Errorf("the ask %s of %s was declined for %s and no build node took it: %s", ask.ID, l.module, bound.Round(time.Second), declinedWords(declines)) } - select { - case <-ctx.Done(): + // The pause is a context's bound, not a timer (ADR 0437 §8). + pause, stop := context.WithTimeout(ctx, b.declinePause()) + <-pause.Done() + stop() + if ctx.Err() != nil { return step, ask, ctx.Err() - case <-time.After(b.declinePause()): } old := ask.ID ask.ID, ask.At = buildstep.NewID(), time.Now() From d739516056dd79cafc29a58b08837cd33bbcfea1 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 1 Nov 2026 15:38:16 +0100 Subject: [PATCH 5/5] Name the agreed toolchain in the checker's check asks; pin mesh-sdk 67354f8 (ADR 0584, issue 1578) A check ask named no toolchain, so a node on the minority version could run it and KeepPasses record the pass as run on the agreed one. The check ask now carries the agreed toolchain and a node off it declines (mesh-sdk), for the checker's decline loop to put again. Pins the builder, indexer and checker. Known limit: the checker puts a declined check ask again at once, with no pause; the declining puller skips that kind until its next pull wait, which paces a lone minority node. Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- modules/mesh-delivery-builder/go.mod | 2 +- modules/mesh-delivery-builder/go.sum | 4 ++-- modules/mesh-delivery-checker/go.mod | 2 +- modules/mesh-delivery-checker/go.sum | 4 ++-- .../internal/check/examiner_test.go | 5 +++++ .../mesh-delivery-checker/internal/check/judge.go | 13 ++++++++++--- modules/mesh-delivery-indexer/go.mod | 2 +- modules/mesh-delivery-indexer/go.sum | 4 ++-- 8 files changed, 24 insertions(+), 12 deletions(-) diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index f1d1a30..55cd25e 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-builder go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101142413-71be15f53066 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-builder/go.sum b/modules/mesh-delivery-builder/go.sum index c368184..05b03d7 100644 --- a/modules/mesh-delivery-builder/go.sum +++ b/modules/mesh-delivery-builder/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101142413-71be15f53066 h1:rpYDAiJ+245xHvmYdta/hE3gSGEoebglr4ujdOoFH/8= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101142413-71be15f53066/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a h1:AChSJUmHyOZ5smiuTrtDZNjetYNTFb+anRFjXIQHbSA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-checker/go.mod b/modules/mesh-delivery-checker/go.mod index 487fb8d..e029c7f 100644 --- a/modules/mesh-delivery-checker/go.mod +++ b/modules/mesh-delivery-checker/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-checker go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101142413-71be15f53066 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-checker/go.sum b/modules/mesh-delivery-checker/go.sum index c368184..05b03d7 100644 --- a/modules/mesh-delivery-checker/go.sum +++ b/modules/mesh-delivery-checker/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101142413-71be15f53066 h1:rpYDAiJ+245xHvmYdta/hE3gSGEoebglr4ujdOoFH/8= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101142413-71be15f53066/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a h1:AChSJUmHyOZ5smiuTrtDZNjetYNTFb+anRFjXIQHbSA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go index 55020ce..628870a 100644 --- a/modules/mesh-delivery-checker/internal/check/examiner_test.go +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -342,6 +342,11 @@ func TestOnlyThePackageWithoutAKeptVerdictRuns(t *testing.T) { // 1288/AC1 if out.Verdict.State != delivery.VerdictPass || len(asks) != 3 || len(asks[0].Needs) != 0 { t.Fatalf("verdict %+v, asks %+v: want a pass of three asks, the first naming no need", out.Verdict, asks) } + for _, a := range asks { // ADR 0584: every check ask names the agreed toolchain, so a node off it declines + if a.Check.Toolchain == nil || a.Check.Toolchain.Version != "go1.27.1" || a.Check.Toolchain.Recipe != "7" { + t.Errorf("a check ask names toolchain %+v, want go1.27.1 recipe 7", a.Check.Toolchain) + } + } for _, a := range asks[1:] { if strings.Join(a.Needs, ",") != delivery.NeedNodeBuildNode || strings.Join(a.Check.Levels, ",") != delivery.LevelIntegration || strings.Join(a.Check.Packages, ",") != "fixture/node" { t.Errorf("a placed ask %+v, want the integration level needing node-build-node", a) diff --git a/modules/mesh-delivery-checker/internal/check/judge.go b/modules/mesh-delivery-checker/internal/check/judge.go index 1c2226f..ba16aaf 100644 --- a/modules/mesh-delivery-checker/internal/check/judge.go +++ b/modules/mesh-delivery-checker/internal/check/judge.go @@ -166,7 +166,9 @@ type target struct { module string node string toolchain string - fetched Fetched + // agreed is the toolchain the ask names, so a node off it declines the check (ADR 0584); nil when none is agreed. + agreed *delivery.ToolchainUsed + fetched Fetched // fetchArgs are the arguments of the fetch its check ask runs. fetchArgs map[string]any // priority are the Go packages (import paths) or the Node module directories whose test files the merge changes. @@ -374,6 +376,7 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict { t := &target{pull: p, module: bm} if tc, none := delivery.AgreedToolchain(st.j.Toolchains.Entries(buildrun.LanguageOf[bm])); none == "" { t.toolchain = tc.Version + t.agreed = &delivery.ToolchainUsed{Version: tc.Version, Recipe: tc.Recipe} } args := map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number), "base": p.Trunk} if bm == buildrun.NodeBuildModule && groupPackage != nil && p.Repository != SDKRepository { @@ -428,7 +431,7 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, levels, needs, packages []string) *delivery.Verdict { p := t.pull placed := len(needs) > 0 - job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: levels, Kept: true, Packages: packages} + job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: levels, Kept: true, Packages: packages, Toolchain: t.agreed} if st.j.Nodes != nil { names, err := st.j.Nodes.Names(ctx) if err == nil && len(names) == 0 { @@ -495,8 +498,12 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l bound = DefaultDeclineBound } if now.Sub(firstDecline) > bound { + lacks := strings.Join(needs, ", ") + if lacks == "" { + lacks = strings.Join(d.Lacks, ", ") // a node off the agreed toolchain declined it (ADR 0584) + } return fail(step.Node, fmt.Sprintf("the check of %s#%d needs a node with %s, and none took it in %s: every node that pulled it lacks it (last %s)", - p.Repository, p.Number, strings.Join(needs, ", "), bound, step.Node)) + p.Repository, p.Number, lacks, bound, step.Node)) } st.j.logf("%s#%d: the ask %s was declined on %s, which lacks %s; asked again", p.Repository, p.Number, a.ID, step.Node, strings.Join(d.Lacks, ", ")) a.ID, a.At = buildstep.NewID(), now.UTC() diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index 5c3e040..d0bd58b 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-indexer go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101142413-71be15f53066 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-indexer/go.sum b/modules/mesh-delivery-indexer/go.sum index c368184..05b03d7 100644 --- a/modules/mesh-delivery-indexer/go.sum +++ b/modules/mesh-delivery-indexer/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101142413-71be15f53066 h1:rpYDAiJ+245xHvmYdta/hE3gSGEoebglr4ujdOoFH/8= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101142413-71be15f53066/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a h1:AChSJUmHyOZ5smiuTrtDZNjetYNTFb+anRFjXIQHbSA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=