Merge pull request 'Agree a toolchain by strict majority; a node on another version sits out (ADR 0584, issue 1578)' (#131) from fix/toolchain-majority into main

modules/mesh-delivery-builder/go.mod

@@ -3,7 +3,7 @@ module mesh-delivery-builder
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a
 	github.com/nats-io/nats-server/v2 v2.11.17
 	github.com/nats-io/nats.go v1.54.0
 )

modules/mesh-delivery-builder/go.sum

@@ -1,5 +1,5 @@
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027 h1:a33dsavoaqllh40eVICbhxi6Kero1EGSjt1V+2fa2Hk=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a h1:AChSJUmHyOZ5smiuTrtDZNjetYNTFb+anRFjXIQHbSA=
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
 github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=

modules/mesh-delivery-builder/internal/build/askagain.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"regexp"
 	"slices"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -177,6 +178,42 @@ func (b *Builder) drop(ctx context.Context, ask delivery.StepAsk, old string) {
 	}
 }
 
+// DefaultDeclineBound is how long one ask may go on being declined, from its first decline, before its key fails (the
+// checker's DefaultDeclineBound, issue 890); DefaultDeclinePause is how long the builder waits before it puts a declined
+// ask again, so a lone node off the agreed toolchain cannot spin on it.
+const (
+	DefaultDeclineBound = 30 * time.Minute
+	DefaultDeclinePause = 5 * time.Second
+)
+
+func (b *Builder) declineBound() time.Duration {
+	if b.DeclineBound > 0 {
+		return b.DeclineBound
+	}
+	return DefaultDeclineBound
+}
+
+func (b *Builder) declinePause() time.Duration {
+	if b.DeclinePause != 0 {
+		return max(b.DeclinePause, 0)
+	}
+	return DefaultDeclinePause
+}
+
+// declinedWords names the nodes that declined an ask and why, in the order of their names.
+func declinedWords(by map[string]string) string {
+	nodes := make([]string, 0, len(by))
+	for n := range by {
+		nodes = append(nodes, n)
+	}
+	sort.Strings(nodes)
+	var parts []string
+	for _, n := range nodes {
+		parts = append(parts, n+": "+by[n])
+	}
+	return strings.Join(parts, "; ")
+}
+
 func (b *Builder) askings() int {
 	if b.Askings > 0 {
 		return b.Askings
@@ -255,6 +292,8 @@ func (b *Builder) unlisten(key, language string, c chan string) {
 var (
 	errCheck = errors.New("the check bound passed")
 	errAgain = errors.New("asked again")
+	// errDeclined: a build node on another toolchain than the agreed one sat out of the ask (ADR 0584 clause 2).
+	errDeclined = errors.New("declined")
 )
 
 type byHand struct{ why string }
@@ -268,11 +307,42 @@ func (b *Builder) answer(ctx context.Context, l *lane, ask delivery.StepAsk, ask
 	again := b.listen(key, ask.Language)
 	defer b.unlisten(key, ask.Language, again)
 	asking := askingOf(asked.step, l.module)
+	var firstDecline time.Time
+	declines := map[string]string{} // node -> why it declined, the last
 	for {
 		step, why, err := b.waitOnce(ctx, ask, again)
 		switch {
 		case errors.Is(err, errCheck):
 			continue
+		case errors.Is(err, errDeclined):
+			// Not an asking: a node that sits out would use them all while it idles and pulls the ask again at once.
+			// Bounded by time from the first decline, then failed loudly naming the nodes and their reasons (as the
+			// checker's decline loop is, issue 890).
+			now := time.Now()
+			if firstDecline.IsZero() {
+				firstDecline = now
+			}
+			declines[step.Node] = why
+			if bound := b.declineBound(); now.Sub(firstDecline) > bound {
+				return step, ask, fmt.Errorf("the ask %s of %s was declined for %s and no build node took it: %s", ask.ID, l.module, bound.Round(time.Second), declinedWords(declines))
+			}
+			// The pause is a context's bound, not a timer (ADR 0437 §8).
+			pause, stop := context.WithTimeout(ctx, b.declinePause())
+			<-pause.Done()
+			stop()
+			if ctx.Err() != nil {
+				return step, ask, ctx.Err()
+			}
+			old := ask.ID
+			ask.ID, ask.At = buildstep.NewID(), time.Now()
+			if err := asked.note(ctx, ask.Language, ask.ID, fmt.Sprintf("put %s again after %s declined it", l.module, step.Node)); err != nil {
+				return step, ask, fmt.Errorf("the ask of %s put again after a decline could not be written to the key: %w", l.module, err)
+			}
+			if err := b.Asks.Put(ctx, ask); err != nil {
+				return step, ask, err
+			}
+			b.drop(ctx, ask, old)
+			b.logf("%s: the ask %s of %s was declined on %s and is put again as %s: %s", key, old, l.module, step.Node, ask.ID, why)
 		case errors.Is(err, errAgain):
 			if asking >= b.askings() {
 				return step, ask, fmt.Errorf("the ask %s could not be answered and was asked %d times, the most it is: %s", ask.ID, asking, why)
@@ -317,6 +387,11 @@ func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-ch
 		defer stop()
 	}
 	step, err := b.Asks.Wait(wait, ask)
+	if _, declined := delivery.DeclinedOf(step); err == nil && declined {
+		// A build node off the agreed toolchain sat out (ADR 0584 clause 2). It is no asking: answer() puts the ask again
+		// after a pause, within DeclineBound.
+		return step, step.Reason, errDeclined
+	}
 	if err == nil && buildrun.IsLost(step) {
 		// The build seat gave the ask up (issue 734): it can no longer be answered, so it is asked again now, with the
 		// words of why, not at the check bound.

modules/mesh-delivery-builder/internal/build/run.go

@@ -67,6 +67,9 @@ type Builder struct {
 	// Askings is how many times one build module's part of a build is asked, the first included, before its key is failed
 	// with why; DefaultAskings when 0.
 	Askings int
+	// DeclineBound and DeclinePause bound and pace the asks build nodes off the agreed toolchain declined (ADR 0584 clause 2):
+	// they are no askings. DefaultDeclineBound and DefaultDeclinePause when 0; a negative DeclinePause puts it again at once.
+	DeclineBound, DeclinePause time.Duration
 	// Said is told each ask asked again (the builder's event build.asked-again). Nil tells nobody.
 	Said func(AskedAgain)
 	// Log is told each build's start and end, the entries of the builder's log (novox/hq ADR 0571 §1). Nil tells nobody.
@@ -415,7 +418,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
 	if planErr == nil && !nothing {
 		if why := b.stale(m, plan); why != "" {
 			// The fingerprint names a toolchain the build nodes no longer agree on: a new fingerprint is coming, and a build
-			// now would be of this source with another toolchain (ADR 0477 clause 2). Nothing is written.
+			// now would be of this source with another toolchain (ADR 0584 clause 1, which changes ADR 0477 clause 2). Nothing is written.
 			b.logf("%s at %s is not built: %s", module, m.FP2, why)
 			return
 		}
@@ -506,10 +509,11 @@ func (b *Builder) stale(m delivery.Module, plan Plan) string {
 			continue // an archive is packed with no toolchain
 		}
 		language := buildrun.LanguageOf[bm]
-		tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language))
+		ag := delivery.ToolchainAgreement(b.Toolchains.Entries(language))
-		if none != "" {
+		if ag.Wait != "" {
-			return fmt.Sprintf("the %s toolchain is not agreed: %s", language, none)
+			return fmt.Sprintf("the %s toolchain is not agreed: %s", language, ag.Wait)
 		}
+		tc := ag.Toolchain
 		want, ok := m.Toolchains[language]
 		if !ok || want.Version != tc.Version || want.Recipe != tc.Recipe {
 			return fmt.Sprintf("the fingerprint names %s %s (recipe %s) and the nodes agree on %s (recipe %s)", language, want.Version, want.Recipe, tc.Version, tc.Recipe)
@@ -543,7 +547,8 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
 	repository := delivery.RepositoryName(m.Repository)
 	var lanes []*lane
 	used := map[string]delivery.ToolchainUsed{}
-	var kept []string // build modules whose artifacts stay as the prior done key has them
+	var sitting []string // the nodes that sit out of a language's build, said in the key's step (ADR 0584 clause 2)
+	var kept []string    // build modules whose artifacts stay as the prior done key has them
 	for _, bm := range plan.Modules() {
 		if prior != nil && len(only) > 0 && !only[bm] {
 			kept = append(kept, bm)
@@ -553,10 +558,12 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
 		if bm != ImageBuildModule && plan.NeedsToolchain(bm) {
 			// The ask names the toolchain the fingerprint was computed with; a node building with another refuses it.
 			language := buildrun.LanguageOf[bm]
-			tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language))
+			ag := delivery.ToolchainAgreement(b.Toolchains.Entries(language))
-			if none != "" {
+			if ag.Wait != "" {
-				return failed("the %s toolchain is not agreed: %s", language, none)
+				return failed("the %s toolchain is not agreed: %s", language, ag.Wait)
 			}
+			tc := ag.Toolchain
+			sitting = append(sitting, SittingOutWords(language, ag)...)
 			if want := m.Toolchains[language]; want.Version != tc.Version || want.Recipe != tc.Recipe {
 				return delivery.BuilderBuild{} // the agreed toolchain moved since try looked: nothing is written
 			}
@@ -653,6 +660,9 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
 		}
 	}
 	step := "built " + strings.Join(steps, "; ")
+	if len(sitting) > 0 {
+		step += " (" + strings.Join(sitting, "; ") + ")"
+	}
 	if len(used) == 0 {
 		used = nil
 	}
@@ -942,3 +952,13 @@ func LogEnded(module string, m delivery.Module, ref, by string, at time.Time, re
 	e.Made = append([]delivery.BuiltArtifact(nil), result.Artifacts...)
 	return e
 }
+
+// SittingOutWords says, per node that is not on the agreed toolchain of a language, that it sits out and which version
+// it has against the agreed one: "ace sits out: typescript 26.12.1, agreed 26.13.0" (novox/hq ADR 0584 clause 2).
+func SittingOutWords(language string, a delivery.Agreement) []string {
+	var out []string
+	for _, s := range a.SittingOut {
+		out = append(out, fmt.Sprintf("%s sits out: %s %s, agreed %s", s.Node, language, s.Version, a.Toolchain.Version))
+	}
+	return out
+}

modules/mesh-delivery-builder/internal/build/run_test.go

@@ -223,6 +223,14 @@ var (
 
 func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7, ADR 0478 clauses 1, 4
 	r := newRig(t, goBundle, archiveA, tsBundle)
+	// ADR 0584: a third node agrees with novox and ace's typescript is the odd one; the build is done and its step says ace sits out.
+	r.tools.Put(buildrun.NodeBuildModule, "g14.typescript", tcValue(t, "typescript", "tsc5"), false)
+	r.tools.Put(buildrun.NodeBuildModule, "ace.typescript", tcValue(t, "typescript", "tsc4"), false)
+	defer func() {
+		if st := r.st.vals[r.key]; !strings.Contains(string(st), "ace sits out: typescript tsc4, agreed tsc5") {
+			t.Errorf("the key does not say ace sits out: %s", st)
+		}
+	}()
 	var logged []delivery.BuildLogEntry
 	var lmu sync.Mutex
 	r.b.Node, r.b.Log = "novox", func(e delivery.BuildLogEntry) { lmu.Lock(); logged = append(logged, e); lmu.Unlock() }
@@ -730,6 +738,25 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
 			t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
 		}
 	})
+	t.Run("declined by a node off the agreed toolchain: put again, no asking used, the bound ends it loudly", func(t *testing.T) { // ADR 0584 clause 2
+		r := newRig(t, goBundle)
+		r.asks.Answer = nil
+		r.b.DeclinePause, r.b.DeclineBound = -1, 150*time.Millisecond
+		r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
+		declined, _ := json.Marshal(delivery.StepDeclined{Declined: true, Lacks: []string{"the agreed toolchain go1.27.1 (recipe 1)"}})
+		waitForAsks(t, r, 1)
+		for ended, deadline := 0, time.Now().Add(5*time.Second); valueOf(t, r.st, r.key).State == delivery.BuildBuilding && time.Now().Before(deadline); time.Sleep(5 * time.Millisecond) {
+			if asks := r.asks.Asks(); len(asks) > ended { // each ask put again is declined in its turn
+				r.asks.End(delivery.Step{ID: asks[ended].ID, Node: "ace", State: delivery.StepRefused, Reason: "this node builds with go1.26.0", Result: declined})
+				ended++
+			}
+		}
+		r.b.Wait()
+		v := valueOf(t, r.st, r.key)
+		if n := len(r.asks.Asks()); v.State != delivery.BuildFailed || n <= DefaultAskings || !strings.Contains(v.Error, "declined for") || !strings.Contains(v.Error, "ace: this node builds with go1.26.0") {
+			t.Fatalf("the key = %+v after %d asks, want it failed after more than %d asks, naming ace and its reason", v, n, DefaultAskings)
+		}
+	})
 	for _, c := range []struct {
 		name    string
 		askings int

modules/mesh-delivery-checker/go.mod

@@ -3,7 +3,7 @@ module mesh-delivery-checker
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a
 	github.com/nats-io/nats-server/v2 v2.11.17
 	github.com/nats-io/nats.go v1.54.0
 )

modules/mesh-delivery-checker/go.sum

@@ -1,5 +1,5 @@
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a h1:AChSJUmHyOZ5smiuTrtDZNjetYNTFb+anRFjXIQHbSA=
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
 github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=

modules/mesh-delivery-checker/internal/check/examiner_test.go

@@ -342,6 +342,11 @@ func TestOnlyThePackageWithoutAKeptVerdictRuns(t *testing.T) { // 1288/AC1
 	if out.Verdict.State != delivery.VerdictPass || len(asks) != 3 || len(asks[0].Needs) != 0 {
 		t.Fatalf("verdict %+v, asks %+v: want a pass of three asks, the first naming no need", out.Verdict, asks)
 	}
+	for _, a := range asks { // ADR 0584: every check ask names the agreed toolchain, so a node off it declines
+		if a.Check.Toolchain == nil || a.Check.Toolchain.Version != "go1.27.1" || a.Check.Toolchain.Recipe != "7" {
+			t.Errorf("a check ask names toolchain %+v, want go1.27.1 recipe 7", a.Check.Toolchain)
+		}
+	}
 	for _, a := range asks[1:] {
 		if strings.Join(a.Needs, ",") != delivery.NeedNodeBuildNode || strings.Join(a.Check.Levels, ",") != delivery.LevelIntegration || strings.Join(a.Check.Packages, ",") != "fixture/node" {
 			t.Errorf("a placed ask %+v, want the integration level needing node-build-node", a)

modules/mesh-delivery-checker/internal/check/judge.go

@@ -166,7 +166,9 @@ type target struct {
 	module    string
 	node      string
 	toolchain string
-	fetched   Fetched
+	// agreed is the toolchain the ask names, so a node off it declines the check (ADR 0584); nil when none is agreed.
+	agreed  *delivery.ToolchainUsed
+	fetched Fetched
 	// fetchArgs are the arguments of the fetch its check ask runs.
 	fetchArgs map[string]any
 	// priority are the Go packages (import paths) or the Node module directories whose test files the merge changes.
@@ -374,6 +376,7 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
 			t := &target{pull: p, module: bm}
 			if tc, none := delivery.AgreedToolchain(st.j.Toolchains.Entries(buildrun.LanguageOf[bm])); none == "" {
 				t.toolchain = tc.Version
+				t.agreed = &delivery.ToolchainUsed{Version: tc.Version, Recipe: tc.Recipe}
 			}
 			args := map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number), "base": p.Trunk}
 			if bm == buildrun.NodeBuildModule && groupPackage != nil && p.Repository != SDKRepository {
@@ -428,7 +431,7 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
 func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, levels, needs, packages []string) *delivery.Verdict {
 	p := t.pull
 	placed := len(needs) > 0
-	job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: levels, Kept: true, Packages: packages}
+	job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: levels, Kept: true, Packages: packages, Toolchain: t.agreed}
 	if st.j.Nodes != nil {
 		names, err := st.j.Nodes.Names(ctx)
 		if err == nil && len(names) == 0 {
@@ -495,8 +498,12 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
 			bound = DefaultDeclineBound
 		}
 		if now.Sub(firstDecline) > bound {
+			lacks := strings.Join(needs, ", ")
+			if lacks == "" {
+				lacks = strings.Join(d.Lacks, ", ") // a node off the agreed toolchain declined it (ADR 0584)
+			}
 			return fail(step.Node, fmt.Sprintf("the check of %s#%d needs a node with %s, and none took it in %s: every node that pulled it lacks it (last %s)",
-				p.Repository, p.Number, strings.Join(needs, ", "), bound, step.Node))
+				p.Repository, p.Number, lacks, bound, step.Node))
 		}
 		st.j.logf("%s#%d: the ask %s was declined on %s, which lacks %s; asked again", p.Repository, p.Number, a.ID, step.Node, strings.Join(d.Lacks, ", "))
 		a.ID, a.At = buildstep.NewID(), now.UTC()

modules/mesh-delivery-indexer/go.mod

@@ -3,7 +3,7 @@ module mesh-delivery-indexer
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a
 	github.com/nats-io/nats-server/v2 v2.11.17
 	github.com/nats-io/nats.go v1.54.0
 )

modules/mesh-delivery-indexer/go.sum

@@ -1,5 +1,5 @@
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a h1:AChSJUmHyOZ5smiuTrtDZNjetYNTFb+anRFjXIQHbSA=
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
 github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=