Bound a build or check ask's step from its start, and drop the checker's whole-check bound, so asks waiting in the streams during a large rebuild are never failed and asked twice (issue 1887, review round 2 item 1; ADR 0525 §3, §6)

This commit is contained in:
2026-10-23 16:45:35 +02:00
parent 97b42e3f0d
commit 5ddc221833
20 changed files with 106 additions and 21 deletions
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023141925-9205222a98bb
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143646-fc4f57c4b3a3
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+4
View File
@@ -1,5 +1,9 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023141925-9205222a98bb h1:D70wIl0aHcEXhb9/7aDV0uDrP+A5wLsBcQ5kEad7QTQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023141925-9205222a98bb/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143221-88fa74ce5e3c h1:+0PXXmcjxMTY/2w1s9PWET3Gl6JZgmsgNgzh265ByKw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143221-88fa74ce5e3c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143646-fc4f57c4b3a3 h1:6X2ZM5dRQ3NnKKxT+6CCCBb636JOITLuRtTZ64rbvIQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143646-fc4f57c4b3a3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -31,8 +31,9 @@ import (
)
const (
// defaultStepBound is how long the builder waits for one step of a build module (a fetch downloads the Go modules, a
// build compiles).
// defaultStepBound is how long one ask's step may run once a build node took it (a fetch downloads the Go modules, a
// build compiles): from its start, never from its put, so an ask waiting in its stream for a free slot is not failed
// (buildrun.Asker.Bound; ADR 0525 §3, §6).
defaultStepBound = 30 * time.Minute
)
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-builder
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023141925-9205222a98bb
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143646-fc4f57c4b3a3
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+4
View File
@@ -1,5 +1,9 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023141925-9205222a98bb h1:D70wIl0aHcEXhb9/7aDV0uDrP+A5wLsBcQ5kEad7QTQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023141925-9205222a98bb/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143221-88fa74ce5e3c h1:+0PXXmcjxMTY/2w1s9PWET3Gl6JZgmsgNgzh265ByKw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143221-88fa74ce5e3c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143646-fc4f57c4b3a3 h1:6X2ZM5dRQ3NnKKxT+6CCCBb636JOITLuRtTZ64rbvIQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143646-fc4f57c4b3a3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -613,3 +613,26 @@ func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // re
t.Errorf("build = %+v, want done with a reference", v)
}
}
// Review round 2 item 1: a build ask that waits in its stream longer than the step bound is not failed and not asked
// again; the key is done when a node takes it.
func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
r := newRig(t, goBundle)
r.asks.Answer = nil
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond}
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
for deadline := time.Now().Add(2 * time.Second); len(r.asks.Asks()) == 0; time.Sleep(time.Millisecond) {
if time.Now().After(deadline) {
t.Fatal("no ask was put")
}
}
time.Sleep(100 * time.Millisecond)
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding {
t.Fatalf("the key = %+v while its ask waits, want building", v)
}
r.asks.End(r.answer(r.asks.Asks()[0]))
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 {
t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
}
}
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-checker
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023141925-9205222a98bb
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143646-fc4f57c4b3a3
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+4
View File
@@ -1,5 +1,9 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023141925-9205222a98bb h1:D70wIl0aHcEXhb9/7aDV0uDrP+A5wLsBcQ5kEad7QTQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023141925-9205222a98bb/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143221-88fa74ce5e3c h1:+0PXXmcjxMTY/2w1s9PWET3Gl6JZgmsgNgzh265ByKw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143221-88fa74ce5e3c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143646-fc4f57c4b3a3 h1:6X2ZM5dRQ3NnKKxT+6CCCBb636JOITLuRtTZ64rbvIQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143646-fc4f57c4b3a3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -11,8 +11,11 @@ import (
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// DefaultBound is how long a check may run before it is ended failed naming the step it was stuck at (ADR 0484, issue 1678).
const DefaultBound = 30 * time.Minute
// DefaultBound is how long a check may run before it is ended failed naming the step it was stuck at (ADR 0484, issue 1678):
// none since R4g. A check's asks wait in the build seats' streams as long as the backlog is (ADR 0525 §9), and each ask's
// step is bounded from its start on the node that took it (buildrun.Asker.Bound, DefaultStepBound); a bound set by
// MESH_CHECKER_BOUND still ends a check that ran past it.
const DefaultBound time.Duration = 0
// NodesWords is in the Words of a failure that is the build nodes' (none qualifies, the toolchains disagree): such a verdict
// is judged again when the nodes change, by a watch and never a timer (ADR 0437 §8).
@@ -66,7 +69,7 @@ type Checker struct {
Owed KV
Judge Judge
Trunks interface{ Heads() map[string]string }
// Bound is how long one check may run; DefaultBound when 0.
// Bound is how long one check may run; none when 0 (DefaultBound).
Bound time.Duration
// Link is the base of the address a status points at; the verdict key follows.
Link string
@@ -765,10 +765,13 @@ func TestNoStepOfACheckCarriesPublish(t *testing.T) { // ADR 0500 §4, 1309 (the
// time) ends the check loudly at the asker's bound; the checker holds no place to free.
func TestAnAskWhoseStepNeverEndsEndsTheCheckLoudly(t *testing.T) {
b := newBench(t)
b.asks.Answer = nil
// A node took the ask and its step stays running: the bound runs from its start (review round 2 item 1).
b.asks.Answer = func(a delivery.StepAsk) delivery.Step {
return delivery.Step{ID: a.ID, Tool: "check ask", Node: "shanks", State: delivery.StepRunning, Started: time.Now()}
}
b.ex.Asks = buildrun.Asker{Publisher: b.asks, Waiter: b.asks, Bound: 20 * time.Millisecond}
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
if f := out.Verdict.Failed; out.Verdict.State != delivery.VerdictFail || !strings.Contains(f.Words, "did not end within") {
if f := out.Verdict.Failed; out.Verdict.State != delivery.VerdictFail || !strings.Contains(f.Words, "on shanks did not end within") {
t.Fatalf("verdict = %+v", out.Verdict)
}
if n := len(b.asks.Asks()); n != 1 || b.asks.Asks()[0].Kind != delivery.AskCheck || b.asks.Asks()[0].Check == nil {
@@ -795,3 +798,29 @@ func TestACheckIsOneAskAndAReJudgeGoesOnTheRecheckStream(t *testing.T) {
t.Fatalf("asks = %+v, want the re-judge on recheck", asks)
}
}
// Review round 2 item 1: a check ask that waits in its stream longer than the step bound (no node has a free slot during a
// large rebuild) is not failed: the check goes on when a node takes it, and no second ask is put.
func TestACheckAskWaitingInItsStreamIsNotBounded(t *testing.T) {
b := newBench(t)
answer := b.asks.Answer
b.asks.Answer = nil
b.ex.Asks = buildrun.Asker{Publisher: b.asks, Waiter: b.asks, Bound: 20 * time.Millisecond}
done := make(chan Outcome, 1)
go func() { done <- b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) }()
for deadline := time.Now().Add(2 * time.Second); len(b.asks.Asks()) == 0; time.Sleep(time.Millisecond) {
if time.Now().After(deadline) {
t.Fatal("no ask was put")
}
}
time.Sleep(100 * time.Millisecond) // five bounds in the stream
select {
case out := <-done:
t.Fatalf("the check ended while its ask waited: %+v", out.Verdict)
default:
}
b.asks.End(answer(b.asks.Asks()[0])) // a node took it and ran it
if out := <-done; out.Verdict.State != delivery.VerdictPass || len(b.asks.Asks()) != 1 {
t.Fatalf("verdict %+v after %d asks, want a pass from the one ask", out.Verdict, len(b.asks.Asks()))
}
}
@@ -382,8 +382,8 @@ func (r Reader) Status() (Status, error) {
return st, nil
}
// DefaultStepBound is how long the checker waits for one step of a build module (a fetch downloads the Go modules, a test
// level runs a repository's tests); past it the check fails naming the step, and the step's place on its node is free.
// DefaultStepBound is how long one check ask's step may run once a build node took it (its key written running; the fetch,
// the digests and the levels), never counting its wait in the stream; past it the check fails naming the node.
const DefaultStepBound = 30 * time.Minute
// leftWhy says why a key in the running state runs no step any more, or "" when it can still be running: its build module
@@ -353,13 +353,14 @@ func (tr *Tryer) run(ctx context.Context, key string) {
tr.logf("%s: %s", key, t.State)
}
// judgeWithin runs a case within the bound (DefaultBound when 0). A judge that fails to judge, and a check that ran past its
// bound, are verdicts of failure naming the machine and the step, and judged says false for them.
// judgeWithin runs a case within the bound (none when 0: the asks' steps are bounded from their start, ADR 0525 §6). A
// judge that fails to judge, and a check that ran past its bound, are verdicts of failure naming the machine and the step,
// and judged says false for them.
func judgeWithin(ctx context.Context, j Judge, cs *Case, bound time.Duration) (Outcome, bool) {
if bound <= 0 {
bound = DefaultBound
run, cancel := context.WithCancel(ctx)
if bound > 0 {
run, cancel = context.WithTimeoutCause(ctx, bound, errBound)
}
run, cancel := context.WithTimeoutCause(ctx, bound, errBound)
defer cancel()
out, err := j.Judge(run, cs)
stage, node := cs.Progress.Now()
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023141925-9205222a98bb
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143646-fc4f57c4b3a3
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+4
View File
@@ -1,5 +1,9 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023141925-9205222a98bb h1:D70wIl0aHcEXhb9/7aDV0uDrP+A5wLsBcQ5kEad7QTQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023141925-9205222a98bb/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143221-88fa74ce5e3c h1:+0PXXmcjxMTY/2w1s9PWET3Gl6JZgmsgNgzh265ByKw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143221-88fa74ce5e3c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143646-fc4f57c4b3a3 h1:6X2ZM5dRQ3NnKKxT+6CCCBb636JOITLuRtTZ64rbvIQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143646-fc4f57c4b3a3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-teller
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023141925-9205222a98bb
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143646-fc4f57c4b3a3
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+4
View File
@@ -1,5 +1,9 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023141925-9205222a98bb h1:D70wIl0aHcEXhb9/7aDV0uDrP+A5wLsBcQ5kEad7QTQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023141925-9205222a98bb/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143221-88fa74ce5e3c h1:+0PXXmcjxMTY/2w1s9PWET3Gl6JZgmsgNgzh265ByKw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143221-88fa74ce5e3c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143646-fc4f57c4b3a3 h1:6X2ZM5dRQ3NnKKxT+6CCCBb636JOITLuRtTZ64rbvIQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143646-fc4f57c4b3a3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-view
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023141925-9205222a98bb
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143646-fc4f57c4b3a3
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+4
View File
@@ -1,5 +1,9 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023141925-9205222a98bb h1:D70wIl0aHcEXhb9/7aDV0uDrP+A5wLsBcQ5kEad7QTQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023141925-9205222a98bb/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143221-88fa74ce5e3c h1:+0PXXmcjxMTY/2w1s9PWET3Gl6JZgmsgNgzh265ByKw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143221-88fa74ce5e3c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143646-fc4f57c4b3a3 h1:6X2ZM5dRQ3NnKKxT+6CCCBb636JOITLuRtTZ64rbvIQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143646-fc4f57c4b3a3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-watchman
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023141925-9205222a98bb
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143646-fc4f57c4b3a3
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+4
View File
@@ -1,5 +1,9 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023141925-9205222a98bb h1:D70wIl0aHcEXhb9/7aDV0uDrP+A5wLsBcQ5kEad7QTQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023141925-9205222a98bb/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143221-88fa74ce5e3c h1:+0PXXmcjxMTY/2w1s9PWET3Gl6JZgmsgNgzh265ByKw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143221-88fa74ce5e3c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143646-fc4f57c4b3a3 h1:6X2ZM5dRQ3NnKKxT+6CCCBb636JOITLuRtTZ64rbvIQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023143646-fc4f57c4b3a3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=