Run the slow test packages in shards and stop compiling the standard library per judge test (hq issue 717)
mesh/merge-gate head 3ec4473b29b2 merged into trunk df8c1ccf1fff — pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every …
mesh/repo-check head 3ec4473b29b2 merged into trunk df8c1ccf1fff — pass: THE CHANGE ALTERS ITS OWN CHECK (merge-check.sh): main's version judged it; the ch…
mesh/delivery delivered

The controller's check spent 3-5 minutes in cmd/mesh-controller, whose few hundred store tests run one
after another, while most packages end in seconds. merge-check.sh now compiles each slow package once and
deals its tests round-robin into processes that run beside each other and beside the rest of the suite;
every test still runs once, under the race detector, through its package's TestMain. The listed store
tests run beside the suite instead of after it.

internal/builder's judge tests ran a real go build with a fresh Go cache each, compiling the standard
library every time: half the suite's CPU. The fake runtime fills an empty cache from one warmed per
process. The lease tests run in parallel. The link test of slow work now sets its one-second window after
the serving loop asserted the controller's thirty-second one, so it proves the heartbeat (it passed
without it) in half the time.
This commit is contained in:
2026-10-14 20:38:35 +02:00
parent 622fb1e9cb
commit 3ec4473b29
7 changed files with 294 additions and 36 deletions
+1 -1
View File
@@ -332,7 +332,7 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
// as the controller says it for its own long handlers, so the server neither hands
// the ask to a second machine nor counts the wait against its deliveries.
working := make(chan struct{})
go stillWorking(msg, working)
go stillWorking(msg, working, heartbeatWhileWorking)
do(ctx, build)
close(working)
}
+6 -5
View File
@@ -220,21 +220,22 @@ func (n *natsInbound) deliver(ctx context.Context, act func(context.Context, Con
// stops saying so, and the bus redelivers exactly as it should.
working := make(chan struct{})
defer close(working)
go stillWorking(msg, working)
go stillWorking(msg, working, heartbeatWhileWorking)
}
act(ctx, m)
}
// heartbeatWhileWorking is how often a handler still running tells the bus so — comfortably inside
// the shortest acknowledgement window the mesh gives any of its consumers.
const heartbeatWhileWorking = 10 * time.Second
// the shortest acknowledgement window the mesh gives any of its consumers. A variable only so a test can
// outlive a window of a second in a few (novox/hq issue 717); nothing else sets it.
var heartbeatWhileWorking = 10 * time.Second
// stillWorking keeps one message alive until the work on it returns.
//
// An error is not worth reporting: what the bus does when it is not told is redeliver, which is
// exactly what happens if this fails, and the handler's own outcome is the thing worth logging.
func stillWorking(msg *nats.Msg, done <-chan struct{}) {
tick := time.NewTicker(heartbeatWhileWorking)
func stillWorking(msg *nats.Msg, done <-chan struct{}, every time.Duration) {
tick := time.NewTicker(every)
defer tick.Stop()
for {
select {
+40 -10
View File
@@ -405,16 +405,15 @@ func TestEverySubjectTheControllerFollowsDecodesToAKind(t *testing.T) {
// The bus waits a fixed time to be told a message was taken and then redelivers, which is right for
// a consumer that died and wrong for one that is busy. Acting on a merge builds modules — minutes
// against a thirty-second window — and the same merge was handed over five times while the first
// build was still running (2026-09-28). Here the window is two seconds and the work takes six.
// build was still running (2026-09-28). Here the window is a second and the work takes three.
func TestNatsWorkSlowerThanTheWindowIsNotHandedOverAgain(t *testing.T) {
// Work that outlives a window of a second three times over, said while it runs four times a window (novox/hq
// issue 717: the test once set its window before the serving loop asserted the controller's own of thirty
// seconds, so it waited nine seconds and passed with the heartbeat removed).
before := heartbeatWhileWorking
heartbeatWhileWorking = 250 * time.Millisecond
t.Cleanup(func() { heartbeatWhileWorking = before })
js := aBus(t)
// The controller's own consumer, with a window short enough to outlive in a test.
if err := js.EnsureConsumer(broker.Consumer{
Name: broker.ControllerName, Stream: "CONTROL", Push: true, AckWaitSeconds: 2,
Why: "a window short enough to outlive in a test",
}); err != nil {
t.Fatal(err)
}
var mu sync.Mutex
handled := 0
slow := make(chan struct{})
@@ -426,10 +425,32 @@ func TestNatsWorkSlowerThanTheWindowIsNotHandedOverAgain(t *testing.T) {
first := handled == 1
mu.Unlock()
if first {
time.Sleep(6 * time.Second)
time.Sleep(3 * time.Second)
close(slow)
}
}}
// The controller's own consumer, given a window short enough to outlive in a test once the serving loop has
// asserted it and bound to it: asserted after this, it would be the controller's window of thirty seconds,
// which is how this test once waited out a window it never had.
for deadline := time.Now().Add(10 * time.Second); ; {
info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName)
if err == nil && info.PushBound {
break
}
if time.Now().After(deadline) {
t.Fatalf("the serving loop never bound the controller's consumer: %v", err)
}
time.Sleep(10 * time.Millisecond)
}
if err := js.EnsureConsumer(broker.Consumer{
Name: broker.ControllerName, Stream: "CONTROL", Push: true, AckWaitSeconds: 1,
Why: "a window short enough to outlive in a test",
}); err != nil {
t.Fatal(err)
}
if info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName); err != nil || info.Config.AckWait != time.Second {
t.Fatalf("the controller's consumer does not have the test's window of a second: %v", err)
}
body, err := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}})
if err != nil {
@@ -444,12 +465,21 @@ func TestNatsWorkSlowerThanTheWindowIsNotHandedOverAgain(t *testing.T) {
t.Fatal("the slow work never finished")
}
// A moment for a redelivery to arrive, if the bus were going to send one.
time.Sleep(3 * time.Second)
time.Sleep(1500 * time.Millisecond)
mu.Lock()
defer mu.Unlock()
if handled != 1 {
t.Fatalf("one report was handled %d times, so slow work is run again while it is running", handled)
}
// Not handed over again at all: the loop drops a copy of what it is already doing, so the count above alone
// would pass on a bus that redelivered the work every window; the bus's own count says it did not.
info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName)
if err != nil {
t.Fatal(err)
}
if info.NumRedelivered != 0 {
t.Fatalf("the bus handed the report over again %d time(s) while it was being worked on", info.NumRedelivered)
}
}
// slowly is a listener that runs whatever it was given.