mesh-delivery-indexer: the restart tests wait for the take-in's end and the bus's close

TestIndexerCountsOnlyThePlacedBuildNodesToolchains failed on the build node under load
(pipeline main.18) in two ways, both of the test, not of the indexer's count of the
placed build nodes' toolchains:

- it restarted the indexer as soon as the module was written, before its take-in
  ended: the stop cancelled the take-in, its source-moved stayed unacknowledged, and
  the branch's durable (one message at a time) held the next one behind it for its
  acknowledgement wait, past the test's 20 s bound. Each restart now waits for the
  source-moved's acknowledgement; TestIndexerWhole's restart the same way.
- a source-moved said right after a restart could reach the stopped program's pull,
  still counted until the bus closed its connection. restart now returns only once
  the bus has closed every connection of the stopped program, heard on the system
  account's disconnect events, with no sleep.

Under load (8 runs side by side, GOMAXPROCS=1) the test failed 5 of 80 runs before,
2 of 240 with the first change alone, and 0 of 600 with both; TestIndexerWhole 0 of 600.
No test added.

Builds: design 47 H §743
This commit is contained in:
2026-11-20 16:54:26 +01:00
parent 351eda0867
commit 20a7bd0f3c
2 changed files with 64 additions and 3 deletions
@@ -49,10 +49,12 @@ import (
) )
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the // busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the
// controller's user and the indexer's own principal <node>.<module>. // controller's user and the indexer's own principal <node>.<module>; and the bus's system account, whose disconnect events
// tell a restart when the stopped indexer's connections are gone (restartWaits).
const busConf = ` const busConf = `
listen: 127.0.0.1:-1 listen: 127.0.0.1:-1
jetstream { store_dir: %q } jetstream { store_dir: %q }
system_account: SYS
accounts { accounts {
MESH { MESH {
jetstream: enabled jetstream: enabled
@@ -61,6 +63,11 @@ accounts {
{ user: "n1.mesh-delivery-indexer", password: "pw" } { user: "n1.mesh-delivery-indexer", password: "pw" }
] ]
} }
SYS {
users = [
{ user: "sys", password: "pw" }
]
}
} }
` `
@@ -420,12 +427,54 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before .
} }
start() start()
t.Cleanup(func() { stop() }) t.Cleanup(func() { stop() })
// restart stops the program and starts it again on the same bus, as the node-engine does with a new build. // restart stops the program and starts it again on the same bus, as the node-engine does with a new build, once the bus has
restart := func() { stop(); start() } // closed every connection of the stopped one (restartWaits).
gone := restartWaits(t, srv)
restart := func() { gone(stop); start() }
return js, ctx, candidates, modules, restart return js, ctx, candidates, modules, restart
} }
// restartWaits gives a stop that returns only once the bus has closed every connection the indexer held, heard on the system
// account's disconnect events, never by sleeping. A program's stop closes its connections on its side alone; until the bus
// has closed them its pulls on a durable still count, and a source-moved said in between is delivered to the stopped
// program, unacknowledged for the durable's whole acknowledgement wait, while the branch's durable delivers one message
// at a time (design 47 H §743).
func restartWaits(t *testing.T, srv *server.Server) func(stop func()) {
t.Helper()
sys, err := nats.Connect(srv.ClientURL(), nats.UserInfo("sys", "pw"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(sys.Close)
return func(stop func()) {
t.Helper()
disconnects, err := sys.SubscribeSync("$SYS.ACCOUNT.MESH.DISCONNECT")
if err != nil {
t.Fatal(err)
}
defer func() { _ = disconnects.Unsubscribe() }()
if err := sys.Flush(); err != nil {
t.Fatal(err)
}
open, err := srv.Connz(&server.ConnzOptions{User: "n1.mesh-delivery-indexer"})
if err != nil {
t.Fatal(err)
}
stop()
for left := open.NumConns; left > 0; {
m, err := disconnects.NextMsg(waitBound)
if err != nil {
t.Fatalf("the bus closed %d of the stopped indexer's %d connections within %s: %v", open.NumConns-left, open.NumConns, waitBound, err)
}
var ev server.DisconnectEventMsg
if json.Unmarshal(m.Data, &ev) == nil && ev.Client.User == "n1.mesh-delivery-indexer" {
left--
}
}
}
}
// The build seats and their holders as the mesh has them (design 47:482-485 and 1583: a language's build module is the // The build seats and their holders as the mesh has them (design 47:482-485 and 1583: a language's build module is the
// module claiming the seat whose `acts` is step build on that language): the seat definitions as modules/mesh-delivery-contracts/seats/<seat>/ // module claiming the seat whose `acts` is step build on that language): the seat definitions as modules/mesh-delivery-contracts/seats/<seat>/
// seat.json has them (acts included) and the manifests as the catalog's modules/mesh-build-go and mesh-build-node claim them. // seat.json has them (acts included) and the manifests as the catalog's modules/mesh-build-go and mesh-build-node claim them.
@@ -691,6 +740,7 @@ func TestIndexerBuildsARunsCandidateAndDropsItWhenEveryMembersPipelineEnded(t *t
func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) { func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381" const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"
var assignments, toolchains jetstream.KeyValue var assignments, toolchains jetstream.KeyValue
var heard *takeIns
place := func(ctx context.Context, t *testing.T, node string, modules ...string) { place := func(ctx context.Context, t *testing.T, node string, modules ...string) {
p := meshmoduleassigner.Assignments{Node: node} p := meshmoduleassigner.Assignments{Node: node}
for _, m := range modules { for _, m := range modules {
@@ -712,6 +762,7 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
assignerLate = up assignerLate = up
t.Cleanup(func() { assignerLate = nil }) t.Cleanup(func() { assignerLate = nil })
js, ctx, _, modules, restart := startIndexerRun(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) { js, ctx, _, modules, restart := startIndexerRun(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) {
heard = hearTakeIns(t, js.Conn())
registerAvahi(t, ctx, js, "main") registerAvahi(t, ctx, js, "main")
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ToolchainsBucket("mesh-build-go"), History: 1}) kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ToolchainsBucket("mesh-build-go"), History: 1})
if err != nil { if err != nil {
@@ -762,6 +813,10 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
if !agreed(held) { if !agreed(held) {
t.Fatalf("within 20 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27 alone", held.Fp2, held.None2, held.Toolchains) t.Fatalf("within 20 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27 alone", held.Fp2, held.None2, held.Toolchains)
} }
// The module is written before its take-in ends: each restart below waits for the source-moved's acknowledgement, the
// take-in's end (design 47 H §743), so the stop cancels no take-in and leaves none unacknowledged ahead of the next one
// on the branch's durable, which delivers one message at a time.
heard.acked(t, "main")
place(ctx, t, "n2", "mesh-build-go") place(ctx, t, "n2", "mesh-build-go")
key, _ := index.ToolchainKey("n2", "go") key, _ := index.ToolchainKey("n2", "go")
value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: "go1.28.1", Recipe: "1", At: ptr(time.Now().Add(time.Second))}) value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: "go1.28.1", Recipe: "1", At: ptr(time.Now().Add(time.Second))})
@@ -789,6 +844,7 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
if m := read(disagree); !disagree(m) { if m := read(disagree); !disagree(m) {
t.Fatalf("within 20 s of a merge after n2's assignments avahi has fp2 %q, none2 %q: want the builders to disagree", m.Fp2, m.None2) t.Fatalf("within 20 s of a merge after n2's assignments avahi has fp2 %q, none2 %q: want the builders to disagree", m.Fp2, m.None2)
} }
heard.acked(t, "main")
// The view moves only at a take-in (design 47 H-A-49, H-A-103): with n2's assignments taken away, the next take-in agrees. // The view moves only at a take-in (design 47 H-A-49, H-A-103): with n2's assignments taken away, the next take-in agrees.
place(ctx, t, "n2") place(ctx, t, "n2")
restart() restart()
@@ -457,6 +457,11 @@ func TestIndexerWhole(t *testing.T) {
} }
sayPipeline(t, ctx, js, false, "main", 12) sayPipeline(t, ctx, js, false, "main", 12)
untilKey(t, ctx, pins, index.PinKey("main", 12), pinned("pipeline.main.12")) untilKey(t, ctx, pins, index.PinKey("main", 12), pinned("pipeline.main.12"))
// The restart waits for the take-ins' ends, their source-moved acknowledged (design 47 H §743: main 10, 11, 11 given
// again and 12), so the stop cancels none and leaves none unacknowledged ahead of main 13.
for range 4 {
heard.acked(t, "main")
}
restart() restart()
for _, name := range []string{"main.10", "main.11"} { for _, name := range []string{"main.10", "main.11"} {
if _, _, err := index.ReadRevision(ctx, objects, name); err != nil { if _, _, err := index.ReadRevision(ctx, objects, name); err != nil {