From 20a7bd0f3c0e5e34fc91766cec1dc2ed6b84f918 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 16:54:26 +0100 Subject: [PATCH] mesh-delivery-indexer: the restart tests wait for the take-in's end and the bus's close MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestIndexerCountsOnlyThePlacedBuildNodesToolchains failed on the build node under load (pipeline main.18) in two ways, both of the test, not of the indexer's count of the placed build nodes' toolchains: - it restarted the indexer as soon as the module was written, before its take-in ended: the stop cancelled the take-in, its source-moved stayed unacknowledged, and the branch's durable (one message at a time) held the next one behind it for its acknowledgement wait, past the test's 20 s bound. Each restart now waits for the source-moved's acknowledgement; TestIndexerWhole's restart the same way. - a source-moved said right after a restart could reach the stopped program's pull, still counted until the bus closed its connection. restart now returns only once the bus has closed every connection of the stopped program, heard on the system account's disconnect events, with no sleep. Under load (8 runs side by side, GOMAXPROCS=1) the test failed 5 of 80 runs before, 2 of 240 with the first change alone, and 0 of 600 with both; TestIndexerWhole 0 of 600. No test added. Builds: design 47 H §743 --- .../mesh-delivery-indexer/integration_test.go | 62 ++++++++++++++++++- .../takein_integration_test.go | 5 ++ 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index 27b82a53..88112068 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -49,10 +49,12 @@ import ( ) // busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the -// controller's user and the indexer's own principal .. +// controller's user and the indexer's own principal .; and the bus's system account, whose disconnect events +// tell a restart when the stopped indexer's connections are gone (restartWaits). const busConf = ` listen: 127.0.0.1:-1 jetstream { store_dir: %q } +system_account: SYS accounts { MESH { jetstream: enabled @@ -61,6 +63,11 @@ accounts { { user: "n1.mesh-delivery-indexer", password: "pw" } ] } + SYS { + users = [ + { user: "sys", password: "pw" } + ] + } } ` @@ -420,12 +427,54 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before . } start() t.Cleanup(func() { stop() }) - // restart stops the program and starts it again on the same bus, as the node-engine does with a new build. - restart := func() { stop(); start() } + // restart stops the program and starts it again on the same bus, as the node-engine does with a new build, once the bus has + // closed every connection of the stopped one (restartWaits). + gone := restartWaits(t, srv) + restart := func() { gone(stop); start() } return js, ctx, candidates, modules, restart } +// restartWaits gives a stop that returns only once the bus has closed every connection the indexer held, heard on the system +// account's disconnect events, never by sleeping. A program's stop closes its connections on its side alone; until the bus +// has closed them its pulls on a durable still count, and a source-moved said in between is delivered to the stopped +// program, unacknowledged for the durable's whole acknowledgement wait, while the branch's durable delivers one message +// at a time (design 47 H §743). +func restartWaits(t *testing.T, srv *server.Server) func(stop func()) { + t.Helper() + sys, err := nats.Connect(srv.ClientURL(), nats.UserInfo("sys", "pw")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(sys.Close) + return func(stop func()) { + t.Helper() + disconnects, err := sys.SubscribeSync("$SYS.ACCOUNT.MESH.DISCONNECT") + if err != nil { + t.Fatal(err) + } + defer func() { _ = disconnects.Unsubscribe() }() + if err := sys.Flush(); err != nil { + t.Fatal(err) + } + open, err := srv.Connz(&server.ConnzOptions{User: "n1.mesh-delivery-indexer"}) + if err != nil { + t.Fatal(err) + } + stop() + for left := open.NumConns; left > 0; { + m, err := disconnects.NextMsg(waitBound) + if err != nil { + t.Fatalf("the bus closed %d of the stopped indexer's %d connections within %s: %v", open.NumConns-left, open.NumConns, waitBound, err) + } + var ev server.DisconnectEventMsg + if json.Unmarshal(m.Data, &ev) == nil && ev.Client.User == "n1.mesh-delivery-indexer" { + left-- + } + } + } +} + // The build seats and their holders as the mesh has them (design 47:482-485 and 1583: a language's build module is the // module claiming the seat whose `acts` is step build on that language): the seat definitions as modules/mesh-delivery-contracts/seats// // seat.json has them (acts included) and the manifests as the catalog's modules/mesh-build-go and mesh-build-node claim them. @@ -691,6 +740,7 @@ func TestIndexerBuildsARunsCandidateAndDropsItWhenEveryMembersPipelineEnded(t *t func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) { const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381" var assignments, toolchains jetstream.KeyValue + var heard *takeIns place := func(ctx context.Context, t *testing.T, node string, modules ...string) { p := meshmoduleassigner.Assignments{Node: node} for _, m := range modules { @@ -712,6 +762,7 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) { assignerLate = up t.Cleanup(func() { assignerLate = nil }) js, ctx, _, modules, restart := startIndexerRun(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) { + heard = hearTakeIns(t, js.Conn()) registerAvahi(t, ctx, js, "main") kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ToolchainsBucket("mesh-build-go"), History: 1}) if err != nil { @@ -762,6 +813,10 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) { if !agreed(held) { t.Fatalf("within 20 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27 alone", held.Fp2, held.None2, held.Toolchains) } + // The module is written before its take-in ends: each restart below waits for the source-moved's acknowledgement, the + // take-in's end (design 47 H §743), so the stop cancels no take-in and leaves none unacknowledged ahead of the next one + // on the branch's durable, which delivers one message at a time. + heard.acked(t, "main") place(ctx, t, "n2", "mesh-build-go") key, _ := index.ToolchainKey("n2", "go") value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: "go1.28.1", Recipe: "1", At: ptr(time.Now().Add(time.Second))}) @@ -789,6 +844,7 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) { if m := read(disagree); !disagree(m) { t.Fatalf("within 20 s of a merge after n2's assignments avahi has fp2 %q, none2 %q: want the builders to disagree", m.Fp2, m.None2) } + heard.acked(t, "main") // The view moves only at a take-in (design 47 H-A-49, H-A-103): with n2's assignments taken away, the next take-in agrees. place(ctx, t, "n2") restart() diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go index 889e7cdc..98cc851a 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go @@ -457,6 +457,11 @@ func TestIndexerWhole(t *testing.T) { } sayPipeline(t, ctx, js, false, "main", 12) untilKey(t, ctx, pins, index.PinKey("main", 12), pinned("pipeline.main.12")) + // The restart waits for the take-ins' ends, their source-moved acknowledged (design 47 H §743: main 10, 11, 11 given + // again and 12), so the stop cancels none and leaves none unacknowledged ahead of main 13. + for range 4 { + heard.acked(t, "main") + } restart() for _, name := range []string{"main.10", "main.11"} { if _, _, err := index.ReadRevision(ctx, objects, name); err != nil {