diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index 27b82a53..88112068 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -49,10 +49,12 @@ import ( ) // busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the -// controller's user and the indexer's own principal .. +// controller's user and the indexer's own principal .; and the bus's system account, whose disconnect events +// tell a restart when the stopped indexer's connections are gone (restartWaits). const busConf = ` listen: 127.0.0.1:-1 jetstream { store_dir: %q } +system_account: SYS accounts { MESH { jetstream: enabled @@ -61,6 +63,11 @@ accounts { { user: "n1.mesh-delivery-indexer", password: "pw" } ] } + SYS { + users = [ + { user: "sys", password: "pw" } + ] + } } ` @@ -420,12 +427,54 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before . } start() t.Cleanup(func() { stop() }) - // restart stops the program and starts it again on the same bus, as the node-engine does with a new build. - restart := func() { stop(); start() } + // restart stops the program and starts it again on the same bus, as the node-engine does with a new build, once the bus has + // closed every connection of the stopped one (restartWaits). + gone := restartWaits(t, srv) + restart := func() { gone(stop); start() } return js, ctx, candidates, modules, restart } +// restartWaits gives a stop that returns only once the bus has closed every connection the indexer held, heard on the system +// account's disconnect events, never by sleeping. A program's stop closes its connections on its side alone; until the bus +// has closed them its pulls on a durable still count, and a source-moved said in between is delivered to the stopped +// program, unacknowledged for the durable's whole acknowledgement wait, while the branch's durable delivers one message +// at a time (design 47 H §743). +func restartWaits(t *testing.T, srv *server.Server) func(stop func()) { + t.Helper() + sys, err := nats.Connect(srv.ClientURL(), nats.UserInfo("sys", "pw")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(sys.Close) + return func(stop func()) { + t.Helper() + disconnects, err := sys.SubscribeSync("$SYS.ACCOUNT.MESH.DISCONNECT") + if err != nil { + t.Fatal(err) + } + defer func() { _ = disconnects.Unsubscribe() }() + if err := sys.Flush(); err != nil { + t.Fatal(err) + } + open, err := srv.Connz(&server.ConnzOptions{User: "n1.mesh-delivery-indexer"}) + if err != nil { + t.Fatal(err) + } + stop() + for left := open.NumConns; left > 0; { + m, err := disconnects.NextMsg(waitBound) + if err != nil { + t.Fatalf("the bus closed %d of the stopped indexer's %d connections within %s: %v", open.NumConns-left, open.NumConns, waitBound, err) + } + var ev server.DisconnectEventMsg + if json.Unmarshal(m.Data, &ev) == nil && ev.Client.User == "n1.mesh-delivery-indexer" { + left-- + } + } + } +} + // The build seats and their holders as the mesh has them (design 47:482-485 and 1583: a language's build module is the // module claiming the seat whose `acts` is step build on that language): the seat definitions as modules/mesh-delivery-contracts/seats// // seat.json has them (acts included) and the manifests as the catalog's modules/mesh-build-go and mesh-build-node claim them. @@ -691,6 +740,7 @@ func TestIndexerBuildsARunsCandidateAndDropsItWhenEveryMembersPipelineEnded(t *t func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) { const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381" var assignments, toolchains jetstream.KeyValue + var heard *takeIns place := func(ctx context.Context, t *testing.T, node string, modules ...string) { p := meshmoduleassigner.Assignments{Node: node} for _, m := range modules { @@ -712,6 +762,7 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) { assignerLate = up t.Cleanup(func() { assignerLate = nil }) js, ctx, _, modules, restart := startIndexerRun(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) { + heard = hearTakeIns(t, js.Conn()) registerAvahi(t, ctx, js, "main") kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ToolchainsBucket("mesh-build-go"), History: 1}) if err != nil { @@ -762,6 +813,10 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) { if !agreed(held) { t.Fatalf("within 20 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27 alone", held.Fp2, held.None2, held.Toolchains) } + // The module is written before its take-in ends: each restart below waits for the source-moved's acknowledgement, the + // take-in's end (design 47 H §743), so the stop cancels no take-in and leaves none unacknowledged ahead of the next one + // on the branch's durable, which delivers one message at a time. + heard.acked(t, "main") place(ctx, t, "n2", "mesh-build-go") key, _ := index.ToolchainKey("n2", "go") value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: "go1.28.1", Recipe: "1", At: ptr(time.Now().Add(time.Second))}) @@ -789,6 +844,7 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) { if m := read(disagree); !disagree(m) { t.Fatalf("within 20 s of a merge after n2's assignments avahi has fp2 %q, none2 %q: want the builders to disagree", m.Fp2, m.None2) } + heard.acked(t, "main") // The view moves only at a take-in (design 47 H-A-49, H-A-103): with n2's assignments taken away, the next take-in agrees. place(ctx, t, "n2") restart() diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go index 889e7cdc..98cc851a 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go @@ -457,6 +457,11 @@ func TestIndexerWhole(t *testing.T) { } sayPipeline(t, ctx, js, false, "main", 12) untilKey(t, ctx, pins, index.PinKey("main", 12), pinned("pipeline.main.12")) + // The restart waits for the take-ins' ends, their source-moved acknowledged (design 47 H §743: main 10, 11, 11 given + // again and 12), so the stop cancels none and leaves none unacknowledged ahead of main 13. + for range 4 { + heard.acked(t, "main") + } restart() for _, name := range []string{"main.10", "main.11"} { if _, _, err := index.ReadRevision(ctx, objects, name); err != nil {