From 08dde53e331825a18bc24fc677eea6355bf34dcb Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 18:30:27 +0100 Subject: [PATCH] mesh-delivery-indexer: the run on request's bed lives in TestIndexerWhole, its restart after the run-started's acknowledgement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestIndexerBuildsARunsCandidateAndDropsItWhenEveryMembersPipelineEnded failed on the build node under load (pipelines main.29 and main.31). It restarted the indexer as soon as the candidate was written, before the run-started was acknowledged (the write comes first), and it polled both waits with sleeps: the same kind as the toolchains bed's restart fixed in 20a7bd0f. A stop between the write and the acknowledgement leaves the run-started for redelivery after its acknowledgement wait, which writes the candidate again once its run has ended. The run is now one more step of TestIndexerWhole around its restart: run-started of run 9 writes candidates.run-9 and the run's key (watched, not polled), the restart waits for the run-started's acknowledgement beside the source-moved's, and the member's pipeline-ended after the restart drops both. The separate test is removed: the module's integration tests go from 541 to 540; no test added. Builds: design 47 H §743 --- .../mesh-delivery-indexer/integration_test.go | 86 ------------------- .../takein_integration_test.go | 62 +++++++++++-- 2 files changed, 57 insertions(+), 91 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index 745cfab5..737b27c3 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -31,7 +31,6 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" @@ -647,91 +646,6 @@ func (l buildLog) end(t *testing.T, repo, module, path string) { } } -// TestIndexerBuildsARunsCandidateAndDropsItWhenEveryMembersPipelineEnded (design 47 G §64, §67 test 64, §190; A-160, A-174): -// the indexer starts with the runner's streams run-started and pipeline-ended and its durables on them, its state runs and a -// registration of avahi; the runner says run-started of run 9 with avahi at a commit on a branch no registration watches; -// within 20 s candidates.run-9 holds one pull of number 0 at that commit, avahi's manifest read through the seat git; the -// indexer is restarted, the runner says avahi's pipeline of run 9 ended, and the candidate and the run's key go. -func TestIndexerBuildsARunsCandidateAndDropsItWhenEveryMembersPipelineEnded(t *testing.T) { - manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json") - if err != nil { - t.Fatal(err) - } - const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381" - const commit = "0123456789abcdef0123456789abcdef01234567" - var runs jetstream.KeyValue - js, ctx, candidates, _, restart := startIndexerRun(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) { - var err error - if runs, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.RunsState), History: 1}); err != nil { - t.Fatal(err) - } - regs, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.RegistrationsState), History: 1}) - if err != nil { - t.Fatal(err) - } - reg, _ := json.Marshal(meshcatalog.Registration{V: 1, Module: "avahi", Origin: "novox/mesh-catalog", Path: "modules/avahi", Branches: []string{"main"}}) - if _, err := regs.Create(ctx, index.RegistrationKey("avahi"), reg); err != nil { - t.Fatal(err) - } - seedBuildModules(t, ctx, js, nil) - // The seat git as the forge module serves it: avahi's directory and its manifest at the run's commit. - reply := func(m *nats.Msg, result any) { - raw, _ := json.Marshal(map[string]any{"result": result, "node": "n0"}) - _ = m.Respond(raw) - } - if _, err := js.Conn().Subscribe("mesh.seat.git.tool.trees", func(m *nats.Msg) { - reply(m, git.TreesAnswer{Root: "r0", Commit: commit, Trees: map[string]string{"modules/avahi": "newtree"}}) - }); err != nil { - t.Fatal(err) - } - if _, err := js.Conn().Subscribe("mesh.seat.git.tool.file", func(m *nats.Msg) { - var a git.FileInput - _ = json.Unmarshal(m.Data, &a) - switch a.Path { - case "modules/avahi/module.json": - reply(m, git.FileAnswer{Path: a.Path, Ref: a.Ref, Content: string(manifest)}) - case "modules/avahi/go.sum": - reply(m, git.FileAnswer{Path: a.Path, Ref: a.Ref, Content: goSum}) - default: - reply(m, git.FileAnswer{Path: a.Path, Ref: a.Ref, Absent: true}) - } - }); err != nil { - t.Fatal(err) - } - }) - started, _ := json.Marshal(meshpipelinerunner.RunStarted{V: 1, Run: "9", By: "lead@n1", Members: []meshpipelinerunner.Member{{Module: "avahi", Commit: commit, Branch: "feature/1966-x"}}}) - if _, err := js.Publish(ctx, meshpipelinerunner.SubjectRunStarted("avahi", "feature-1966-x", commit), started); err != nil { - t.Fatal(err) - } - key, _ := meshcatalogclient.CandidatesKey("run-9") - var c meshcatalog.CandidatesByGroupValue - for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { - if e, err := candidates.Get(ctx, key); err == nil { - if c, err = index.DecodeCandidate(e.Value()); err == nil { - break - } - } - } - if len(c.Pulls) != 1 || c.Pulls[0].Number != 0 || c.Pulls[0].Head != commit || c.Pulls[0].Fps["avahi"].Manifest != meshcatalogclient.ManifestDigest(manifest) { - t.Fatalf("candidates.run-9 = %+v within 20 s of run-started, want avahi at the run's commit with its manifest", c) - } - restart() - ended, _ := json.Marshal(meshpipelinerunner.PipelineEnded{V: 1, Pipeline: "feature-1966-x.0", Branch: "feature/1966-x", Run: "9", - Movers: []meshpipelinerunner.MoverEnded{{Module: "avahi", Steps: []meshpipelinerunner.StepResult{}}}, Outcome: meshpipelinerunner.PipelineEndedOutcomePassed}) - if _, err := js.Publish(ctx, meshpipelinerunner.SubjectPipelineEnded("feature-1966-x", "0"), ended); err != nil { - t.Fatal(err) - } - gone := false - for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && !gone; time.Sleep(50 * time.Millisecond) { - _, cerr := candidates.Get(ctx, key) - _, rerr := runs.Get(ctx, "run-9") - gone = errors.Is(cerr, jetstream.ErrKeyNotFound) && errors.Is(rerr, jetstream.ErrKeyNotFound) - } - if !gone { - t.Error("the run's candidate or its runs key stays 20 s after its member's pipeline ended across a restart") - } -} - // TestIndexerCountsOnlyThePlacedBuildNodesToolchains (issue 1940, issue 774): the toolchains bucket holds the report of // n2, where the catalogue no longer places mesh-build-go, at another version than n1's, as ace's v26.12.0 stayed beside // the v26.12.1 of the nodes mesh-build-node runs on. The indexer gives the merged module a fingerprint from n1's diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go index cfcc1b8f..c42a2a37 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go @@ -17,6 +17,7 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" @@ -187,12 +188,14 @@ func sayMoved(t *testing.T, ctx context.Context, js jetstream.JetStream, branch type takeIns struct { taken chan meshcatalog.TakeInTaken pending chan meshcatalog.TakeInPending - acks chan string // the escaped branch whose source-moved the indexer acknowledged + acks chan string // the escaped branch whose source-moved the indexer acknowledged + runAcks chan struct{} // a run-started the indexer acknowledged } func hearTakeIns(t *testing.T, nc *nats.Conn) *takeIns { t.Helper() - h := &takeIns{taken: make(chan meshcatalog.TakeInTaken, 64), pending: make(chan meshcatalog.TakeInPending, 64), acks: make(chan string, 64)} + h := &takeIns{taken: make(chan meshcatalog.TakeInTaken, 64), pending: make(chan meshcatalog.TakeInPending, 64), acks: make(chan string, 64), + runAcks: make(chan struct{}, 64)} prefix := "$JS.ACK." + meshdelivery.StreamSourceMoved + "." acks, err := nc.Subscribe(prefix+"*.>", func(m *nats.Msg) { consumer, _, _ := strings.Cut(strings.TrimPrefix(m.Subject, prefix), ".") @@ -204,6 +207,15 @@ func hearTakeIns(t *testing.T, nc *nats.Conn) *takeIns { t.Fatal(err) } t.Cleanup(func() { _ = acks.Unsubscribe() }) + runAcks, err := nc.Subscribe("$JS.ACK."+meshpipelinerunner.StreamRunStarted+".n1_mesh-delivery-indexer.>", func(m *nats.Msg) { + if string(m.Data) == "+ACK" || len(m.Data) == 0 { + h.runAcks <- struct{}{} + } + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = runAcks.Unsubscribe() }) sub, err := nc.Subscribe("mesh.mod."+testIndexer+".event.take-in.*", func(m *nats.Msg) { switch m.Subject { case "mesh.mod." + testIndexer + ".event.take-in.taken": @@ -253,6 +265,16 @@ func (h *takeIns) acked(t *testing.T, branch string) { } } +// runAcked waits for the indexer's acknowledgement of a run-started. +func (h *takeIns) runAcked(t *testing.T) { + t.Helper() + select { + case <-h.runAcks: + case <-time.After(waitBound): + t.Fatalf("no run-started acknowledged within %s", waitBound) + } +} + // nextPending is the next take-in.pending said, which must be of that branch and source. func (h *takeIns) nextPending(t *testing.T, branch string, source int64) meshcatalog.TakeInPending { t.Helper() @@ -371,6 +393,9 @@ func absent(v []byte) bool { return v == nil } // given again says the same from the kept movers and keeps no second revision; main.10, the branch's previous revision, is // pinned pipeline.main.11 in the same write as validate's base (H-A-147); // - pipeline-started of main.12 pins it, a restart keeps main.11 and main.10 while main.11's pipeline runs; +// - a run on request's run-started (run 9, avahi at main 12's commit on a branch no registration watches) writes +// candidates.run-9, one pull of number 0 at that commit with avahi's manifest read through the seat git, and the run's key; across +// the restart, the runner's pipeline-ended of its one member drops both (design 47 G §64, §190; A-160, A-174); // - with the store full, main.13 is held, said once as take-in.pending, written whole in the state held and acknowledged // (H-A-114, H-A-118); nothing takes it in on a // timer: main.11's pipeline-ended frees both its pins, main.10 goes (main.11 stays, main.12's base and its passed release), the @@ -386,8 +411,8 @@ func TestIndexerWhole(t *testing.T) { ) var forge *forgeCommits var heard *takeIns - var pins, held, takeinsKV, regs jetstream.KeyValue - js, ctx, _, _, restart := startIndexerRun(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) { + var pins, held, takeinsKV, regs, runs jetstream.KeyValue + js, ctx, candidates, _, restart := startIndexerRun(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) { heard = hearTakeIns(t, js.Conn()) forge = serveCommits(t, js.Conn()) for commit, tree := range map[string]string{c10: "t10", c11: "t11", c12: "t12", c13: "t13"} { @@ -397,7 +422,7 @@ func TestIndexerWhole(t *testing.T) { for _, b := range []struct { state string kv *jetstream.KeyValue - }{{index.HeldState, &held}, {index.TakeInsState, &takeinsKV}} { + }{{index.HeldState, &held}, {index.TakeInsState, &takeinsKV}, {index.RunsState, &runs}} { if *b.kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, b.state), History: 1}); err != nil { t.Fatal(err) } @@ -465,6 +490,25 @@ func TestIndexerWhole(t *testing.T) { } sayPipeline(t, ctx, js, false, "main", 12) untilKey(t, ctx, pins, index.PinKey("main", 12), pinned("pipeline.main.12")) + // A run on request: its candidate and its key written on run-started, from avahi's files at the run's commit (A-160, A-174). + manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json") + if err != nil { + t.Fatal(err) + } + started, _ := json.Marshal(meshpipelinerunner.RunStarted{V: 1, Run: "9", By: "lead@n1", Members: []meshpipelinerunner.Member{{Module: "avahi", Commit: c12, Branch: "feature/1966-x"}}}) + if _, err := js.Publish(ctx, meshpipelinerunner.SubjectRunStarted("avahi", "feature-1966-x", c12), started); err != nil { + t.Fatal(err) + } + runKey, _ := meshcatalogclient.CandidatesKey("run-9") + untilKey(t, ctx, candidates, runKey, func(v []byte) bool { + c, err := index.DecodeCandidate(v) + return v != nil && err == nil && len(c.Pulls) == 1 && c.Pulls[0].Number == 0 && c.Pulls[0].Head == c12 && + c.Pulls[0].Fps["avahi"].Manifest == meshcatalogclient.ManifestDigest(manifest) + }) + untilKey(t, ctx, runs, "run-9", func(v []byte) bool { return v != nil }) + // The candidate is written before its run-started is acknowledged: the restart waits for that too, so the stop leaves it + // unacknowledged for no redelivery to write the candidate again after its run ended. + heard.runAcked(t) // The restart waits for the take-ins' ends, their source-moved acknowledged (design 47 H §743: main 10, 11, 11 given // again and 12), so the stop cancels none and leaves none unacknowledged ahead of main 13. for range 4 { @@ -476,6 +520,14 @@ func TestIndexerWhole(t *testing.T) { t.Fatalf("%s is not kept across a restart while main.11's pipeline runs: %v", name, err) } } + // The run's member's pipeline ends across the restart: its candidate and its key go (A-174). + ended, _ := json.Marshal(meshpipelinerunner.PipelineEnded{V: 1, Pipeline: "feature-1966-x.0", Branch: "feature/1966-x", Run: "9", + Movers: []meshpipelinerunner.MoverEnded{{Module: "avahi", Steps: []meshpipelinerunner.StepResult{}}}, Outcome: meshpipelinerunner.PipelineEndedOutcomePassed}) + if _, err := js.Publish(ctx, meshpipelinerunner.SubjectPipelineEnded("feature-1966-x", "0"), ended); err != nil { + t.Fatal(err) + } + untilKey(t, ctx, candidates, runKey, absent) + untilKey(t, ctx, runs, "run-9", absent) // The store full: held, said once, in the state held; taken in on the indexer's own room once a freed pin makes it. used, _, err := objects.Fill(ctx)