mesh-delivery-indexer: the run on request's bed lives in TestIndexerWhole, its restart after the run-started's acknowledgement

TestIndexerBuildsARunsCandidateAndDropsItWhenEveryMembersPipelineEnded failed on the build node
under load (pipelines main.29 and main.31). It restarted the indexer as soon as the candidate was
written, before the run-started was acknowledged (the write comes first), and it polled both
waits with sleeps: the same kind as the toolchains bed's restart fixed in 20a7bd0f. A stop
between the write and the acknowledgement leaves the run-started for redelivery after its
acknowledgement wait, which writes the candidate again once its run has ended.

The run is now one more step of TestIndexerWhole around its restart: run-started of run 9 writes
candidates.run-9 and the run's key (watched, not polled), the restart waits for the run-started's
acknowledgement beside the source-moved's, and the member's pipeline-ended after the restart drops
both. The separate test is removed: the module's integration tests go from 541 to 540; no test
added.

Builds: design 47 H §743
This commit is contained in:
2026-11-20 18:30:27 +01:00
parent e1f0c1a1cc
commit 08dde53e33
2 changed files with 57 additions and 91 deletions
@@ -31,7 +31,6 @@ import (
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
@@ -647,91 +646,6 @@ func (l buildLog) end(t *testing.T, repo, module, path string) {
} }
} }
// TestIndexerBuildsARunsCandidateAndDropsItWhenEveryMembersPipelineEnded (design 47 G §64, §67 test 64, §190; A-160, A-174):
// the indexer starts with the runner's streams run-started and pipeline-ended and its durables on them, its state runs and a
// registration of avahi; the runner says run-started of run 9 with avahi at a commit on a branch no registration watches;
// within 20 s candidates.run-9 holds one pull of number 0 at that commit, avahi's manifest read through the seat git; the
// indexer is restarted, the runner says avahi's pipeline of run 9 ended, and the candidate and the run's key go.
func TestIndexerBuildsARunsCandidateAndDropsItWhenEveryMembersPipelineEnded(t *testing.T) {
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
if err != nil {
t.Fatal(err)
}
const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"
const commit = "0123456789abcdef0123456789abcdef01234567"
var runs jetstream.KeyValue
js, ctx, candidates, _, restart := startIndexerRun(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) {
var err error
if runs, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.RunsState), History: 1}); err != nil {
t.Fatal(err)
}
regs, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.RegistrationsState), History: 1})
if err != nil {
t.Fatal(err)
}
reg, _ := json.Marshal(meshcatalog.Registration{V: 1, Module: "avahi", Origin: "novox/mesh-catalog", Path: "modules/avahi", Branches: []string{"main"}})
if _, err := regs.Create(ctx, index.RegistrationKey("avahi"), reg); err != nil {
t.Fatal(err)
}
seedBuildModules(t, ctx, js, nil)
// The seat git as the forge module serves it: avahi's directory and its manifest at the run's commit.
reply := func(m *nats.Msg, result any) {
raw, _ := json.Marshal(map[string]any{"result": result, "node": "n0"})
_ = m.Respond(raw)
}
if _, err := js.Conn().Subscribe("mesh.seat.git.tool.trees", func(m *nats.Msg) {
reply(m, git.TreesAnswer{Root: "r0", Commit: commit, Trees: map[string]string{"modules/avahi": "newtree"}})
}); err != nil {
t.Fatal(err)
}
if _, err := js.Conn().Subscribe("mesh.seat.git.tool.file", func(m *nats.Msg) {
var a git.FileInput
_ = json.Unmarshal(m.Data, &a)
switch a.Path {
case "modules/avahi/module.json":
reply(m, git.FileAnswer{Path: a.Path, Ref: a.Ref, Content: string(manifest)})
case "modules/avahi/go.sum":
reply(m, git.FileAnswer{Path: a.Path, Ref: a.Ref, Content: goSum})
default:
reply(m, git.FileAnswer{Path: a.Path, Ref: a.Ref, Absent: true})
}
}); err != nil {
t.Fatal(err)
}
})
started, _ := json.Marshal(meshpipelinerunner.RunStarted{V: 1, Run: "9", By: "lead@n1", Members: []meshpipelinerunner.Member{{Module: "avahi", Commit: commit, Branch: "feature/1966-x"}}})
if _, err := js.Publish(ctx, meshpipelinerunner.SubjectRunStarted("avahi", "feature-1966-x", commit), started); err != nil {
t.Fatal(err)
}
key, _ := meshcatalogclient.CandidatesKey("run-9")
var c meshcatalog.CandidatesByGroupValue
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := candidates.Get(ctx, key); err == nil {
if c, err = index.DecodeCandidate(e.Value()); err == nil {
break
}
}
}
if len(c.Pulls) != 1 || c.Pulls[0].Number != 0 || c.Pulls[0].Head != commit || c.Pulls[0].Fps["avahi"].Manifest != meshcatalogclient.ManifestDigest(manifest) {
t.Fatalf("candidates.run-9 = %+v within 20 s of run-started, want avahi at the run's commit with its manifest", c)
}
restart()
ended, _ := json.Marshal(meshpipelinerunner.PipelineEnded{V: 1, Pipeline: "feature-1966-x.0", Branch: "feature/1966-x", Run: "9",
Movers: []meshpipelinerunner.MoverEnded{{Module: "avahi", Steps: []meshpipelinerunner.StepResult{}}}, Outcome: meshpipelinerunner.PipelineEndedOutcomePassed})
if _, err := js.Publish(ctx, meshpipelinerunner.SubjectPipelineEnded("feature-1966-x", "0"), ended); err != nil {
t.Fatal(err)
}
gone := false
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && !gone; time.Sleep(50 * time.Millisecond) {
_, cerr := candidates.Get(ctx, key)
_, rerr := runs.Get(ctx, "run-9")
gone = errors.Is(cerr, jetstream.ErrKeyNotFound) && errors.Is(rerr, jetstream.ErrKeyNotFound)
}
if !gone {
t.Error("the run's candidate or its runs key stays 20 s after its member's pipeline ended across a restart")
}
}
// TestIndexerCountsOnlyThePlacedBuildNodesToolchains (issue 1940, issue 774): the toolchains bucket holds the report of // TestIndexerCountsOnlyThePlacedBuildNodesToolchains (issue 1940, issue 774): the toolchains bucket holds the report of
// n2, where the catalogue no longer places mesh-build-go, at another version than n1's, as ace's v26.12.0 stayed beside // n2, where the catalogue no longer places mesh-build-go, at another version than n1's, as ace's v26.12.0 stayed beside
// the v26.12.1 of the nodes mesh-build-node runs on. The indexer gives the merged module a fingerprint from n1's // the v26.12.1 of the nodes mesh-build-node runs on. The indexer gives the merged module a fingerprint from n1's
@@ -17,6 +17,7 @@ import (
"github.com/nats-io/nats.go" "github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream" "github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
@@ -188,11 +189,13 @@ type takeIns struct {
taken chan meshcatalog.TakeInTaken taken chan meshcatalog.TakeInTaken
pending chan meshcatalog.TakeInPending pending chan meshcatalog.TakeInPending
acks chan string // the escaped branch whose source-moved the indexer acknowledged acks chan string // the escaped branch whose source-moved the indexer acknowledged
runAcks chan struct{} // a run-started the indexer acknowledged
} }
func hearTakeIns(t *testing.T, nc *nats.Conn) *takeIns { func hearTakeIns(t *testing.T, nc *nats.Conn) *takeIns {
t.Helper() t.Helper()
h := &takeIns{taken: make(chan meshcatalog.TakeInTaken, 64), pending: make(chan meshcatalog.TakeInPending, 64), acks: make(chan string, 64)} h := &takeIns{taken: make(chan meshcatalog.TakeInTaken, 64), pending: make(chan meshcatalog.TakeInPending, 64), acks: make(chan string, 64),
runAcks: make(chan struct{}, 64)}
prefix := "$JS.ACK." + meshdelivery.StreamSourceMoved + "." prefix := "$JS.ACK." + meshdelivery.StreamSourceMoved + "."
acks, err := nc.Subscribe(prefix+"*.>", func(m *nats.Msg) { acks, err := nc.Subscribe(prefix+"*.>", func(m *nats.Msg) {
consumer, _, _ := strings.Cut(strings.TrimPrefix(m.Subject, prefix), ".") consumer, _, _ := strings.Cut(strings.TrimPrefix(m.Subject, prefix), ".")
@@ -204,6 +207,15 @@ func hearTakeIns(t *testing.T, nc *nats.Conn) *takeIns {
t.Fatal(err) t.Fatal(err)
} }
t.Cleanup(func() { _ = acks.Unsubscribe() }) t.Cleanup(func() { _ = acks.Unsubscribe() })
runAcks, err := nc.Subscribe("$JS.ACK."+meshpipelinerunner.StreamRunStarted+".n1_mesh-delivery-indexer.>", func(m *nats.Msg) {
if string(m.Data) == "+ACK" || len(m.Data) == 0 {
h.runAcks <- struct{}{}
}
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = runAcks.Unsubscribe() })
sub, err := nc.Subscribe("mesh.mod."+testIndexer+".event.take-in.*", func(m *nats.Msg) { sub, err := nc.Subscribe("mesh.mod."+testIndexer+".event.take-in.*", func(m *nats.Msg) {
switch m.Subject { switch m.Subject {
case "mesh.mod." + testIndexer + ".event.take-in.taken": case "mesh.mod." + testIndexer + ".event.take-in.taken":
@@ -253,6 +265,16 @@ func (h *takeIns) acked(t *testing.T, branch string) {
} }
} }
// runAcked waits for the indexer's acknowledgement of a run-started.
func (h *takeIns) runAcked(t *testing.T) {
t.Helper()
select {
case <-h.runAcks:
case <-time.After(waitBound):
t.Fatalf("no run-started acknowledged within %s", waitBound)
}
}
// nextPending is the next take-in.pending said, which must be of that branch and source. // nextPending is the next take-in.pending said, which must be of that branch and source.
func (h *takeIns) nextPending(t *testing.T, branch string, source int64) meshcatalog.TakeInPending { func (h *takeIns) nextPending(t *testing.T, branch string, source int64) meshcatalog.TakeInPending {
t.Helper() t.Helper()
@@ -371,6 +393,9 @@ func absent(v []byte) bool { return v == nil }
// given again says the same from the kept movers and keeps no second revision; main.10, the branch's previous revision, is // given again says the same from the kept movers and keeps no second revision; main.10, the branch's previous revision, is
// pinned pipeline.main.11 in the same write as validate's base (H-A-147); // pinned pipeline.main.11 in the same write as validate's base (H-A-147);
// - pipeline-started of main.12 pins it, a restart keeps main.11 and main.10 while main.11's pipeline runs; // - pipeline-started of main.12 pins it, a restart keeps main.11 and main.10 while main.11's pipeline runs;
// - a run on request's run-started (run 9, avahi at main 12's commit on a branch no registration watches) writes
// candidates.run-9, one pull of number 0 at that commit with avahi's manifest read through the seat git, and the run's key; across
// the restart, the runner's pipeline-ended of its one member drops both (design 47 G §64, §190; A-160, A-174);
// - with the store full, main.13 is held, said once as take-in.pending, written whole in the state held and acknowledged // - with the store full, main.13 is held, said once as take-in.pending, written whole in the state held and acknowledged
// (H-A-114, H-A-118); nothing takes it in on a // (H-A-114, H-A-118); nothing takes it in on a
// timer: main.11's pipeline-ended frees both its pins, main.10 goes (main.11 stays, main.12's base and its passed release), the // timer: main.11's pipeline-ended frees both its pins, main.10 goes (main.11 stays, main.12's base and its passed release), the
@@ -386,8 +411,8 @@ func TestIndexerWhole(t *testing.T) {
) )
var forge *forgeCommits var forge *forgeCommits
var heard *takeIns var heard *takeIns
var pins, held, takeinsKV, regs jetstream.KeyValue var pins, held, takeinsKV, regs, runs jetstream.KeyValue
js, ctx, _, _, restart := startIndexerRun(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) { js, ctx, candidates, _, restart := startIndexerRun(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) {
heard = hearTakeIns(t, js.Conn()) heard = hearTakeIns(t, js.Conn())
forge = serveCommits(t, js.Conn()) forge = serveCommits(t, js.Conn())
for commit, tree := range map[string]string{c10: "t10", c11: "t11", c12: "t12", c13: "t13"} { for commit, tree := range map[string]string{c10: "t10", c11: "t11", c12: "t12", c13: "t13"} {
@@ -397,7 +422,7 @@ func TestIndexerWhole(t *testing.T) {
for _, b := range []struct { for _, b := range []struct {
state string state string
kv *jetstream.KeyValue kv *jetstream.KeyValue
}{{index.HeldState, &held}, {index.TakeInsState, &takeinsKV}} { }{{index.HeldState, &held}, {index.TakeInsState, &takeinsKV}, {index.RunsState, &runs}} {
if *b.kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, b.state), History: 1}); err != nil { if *b.kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, b.state), History: 1}); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -465,6 +490,25 @@ func TestIndexerWhole(t *testing.T) {
} }
sayPipeline(t, ctx, js, false, "main", 12) sayPipeline(t, ctx, js, false, "main", 12)
untilKey(t, ctx, pins, index.PinKey("main", 12), pinned("pipeline.main.12")) untilKey(t, ctx, pins, index.PinKey("main", 12), pinned("pipeline.main.12"))
// A run on request: its candidate and its key written on run-started, from avahi's files at the run's commit (A-160, A-174).
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
if err != nil {
t.Fatal(err)
}
started, _ := json.Marshal(meshpipelinerunner.RunStarted{V: 1, Run: "9", By: "lead@n1", Members: []meshpipelinerunner.Member{{Module: "avahi", Commit: c12, Branch: "feature/1966-x"}}})
if _, err := js.Publish(ctx, meshpipelinerunner.SubjectRunStarted("avahi", "feature-1966-x", c12), started); err != nil {
t.Fatal(err)
}
runKey, _ := meshcatalogclient.CandidatesKey("run-9")
untilKey(t, ctx, candidates, runKey, func(v []byte) bool {
c, err := index.DecodeCandidate(v)
return v != nil && err == nil && len(c.Pulls) == 1 && c.Pulls[0].Number == 0 && c.Pulls[0].Head == c12 &&
c.Pulls[0].Fps["avahi"].Manifest == meshcatalogclient.ManifestDigest(manifest)
})
untilKey(t, ctx, runs, "run-9", func(v []byte) bool { return v != nil })
// The candidate is written before its run-started is acknowledged: the restart waits for that too, so the stop leaves it
// unacknowledged for no redelivery to write the candidate again after its run ended.
heard.runAcked(t)
// The restart waits for the take-ins' ends, their source-moved acknowledged (design 47 H §743: main 10, 11, 11 given // The restart waits for the take-ins' ends, their source-moved acknowledged (design 47 H §743: main 10, 11, 11 given
// again and 12), so the stop cancels none and leaves none unacknowledged ahead of main 13. // again and 12), so the stop cancels none and leaves none unacknowledged ahead of main 13.
for range 4 { for range 4 {
@@ -476,6 +520,14 @@ func TestIndexerWhole(t *testing.T) {
t.Fatalf("%s is not kept across a restart while main.11's pipeline runs: %v", name, err) t.Fatalf("%s is not kept across a restart while main.11's pipeline runs: %v", name, err)
} }
} }
// The run's member's pipeline ends across the restart: its candidate and its key go (A-174).
ended, _ := json.Marshal(meshpipelinerunner.PipelineEnded{V: 1, Pipeline: "feature-1966-x.0", Branch: "feature/1966-x", Run: "9",
Movers: []meshpipelinerunner.MoverEnded{{Module: "avahi", Steps: []meshpipelinerunner.StepResult{}}}, Outcome: meshpipelinerunner.PipelineEndedOutcomePassed})
if _, err := js.Publish(ctx, meshpipelinerunner.SubjectPipelineEnded("feature-1966-x", "0"), ended); err != nil {
t.Fatal(err)
}
untilKey(t, ctx, candidates, runKey, absent)
untilKey(t, ctx, runs, "run-9", absent)
// The store full: held, said once, in the state held; taken in on the indexer's own room once a freed pin makes it. // The store full: held, said once, in the state held; taken in on the indexer's own room once a freed pin makes it.
used, _, err := objects.Fill(ctx) used, _, err := objects.Fill(ctx)