mesh-delivery-indexer: the run on request's bed lives in TestIndexerWhole, its restart after the run-started's acknowledgement
TestIndexerBuildsARunsCandidateAndDropsItWhenEveryMembersPipelineEnded failed on the build node
under load (pipelines main.29 and main.31). It restarted the indexer as soon as the candidate was
written, before the run-started was acknowledged (the write comes first), and it polled both
waits with sleeps: the same kind as the toolchains bed's restart fixed in 20a7bd0f. A stop
between the write and the acknowledgement leaves the run-started for redelivery after its
acknowledgement wait, which writes the candidate again once its run has ended.
The run is now one more step of TestIndexerWhole around its restart: run-started of run 9 writes
candidates.run-9 and the run's key (watched, not polled), the restart waits for the run-started's
acknowledgement beside the source-moved's, and the member's pipeline-ended after the restart drops
both. The separate test is removed: the module's integration tests go from 541 to 540; no test
added.
Builds: design 47 H §743
This commit is contained in:
@@ -31,7 +31,6 @@ import (
|
|||||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
|
||||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||||
@@ -647,91 +646,6 @@ func (l buildLog) end(t *testing.T, repo, module, path string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestIndexerBuildsARunsCandidateAndDropsItWhenEveryMembersPipelineEnded (design 47 G §64, §67 test 64, §190; A-160, A-174):
|
|
||||||
// the indexer starts with the runner's streams run-started and pipeline-ended and its durables on them, its state runs and a
|
|
||||||
// registration of avahi; the runner says run-started of run 9 with avahi at a commit on a branch no registration watches;
|
|
||||||
// within 20 s candidates.run-9 holds one pull of number 0 at that commit, avahi's manifest read through the seat git; the
|
|
||||||
// indexer is restarted, the runner says avahi's pipeline of run 9 ended, and the candidate and the run's key go.
|
|
||||||
func TestIndexerBuildsARunsCandidateAndDropsItWhenEveryMembersPipelineEnded(t *testing.T) {
|
|
||||||
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"
|
|
||||||
const commit = "0123456789abcdef0123456789abcdef01234567"
|
|
||||||
var runs jetstream.KeyValue
|
|
||||||
js, ctx, candidates, _, restart := startIndexerRun(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) {
|
|
||||||
var err error
|
|
||||||
if runs, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.RunsState), History: 1}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
regs, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.RegistrationsState), History: 1})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
reg, _ := json.Marshal(meshcatalog.Registration{V: 1, Module: "avahi", Origin: "novox/mesh-catalog", Path: "modules/avahi", Branches: []string{"main"}})
|
|
||||||
if _, err := regs.Create(ctx, index.RegistrationKey("avahi"), reg); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
seedBuildModules(t, ctx, js, nil)
|
|
||||||
// The seat git as the forge module serves it: avahi's directory and its manifest at the run's commit.
|
|
||||||
reply := func(m *nats.Msg, result any) {
|
|
||||||
raw, _ := json.Marshal(map[string]any{"result": result, "node": "n0"})
|
|
||||||
_ = m.Respond(raw)
|
|
||||||
}
|
|
||||||
if _, err := js.Conn().Subscribe("mesh.seat.git.tool.trees", func(m *nats.Msg) {
|
|
||||||
reply(m, git.TreesAnswer{Root: "r0", Commit: commit, Trees: map[string]string{"modules/avahi": "newtree"}})
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := js.Conn().Subscribe("mesh.seat.git.tool.file", func(m *nats.Msg) {
|
|
||||||
var a git.FileInput
|
|
||||||
_ = json.Unmarshal(m.Data, &a)
|
|
||||||
switch a.Path {
|
|
||||||
case "modules/avahi/module.json":
|
|
||||||
reply(m, git.FileAnswer{Path: a.Path, Ref: a.Ref, Content: string(manifest)})
|
|
||||||
case "modules/avahi/go.sum":
|
|
||||||
reply(m, git.FileAnswer{Path: a.Path, Ref: a.Ref, Content: goSum})
|
|
||||||
default:
|
|
||||||
reply(m, git.FileAnswer{Path: a.Path, Ref: a.Ref, Absent: true})
|
|
||||||
}
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
started, _ := json.Marshal(meshpipelinerunner.RunStarted{V: 1, Run: "9", By: "lead@n1", Members: []meshpipelinerunner.Member{{Module: "avahi", Commit: commit, Branch: "feature/1966-x"}}})
|
|
||||||
if _, err := js.Publish(ctx, meshpipelinerunner.SubjectRunStarted("avahi", "feature-1966-x", commit), started); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
key, _ := meshcatalogclient.CandidatesKey("run-9")
|
|
||||||
var c meshcatalog.CandidatesByGroupValue
|
|
||||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
|
||||||
if e, err := candidates.Get(ctx, key); err == nil {
|
|
||||||
if c, err = index.DecodeCandidate(e.Value()); err == nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(c.Pulls) != 1 || c.Pulls[0].Number != 0 || c.Pulls[0].Head != commit || c.Pulls[0].Fps["avahi"].Manifest != meshcatalogclient.ManifestDigest(manifest) {
|
|
||||||
t.Fatalf("candidates.run-9 = %+v within 20 s of run-started, want avahi at the run's commit with its manifest", c)
|
|
||||||
}
|
|
||||||
restart()
|
|
||||||
ended, _ := json.Marshal(meshpipelinerunner.PipelineEnded{V: 1, Pipeline: "feature-1966-x.0", Branch: "feature/1966-x", Run: "9",
|
|
||||||
Movers: []meshpipelinerunner.MoverEnded{{Module: "avahi", Steps: []meshpipelinerunner.StepResult{}}}, Outcome: meshpipelinerunner.PipelineEndedOutcomePassed})
|
|
||||||
if _, err := js.Publish(ctx, meshpipelinerunner.SubjectPipelineEnded("feature-1966-x", "0"), ended); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
gone := false
|
|
||||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && !gone; time.Sleep(50 * time.Millisecond) {
|
|
||||||
_, cerr := candidates.Get(ctx, key)
|
|
||||||
_, rerr := runs.Get(ctx, "run-9")
|
|
||||||
gone = errors.Is(cerr, jetstream.ErrKeyNotFound) && errors.Is(rerr, jetstream.ErrKeyNotFound)
|
|
||||||
}
|
|
||||||
if !gone {
|
|
||||||
t.Error("the run's candidate or its runs key stays 20 s after its member's pipeline ended across a restart")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestIndexerCountsOnlyThePlacedBuildNodesToolchains (issue 1940, issue 774): the toolchains bucket holds the report of
|
// TestIndexerCountsOnlyThePlacedBuildNodesToolchains (issue 1940, issue 774): the toolchains bucket holds the report of
|
||||||
// n2, where the catalogue no longer places mesh-build-go, at another version than n1's, as ace's v26.12.0 stayed beside
|
// n2, where the catalogue no longer places mesh-build-go, at another version than n1's, as ace's v26.12.0 stayed beside
|
||||||
// the v26.12.1 of the nodes mesh-build-node runs on. The indexer gives the merged module a fingerprint from n1's
|
// the v26.12.1 of the nodes mesh-build-node runs on. The indexer gives the merged module a fingerprint from n1's
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import (
|
|||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||||
@@ -188,11 +189,13 @@ type takeIns struct {
|
|||||||
taken chan meshcatalog.TakeInTaken
|
taken chan meshcatalog.TakeInTaken
|
||||||
pending chan meshcatalog.TakeInPending
|
pending chan meshcatalog.TakeInPending
|
||||||
acks chan string // the escaped branch whose source-moved the indexer acknowledged
|
acks chan string // the escaped branch whose source-moved the indexer acknowledged
|
||||||
|
runAcks chan struct{} // a run-started the indexer acknowledged
|
||||||
}
|
}
|
||||||
|
|
||||||
func hearTakeIns(t *testing.T, nc *nats.Conn) *takeIns {
|
func hearTakeIns(t *testing.T, nc *nats.Conn) *takeIns {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
h := &takeIns{taken: make(chan meshcatalog.TakeInTaken, 64), pending: make(chan meshcatalog.TakeInPending, 64), acks: make(chan string, 64)}
|
h := &takeIns{taken: make(chan meshcatalog.TakeInTaken, 64), pending: make(chan meshcatalog.TakeInPending, 64), acks: make(chan string, 64),
|
||||||
|
runAcks: make(chan struct{}, 64)}
|
||||||
prefix := "$JS.ACK." + meshdelivery.StreamSourceMoved + "."
|
prefix := "$JS.ACK." + meshdelivery.StreamSourceMoved + "."
|
||||||
acks, err := nc.Subscribe(prefix+"*.>", func(m *nats.Msg) {
|
acks, err := nc.Subscribe(prefix+"*.>", func(m *nats.Msg) {
|
||||||
consumer, _, _ := strings.Cut(strings.TrimPrefix(m.Subject, prefix), ".")
|
consumer, _, _ := strings.Cut(strings.TrimPrefix(m.Subject, prefix), ".")
|
||||||
@@ -204,6 +207,15 @@ func hearTakeIns(t *testing.T, nc *nats.Conn) *takeIns {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
t.Cleanup(func() { _ = acks.Unsubscribe() })
|
t.Cleanup(func() { _ = acks.Unsubscribe() })
|
||||||
|
runAcks, err := nc.Subscribe("$JS.ACK."+meshpipelinerunner.StreamRunStarted+".n1_mesh-delivery-indexer.>", func(m *nats.Msg) {
|
||||||
|
if string(m.Data) == "+ACK" || len(m.Data) == 0 {
|
||||||
|
h.runAcks <- struct{}{}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = runAcks.Unsubscribe() })
|
||||||
sub, err := nc.Subscribe("mesh.mod."+testIndexer+".event.take-in.*", func(m *nats.Msg) {
|
sub, err := nc.Subscribe("mesh.mod."+testIndexer+".event.take-in.*", func(m *nats.Msg) {
|
||||||
switch m.Subject {
|
switch m.Subject {
|
||||||
case "mesh.mod." + testIndexer + ".event.take-in.taken":
|
case "mesh.mod." + testIndexer + ".event.take-in.taken":
|
||||||
@@ -253,6 +265,16 @@ func (h *takeIns) acked(t *testing.T, branch string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// runAcked waits for the indexer's acknowledgement of a run-started.
|
||||||
|
func (h *takeIns) runAcked(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
select {
|
||||||
|
case <-h.runAcks:
|
||||||
|
case <-time.After(waitBound):
|
||||||
|
t.Fatalf("no run-started acknowledged within %s", waitBound)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// nextPending is the next take-in.pending said, which must be of that branch and source.
|
// nextPending is the next take-in.pending said, which must be of that branch and source.
|
||||||
func (h *takeIns) nextPending(t *testing.T, branch string, source int64) meshcatalog.TakeInPending {
|
func (h *takeIns) nextPending(t *testing.T, branch string, source int64) meshcatalog.TakeInPending {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
@@ -371,6 +393,9 @@ func absent(v []byte) bool { return v == nil }
|
|||||||
// given again says the same from the kept movers and keeps no second revision; main.10, the branch's previous revision, is
|
// given again says the same from the kept movers and keeps no second revision; main.10, the branch's previous revision, is
|
||||||
// pinned pipeline.main.11 in the same write as validate's base (H-A-147);
|
// pinned pipeline.main.11 in the same write as validate's base (H-A-147);
|
||||||
// - pipeline-started of main.12 pins it, a restart keeps main.11 and main.10 while main.11's pipeline runs;
|
// - pipeline-started of main.12 pins it, a restart keeps main.11 and main.10 while main.11's pipeline runs;
|
||||||
|
// - a run on request's run-started (run 9, avahi at main 12's commit on a branch no registration watches) writes
|
||||||
|
// candidates.run-9, one pull of number 0 at that commit with avahi's manifest read through the seat git, and the run's key; across
|
||||||
|
// the restart, the runner's pipeline-ended of its one member drops both (design 47 G §64, §190; A-160, A-174);
|
||||||
// - with the store full, main.13 is held, said once as take-in.pending, written whole in the state held and acknowledged
|
// - with the store full, main.13 is held, said once as take-in.pending, written whole in the state held and acknowledged
|
||||||
// (H-A-114, H-A-118); nothing takes it in on a
|
// (H-A-114, H-A-118); nothing takes it in on a
|
||||||
// timer: main.11's pipeline-ended frees both its pins, main.10 goes (main.11 stays, main.12's base and its passed release), the
|
// timer: main.11's pipeline-ended frees both its pins, main.10 goes (main.11 stays, main.12's base and its passed release), the
|
||||||
@@ -386,8 +411,8 @@ func TestIndexerWhole(t *testing.T) {
|
|||||||
)
|
)
|
||||||
var forge *forgeCommits
|
var forge *forgeCommits
|
||||||
var heard *takeIns
|
var heard *takeIns
|
||||||
var pins, held, takeinsKV, regs jetstream.KeyValue
|
var pins, held, takeinsKV, regs, runs jetstream.KeyValue
|
||||||
js, ctx, _, _, restart := startIndexerRun(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) {
|
js, ctx, candidates, _, restart := startIndexerRun(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) {
|
||||||
heard = hearTakeIns(t, js.Conn())
|
heard = hearTakeIns(t, js.Conn())
|
||||||
forge = serveCommits(t, js.Conn())
|
forge = serveCommits(t, js.Conn())
|
||||||
for commit, tree := range map[string]string{c10: "t10", c11: "t11", c12: "t12", c13: "t13"} {
|
for commit, tree := range map[string]string{c10: "t10", c11: "t11", c12: "t12", c13: "t13"} {
|
||||||
@@ -397,7 +422,7 @@ func TestIndexerWhole(t *testing.T) {
|
|||||||
for _, b := range []struct {
|
for _, b := range []struct {
|
||||||
state string
|
state string
|
||||||
kv *jetstream.KeyValue
|
kv *jetstream.KeyValue
|
||||||
}{{index.HeldState, &held}, {index.TakeInsState, &takeinsKV}} {
|
}{{index.HeldState, &held}, {index.TakeInsState, &takeinsKV}, {index.RunsState, &runs}} {
|
||||||
if *b.kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, b.state), History: 1}); err != nil {
|
if *b.kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, b.state), History: 1}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -465,6 +490,25 @@ func TestIndexerWhole(t *testing.T) {
|
|||||||
}
|
}
|
||||||
sayPipeline(t, ctx, js, false, "main", 12)
|
sayPipeline(t, ctx, js, false, "main", 12)
|
||||||
untilKey(t, ctx, pins, index.PinKey("main", 12), pinned("pipeline.main.12"))
|
untilKey(t, ctx, pins, index.PinKey("main", 12), pinned("pipeline.main.12"))
|
||||||
|
// A run on request: its candidate and its key written on run-started, from avahi's files at the run's commit (A-160, A-174).
|
||||||
|
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
started, _ := json.Marshal(meshpipelinerunner.RunStarted{V: 1, Run: "9", By: "lead@n1", Members: []meshpipelinerunner.Member{{Module: "avahi", Commit: c12, Branch: "feature/1966-x"}}})
|
||||||
|
if _, err := js.Publish(ctx, meshpipelinerunner.SubjectRunStarted("avahi", "feature-1966-x", c12), started); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
runKey, _ := meshcatalogclient.CandidatesKey("run-9")
|
||||||
|
untilKey(t, ctx, candidates, runKey, func(v []byte) bool {
|
||||||
|
c, err := index.DecodeCandidate(v)
|
||||||
|
return v != nil && err == nil && len(c.Pulls) == 1 && c.Pulls[0].Number == 0 && c.Pulls[0].Head == c12 &&
|
||||||
|
c.Pulls[0].Fps["avahi"].Manifest == meshcatalogclient.ManifestDigest(manifest)
|
||||||
|
})
|
||||||
|
untilKey(t, ctx, runs, "run-9", func(v []byte) bool { return v != nil })
|
||||||
|
// The candidate is written before its run-started is acknowledged: the restart waits for that too, so the stop leaves it
|
||||||
|
// unacknowledged for no redelivery to write the candidate again after its run ended.
|
||||||
|
heard.runAcked(t)
|
||||||
// The restart waits for the take-ins' ends, their source-moved acknowledged (design 47 H §743: main 10, 11, 11 given
|
// The restart waits for the take-ins' ends, their source-moved acknowledged (design 47 H §743: main 10, 11, 11 given
|
||||||
// again and 12), so the stop cancels none and leaves none unacknowledged ahead of main 13.
|
// again and 12), so the stop cancels none and leaves none unacknowledged ahead of main 13.
|
||||||
for range 4 {
|
for range 4 {
|
||||||
@@ -476,6 +520,14 @@ func TestIndexerWhole(t *testing.T) {
|
|||||||
t.Fatalf("%s is not kept across a restart while main.11's pipeline runs: %v", name, err)
|
t.Fatalf("%s is not kept across a restart while main.11's pipeline runs: %v", name, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// The run's member's pipeline ends across the restart: its candidate and its key go (A-174).
|
||||||
|
ended, _ := json.Marshal(meshpipelinerunner.PipelineEnded{V: 1, Pipeline: "feature-1966-x.0", Branch: "feature/1966-x", Run: "9",
|
||||||
|
Movers: []meshpipelinerunner.MoverEnded{{Module: "avahi", Steps: []meshpipelinerunner.StepResult{}}}, Outcome: meshpipelinerunner.PipelineEndedOutcomePassed})
|
||||||
|
if _, err := js.Publish(ctx, meshpipelinerunner.SubjectPipelineEnded("feature-1966-x", "0"), ended); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
untilKey(t, ctx, candidates, runKey, absent)
|
||||||
|
untilKey(t, ctx, runs, "run-9", absent)
|
||||||
|
|
||||||
// The store full: held, said once, in the state held; taken in on the indexer's own room once a freed pin makes it.
|
// The store full: held, said once, in the state held; taken in on the indexer's own room once a freed pin makes it.
|
||||||
used, _, err := objects.Fill(ctx)
|
used, _, err := objects.Fill(ctx)
|
||||||
|
|||||||
Reference in New Issue
Block a user