Design 47 H1: the indexer's runs-candidate test is folded into TestIndexerWhole and waits for the run-started's acknowledgement (H §743) #282

Merged
mesh-tools merged 1 commits from fix/47-H1-indexer-runs-test into main 2026-11-20 17:35:17 +00:00
2 changed files with 57 additions and 91 deletions
@@ -31,7 +31,6 @@ import (
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
@@ -647,91 +646,6 @@ func (l buildLog) end(t *testing.T, repo, module, path string) {
}
}
// TestIndexerBuildsARunsCandidateAndDropsItWhenEveryMembersPipelineEnded (design 47 G §64, §67 test 64, §190; A-160, A-174):
// the indexer starts with the runner's streams run-started and pipeline-ended and its durables on them, its state runs and a
// registration of avahi; the runner says run-started of run 9 with avahi at a commit on a branch no registration watches;
// within 20 s candidates.run-9 holds one pull of number 0 at that commit, avahi's manifest read through the seat git; the
// indexer is restarted, the runner says avahi's pipeline of run 9 ended, and the candidate and the run's key go.
func TestIndexerBuildsARunsCandidateAndDropsItWhenEveryMembersPipelineEnded(t *testing.T) {
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
if err != nil {
t.Fatal(err)
}
const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"
const commit = "0123456789abcdef0123456789abcdef01234567"
var runs jetstream.KeyValue
js, ctx, candidates, _, restart := startIndexerRun(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) {
var err error
if runs, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.RunsState), History: 1}); err != nil {
t.Fatal(err)
}
regs, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.RegistrationsState), History: 1})
if err != nil {
t.Fatal(err)
}
reg, _ := json.Marshal(meshcatalog.Registration{V: 1, Module: "avahi", Origin: "novox/mesh-catalog", Path: "modules/avahi", Branches: []string{"main"}})
if _, err := regs.Create(ctx, index.RegistrationKey("avahi"), reg); err != nil {
t.Fatal(err)
}
seedBuildModules(t, ctx, js, nil)
// The seat git as the forge module serves it: avahi's directory and its manifest at the run's commit.
reply := func(m *nats.Msg, result any) {
raw, _ := json.Marshal(map[string]any{"result": result, "node": "n0"})
_ = m.Respond(raw)
}
if _, err := js.Conn().Subscribe("mesh.seat.git.tool.trees", func(m *nats.Msg) {
reply(m, git.TreesAnswer{Root: "r0", Commit: commit, Trees: map[string]string{"modules/avahi": "newtree"}})
}); err != nil {
t.Fatal(err)
}
if _, err := js.Conn().Subscribe("mesh.seat.git.tool.file", func(m *nats.Msg) {
var a git.FileInput
_ = json.Unmarshal(m.Data, &a)
switch a.Path {
case "modules/avahi/module.json":
reply(m, git.FileAnswer{Path: a.Path, Ref: a.Ref, Content: string(manifest)})
case "modules/avahi/go.sum":
reply(m, git.FileAnswer{Path: a.Path, Ref: a.Ref, Content: goSum})
default:
reply(m, git.FileAnswer{Path: a.Path, Ref: a.Ref, Absent: true})
}
}); err != nil {
t.Fatal(err)
}
})
started, _ := json.Marshal(meshpipelinerunner.RunStarted{V: 1, Run: "9", By: "lead@n1", Members: []meshpipelinerunner.Member{{Module: "avahi", Commit: commit, Branch: "feature/1966-x"}}})
if _, err := js.Publish(ctx, meshpipelinerunner.SubjectRunStarted("avahi", "feature-1966-x", commit), started); err != nil {
t.Fatal(err)
}
key, _ := meshcatalogclient.CandidatesKey("run-9")
var c meshcatalog.CandidatesByGroupValue
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := candidates.Get(ctx, key); err == nil {
if c, err = index.DecodeCandidate(e.Value()); err == nil {
break
}
}
}
if len(c.Pulls) != 1 || c.Pulls[0].Number != 0 || c.Pulls[0].Head != commit || c.Pulls[0].Fps["avahi"].Manifest != meshcatalogclient.ManifestDigest(manifest) {
t.Fatalf("candidates.run-9 = %+v within 20 s of run-started, want avahi at the run's commit with its manifest", c)
}
restart()
ended, _ := json.Marshal(meshpipelinerunner.PipelineEnded{V: 1, Pipeline: "feature-1966-x.0", Branch: "feature/1966-x", Run: "9",
Movers: []meshpipelinerunner.MoverEnded{{Module: "avahi", Steps: []meshpipelinerunner.StepResult{}}}, Outcome: meshpipelinerunner.PipelineEndedOutcomePassed})
if _, err := js.Publish(ctx, meshpipelinerunner.SubjectPipelineEnded("feature-1966-x", "0"), ended); err != nil {
t.Fatal(err)
}
gone := false
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && !gone; time.Sleep(50 * time.Millisecond) {
_, cerr := candidates.Get(ctx, key)
_, rerr := runs.Get(ctx, "run-9")
gone = errors.Is(cerr, jetstream.ErrKeyNotFound) && errors.Is(rerr, jetstream.ErrKeyNotFound)
}
if !gone {
t.Error("the run's candidate or its runs key stays 20 s after its member's pipeline ended across a restart")
}
}
// TestIndexerCountsOnlyThePlacedBuildNodesToolchains (issue 1940, issue 774): the toolchains bucket holds the report of
// n2, where the catalogue no longer places mesh-build-go, at another version than n1's, as ace's v26.12.0 stayed beside
// the v26.12.1 of the nodes mesh-build-node runs on. The indexer gives the merged module a fingerprint from n1's
@@ -17,6 +17,7 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
@@ -187,12 +188,14 @@ func sayMoved(t *testing.T, ctx context.Context, js jetstream.JetStream, branch
type takeIns struct {
taken chan meshcatalog.TakeInTaken
pending chan meshcatalog.TakeInPending
acks chan string // the escaped branch whose source-moved the indexer acknowledged
acks chan string // the escaped branch whose source-moved the indexer acknowledged
runAcks chan struct{} // a run-started the indexer acknowledged
}
func hearTakeIns(t *testing.T, nc *nats.Conn) *takeIns {
t.Helper()
h := &takeIns{taken: make(chan meshcatalog.TakeInTaken, 64), pending: make(chan meshcatalog.TakeInPending, 64), acks: make(chan string, 64)}
h := &takeIns{taken: make(chan meshcatalog.TakeInTaken, 64), pending: make(chan meshcatalog.TakeInPending, 64), acks: make(chan string, 64),
runAcks: make(chan struct{}, 64)}
prefix := "$JS.ACK." + meshdelivery.StreamSourceMoved + "."
acks, err := nc.Subscribe(prefix+"*.>", func(m *nats.Msg) {
consumer, _, _ := strings.Cut(strings.TrimPrefix(m.Subject, prefix), ".")
@@ -204,6 +207,15 @@ func hearTakeIns(t *testing.T, nc *nats.Conn) *takeIns {
t.Fatal(err)
}
t.Cleanup(func() { _ = acks.Unsubscribe() })
runAcks, err := nc.Subscribe("$JS.ACK."+meshpipelinerunner.StreamRunStarted+".n1_mesh-delivery-indexer.>", func(m *nats.Msg) {
if string(m.Data) == "+ACK" || len(m.Data) == 0 {
h.runAcks <- struct{}{}
}
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = runAcks.Unsubscribe() })
sub, err := nc.Subscribe("mesh.mod."+testIndexer+".event.take-in.*", func(m *nats.Msg) {
switch m.Subject {
case "mesh.mod." + testIndexer + ".event.take-in.taken":
@@ -253,6 +265,16 @@ func (h *takeIns) acked(t *testing.T, branch string) {
}
}
// runAcked waits for the indexer's acknowledgement of a run-started.
func (h *takeIns) runAcked(t *testing.T) {
t.Helper()
select {
case <-h.runAcks:
case <-time.After(waitBound):
t.Fatalf("no run-started acknowledged within %s", waitBound)
}
}
// nextPending is the next take-in.pending said, which must be of that branch and source.
func (h *takeIns) nextPending(t *testing.T, branch string, source int64) meshcatalog.TakeInPending {
t.Helper()
@@ -371,6 +393,9 @@ func absent(v []byte) bool { return v == nil }
// given again says the same from the kept movers and keeps no second revision; main.10, the branch's previous revision, is
// pinned pipeline.main.11 in the same write as validate's base (H-A-147);
// - pipeline-started of main.12 pins it, a restart keeps main.11 and main.10 while main.11's pipeline runs;
// - a run on request's run-started (run 9, avahi at main 12's commit on a branch no registration watches) writes
// candidates.run-9, one pull of number 0 at that commit with avahi's manifest read through the seat git, and the run's key; across
// the restart, the runner's pipeline-ended of its one member drops both (design 47 G §64, §190; A-160, A-174);
// - with the store full, main.13 is held, said once as take-in.pending, written whole in the state held and acknowledged
// (H-A-114, H-A-118); nothing takes it in on a
// timer: main.11's pipeline-ended frees both its pins, main.10 goes (main.11 stays, main.12's base and its passed release), the
@@ -386,8 +411,8 @@ func TestIndexerWhole(t *testing.T) {
)
var forge *forgeCommits
var heard *takeIns
var pins, held, takeinsKV, regs jetstream.KeyValue
js, ctx, _, _, restart := startIndexerRun(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) {
var pins, held, takeinsKV, regs, runs jetstream.KeyValue
js, ctx, candidates, _, restart := startIndexerRun(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) {
heard = hearTakeIns(t, js.Conn())
forge = serveCommits(t, js.Conn())
for commit, tree := range map[string]string{c10: "t10", c11: "t11", c12: "t12", c13: "t13"} {
@@ -397,7 +422,7 @@ func TestIndexerWhole(t *testing.T) {
for _, b := range []struct {
state string
kv *jetstream.KeyValue
}{{index.HeldState, &held}, {index.TakeInsState, &takeinsKV}} {
}{{index.HeldState, &held}, {index.TakeInsState, &takeinsKV}, {index.RunsState, &runs}} {
if *b.kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, b.state), History: 1}); err != nil {
t.Fatal(err)
}
@@ -465,6 +490,25 @@ func TestIndexerWhole(t *testing.T) {
}
sayPipeline(t, ctx, js, false, "main", 12)
untilKey(t, ctx, pins, index.PinKey("main", 12), pinned("pipeline.main.12"))
// A run on request: its candidate and its key written on run-started, from avahi's files at the run's commit (A-160, A-174).
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
if err != nil {
t.Fatal(err)
}
started, _ := json.Marshal(meshpipelinerunner.RunStarted{V: 1, Run: "9", By: "lead@n1", Members: []meshpipelinerunner.Member{{Module: "avahi", Commit: c12, Branch: "feature/1966-x"}}})
if _, err := js.Publish(ctx, meshpipelinerunner.SubjectRunStarted("avahi", "feature-1966-x", c12), started); err != nil {
t.Fatal(err)
}
runKey, _ := meshcatalogclient.CandidatesKey("run-9")
untilKey(t, ctx, candidates, runKey, func(v []byte) bool {
c, err := index.DecodeCandidate(v)
return v != nil && err == nil && len(c.Pulls) == 1 && c.Pulls[0].Number == 0 && c.Pulls[0].Head == c12 &&
c.Pulls[0].Fps["avahi"].Manifest == meshcatalogclient.ManifestDigest(manifest)
})
untilKey(t, ctx, runs, "run-9", func(v []byte) bool { return v != nil })
// The candidate is written before its run-started is acknowledged: the restart waits for that too, so the stop leaves it
// unacknowledged for no redelivery to write the candidate again after its run ended.
heard.runAcked(t)
// The restart waits for the take-ins' ends, their source-moved acknowledged (design 47 H §743: main 10, 11, 11 given
// again and 12), so the stop cancels none and leaves none unacknowledged ahead of main 13.
for range 4 {
@@ -476,6 +520,14 @@ func TestIndexerWhole(t *testing.T) {
t.Fatalf("%s is not kept across a restart while main.11's pipeline runs: %v", name, err)
}
}
// The run's member's pipeline ends across the restart: its candidate and its key go (A-174).
ended, _ := json.Marshal(meshpipelinerunner.PipelineEnded{V: 1, Pipeline: "feature-1966-x.0", Branch: "feature/1966-x", Run: "9",
Movers: []meshpipelinerunner.MoverEnded{{Module: "avahi", Steps: []meshpipelinerunner.StepResult{}}}, Outcome: meshpipelinerunner.PipelineEndedOutcomePassed})
if _, err := js.Publish(ctx, meshpipelinerunner.SubjectPipelineEnded("feature-1966-x", "0"), ended); err != nil {
t.Fatal(err)
}
untilKey(t, ctx, candidates, runKey, absent)
untilKey(t, ctx, runs, "run-9", absent)
// The store full: held, said once, in the state held; taken in on the indexer's own room once a freed pin makes it.
used, _, err := objects.Fill(ctx)