From 18786072084d9797e9262d46bf4d0712e2d96bb5 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 12:07:56 +0100 Subject: [PATCH] assigner: test 108, the composition runs started whole against the bus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One test of the whole assigner: one run per candidate and machine read by every step, a module's fault failing only its own steps, the bound of owed progress, the untaken ask, a step-ended after the bound ignored, a rerun moving the reader that did not pass, a superseded step leaving its run, a restart taking a run up by its ask, the drop verb and the cap. Builds: design 47 G §733 (test 108); A-470f --- .../compositionruns_integration_test.go | 378 ++++++++++++++++++ .../integration_test.go | 19 +- 2 files changed, 392 insertions(+), 5 deletions(-) create mode 100644 modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go new file mode 100644 index 00000000..4b293453 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go @@ -0,0 +1,378 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "fmt" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" + + "mesh-delivery-assigner/internal/buckets" +) + +// Test 108 (design 47 G §733; §718 to §723, §727, §728; A-468 to A-470f): the assigner started whole against the bus composes +// each machine once per candidate. Two or three composition steps of one candidate on two machines make one key, one request and +// one run ask per machine, and each step reads both verdicts; a machine blaming a and b fails their steps and passes c's with a +// warning; a verdict with no `blamed` is judged as before, "the composer names no culprits", and a machine blaming none fails +// every step; a step of another candidate starts its own runs. A taken ask that keeps saying progress is not ended by the bound, +// one silent past it ends could not run with its ask dropped, a step-ended after that is ignored and journaled, and an ask never +// taken ends at run-untaken-bound. A rerun of a starts one run on the machine it did not pass and moves b; a superseded step leaves +// a run running for its other reader; an assigner restarted during a run takes it up by its ask; compositions-drop ends the +// readers and deletes the key; at the cap a step ends could not run, naming the fill. +func TestOneCompositionRunPerCandidateAndMachine(t *testing.T) { + b := startBed(t, "") + b.settings = map[string]string{"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": "2s", "MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": "15s", // past a restart + "MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "1"} + nc, err := nats.Connect(b.srv.ClientURL()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + const commit = "1111111111111111111111111111111111111111" + if _, err := b.js.CreateOrUpdateConsumer(b.ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner", + FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>", meshcatalog.SubjectCatalogRevisionAll, + nodebuildgo.SubjectStepEndedAll, nodebuildgo.SubjectAskTakenAll, nodebuildgo.SubjectStepProgressAll, pipelineEndedSubject}, + AckPolicy: jetstream.AckExplicitPolicy, AckWait: 2 * time.Second}); err != nil { // an event handed to the stopped assigner's pull comes again soon + t.Fatal(err) + } + for _, node := range []string{"n1", "n2"} { + raw, _ := json.Marshal(nodeengine.Running{V: 1, Node: node, Module: "engine", Fingerprint: "src2:e1", At: time.Now().UTC()}) + if _, err := b.js.Publish(b.ctx, nodeengine.SubjectRunning(node, "engine"), raw); err != nil { + t.Fatal(err) + } + } + awaitKept(t, b, nodeengine.StreamRunning, 2) + engine := meshcatalog.CatalogEntry{Module: "engine", Fp: "src2:e1", Repository: "novox/mesh-host", Commit: commit, Path: ".", + Serves: []string{nodeengine.Seat + ".composition"}, Manifest: map[string]any{"module": "engine", "version": "0.1"}} + b.cat.mu.Lock() + b.cat.entries[trunkBranch] = map[string]meshcatalog.CatalogEntry{"engine": engine} + for _, m := range []string{"a", "b", "c"} { + b.cat.entries[trunkBranch][m] = meshcatalog.CatalogEntry{Module: m, Fp: "src2:" + m, Repository: "novox/" + m, Commit: commit, Path: ".", Manifest: map[string]any{"module": m}} + } + b.cat.rev[trunkBranch] = 3 + b.cat.mu.Unlock() + for _, node := range []string{"n1", "n2"} { + for _, m := range []string{"a", "b", "c"} { + b.place(t, node, m) + } + } + var mu sync.Mutex + inputsAsked := map[string]int{} // by candidate and node + var drops []string + respond := func(subject string, f func(m *nats.Msg) any) { + sub, err := nc.Subscribe(subject, func(m *nats.Msg) { + raw, _ := json.Marshal(map[string]any{"result": f(m), "node": "n0"}) + _ = m.Respond(raw) + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + } + respond("mesh.seat.node-engine.call.compose-candidate.*.*", func(m *nats.Msg) any { + node := strings.Split(m.Subject, ".")[5] + var in nodeengine.ComposeCandidateInput + _ = json.Unmarshal(m.Data, &in) + mu.Lock() + inputsAsked[in.Candidate+" "+node]++ + mu.Unlock() + return map[string]any{"candidate": in.Candidate, "node": node, "verdict": "composed", "input": map[string]any{"node": node}, "modules": []any{}, "own": map[string]any{}, "took": 1, "v": 1} + }) + respond("mesh.seat.mesh-catalog.call.entry.*", func(*nats.Msg) any { return meshcatalog.EntryAnswer{Entry: engine} }) + respond("mesh.seat.node-build-go.call.drop.*", func(m *nats.Msg) any { + var in nodebuildgo.DropInput + _ = json.Unmarshal(m.Data, &in) + mu.Lock() + drops = append(drops, in.ID) + mu.Unlock() + return map[string]any{} + }) + respond("mesh.seat.mesh-hand-acts.call.hand-act.*", func(*nats.Msg) any { return map[string]any{"log": "hand-act-1"} }) + ended := make(chan meshwants.StepEnded, 32) + sub, err := nc.Subscribe("mesh.seat.mesh-wants.event.step-ended.n1", func(m *nats.Msg) { + var e meshwants.StepEnded + _ = json.Unmarshal(m.Data, &e) + ended <- e + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + subscribed(t, nc) + + from := b.journal.mark() + _, stop := b.startStoppable(t) + b.journal.await(t, from, "the composition step is heard on", "the composition step served", 30*time.Second) + select { + case <-b.seatListRead: + case <-time.After(30 * time.Second): + t.Fatal("the assigner did not read the seat list within 30 s") + } + queue, _ := buildstep.AskStream(buildstep.AskGo) + worker, err := b.js.CreateOrUpdateConsumer(b.ctx, queue, jetstream.ConsumerConfig{Durable: "stand-in", FilterSubject: "mesh.seat.node-build-go.accept.check.mesh-wants", + AckPolicy: jetstream.AckExplicitPolicy}) + if err != nil { + t.Fatal(err) + } + step := func(id, module, run string, rerun bool) { + t.Helper() + s := meshwants.StepRun{Pipeline: "main.3." + module, Step: meshwants.StepRunStepComposition, Module: module, Branch: trunkBranch, Revision: 3, + Commit: commit, Fingerprint: "src2:" + module, Run: run, Rerun: rerun} + if run != "" { + s.Pipeline = "run-" + run + "." + module + } + if rerun { + s.By = "op@n1" + } + raw, _ := json.Marshal(s) + msg := nats.NewMsg(meshwants.AcceptSubjectStep) + msg.Header.Set("x-event-id", id) + msg.Data = raw + if err := nc.PublishMsg(msg); err != nil { + t.Fatal(err) + } + } + type asked struct { + ask buildstep.StepAsk + machine string + } + // asks takes n run asks of the stand-in build seat, each with the machine its request composes. + asks := func(n int) []asked { + t.Helper() + var out []asked + for range n { + got, err := worker.Next(jetstream.FetchMaxWait(20 * time.Second)) + if err != nil { + t.Fatalf("%d of %d run asks put: %v", len(out), n, err) + } + _ = got.Ack() + a, err := buildstep.DecodeStepAsk(got.Data()) + if err != nil || a.Run == nil { + t.Fatalf("the run ask %s does not read: %v", got.Data(), err) + } + req, err := composer.Decode(readInput(t, nc, a.Run.Input)) + if err != nil || len(req.Machines) != 1 || req.BlameTries != 20 { + t.Fatalf("the request of %s: %+v %v", a.ID, req, err) + } + out = append(out, asked{a, req.Machines[0]}) + } + return out + } + noMoreAsks := func(what string) { + t.Helper() + if got, err := worker.Next(jetstream.FetchMaxWait(300 * time.Millisecond)); err == nil { + t.Fatalf("%s: one more run ask is put: %s", what, got.Data()) + } + } + say := func(event string, node string, body any) { + t.Helper() + raw, _ := json.Marshal(body) + if _, err := b.js.Publish(b.ctx, "mesh.seat.node-build-go.event."+event+"."+node, raw); err != nil { + t.Fatal(err) + } + } + answer := func(a asked, machine string) { + t.Helper() + v := fmt.Sprintf(`{"v":%d,"cases":[{"name":"change","manifests":{},"machines":{%q:%s}}]}`, composer.Version, a.machine, machine) + say("step-ended", "b1", buildstep.StepEnded{V: 1, ID: a.ask.ID, Ask: a.ask.ID, Asker: runAsker, Node: "b1", Outcome: buildstep.OutcomePassed, Output: v, At: time.Now().UTC()}) + } + ends := func(n int, what string) map[string]meshwants.StepEnded { + t.Helper() + out := map[string]meshwants.StepEnded{} + for len(out) < n { + select { + case e := <-ended: + out[e.ID] = e + case <-time.After(20 * time.Second): + t.Fatalf("%s: %d of %d steps ended: %+v", what, len(out), n, out) + } + } + return out + } + runKeys := func(candidate string) []string { + all, _ := b.buckets[buckets.CompositionRuns()].Keys(b.ctx) + var out []string + for _, k := range all { + if isRunKey(k) && strings.HasPrefix(k, keysPart(candidate)+".") { + out = append(out, k) + } + } + return out + } + // readers waits, through a watch of the key, until the run of key has n readers. + readers := func(key string, n int) { + t.Helper() + w, err := b.buckets[buckets.CompositionRuns()].Watch(b.ctx, key) + if err != nil { + t.Fatal(err) + } + defer func() { _ = w.Stop() }() + deadline := time.After(10 * time.Second) + for { + select { + case e := <-w.Updates(): + var run meshwants.CompositionRun + if e != nil && json.Unmarshal(e.Value(), &run) == nil && len(run.Readers) == n { + return + } + case <-deadline: + t.Fatalf("the run %s has not %d readers", key, n) + } + } + } + blames := `{"composes":false,"reason":"two modules own one path","blamed":[{"module":"a","problem":"/etc/x is a's and b's"},{"module":"b","problem":"/etc/x is a's and b's"}]}` + composes := `{"composes":true,"blamed":[]}` + + // One candidate, three steps, two machines: two runs. + for _, m := range []string{"a", "b", "c"} { + step("step-"+m, m, "", false) + } + first := asks(2) + noMoreAsks("three steps of one candidate") + for _, a := range first { + answer(a, map[string]string{"n1": composes, "n2": blames}[a.machine]) + } + got := ends(3, "the three steps") + if a, bb, c := got["step-a"], got["step-b"], got["step-c"]; a.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(a.Said, "n2 holds a") || + bb.Outcome != meshwants.StepEndedOutcomeFailed || c.Outcome != meshwants.StepEndedOutcomePassed || len(c.Warnings) != 2 || + !strings.Contains(c.Warnings[0], "n2 holds a") || len(c.Shared) != 2 || c.Shared[0].Ask == c.Shared[1].Ask { + t.Fatalf("the steps of one candidate: %+v", got) + } + if keys := runKeys("main.3"); len(keys) != 2 || inputsAsked["main.3 n1"] != 1 || inputsAsked["main.3 n2"] != 1 { + t.Fatalf("keys %v, inputs asked %v: want one run and one input per machine", keys, inputsAsked) + } + + // Another candidate (a run on request): its own runs; an old composer and a machine blaming none fail. + step("step-r1", "a", "r1", false) + for _, a := range asks(2) { + answer(a, map[string]string{"n1": `{"composes":false,"reason":"x"}`, "n2": `{"composes":false,"reason":"own","blamed":[]}`}[a.machine]) + } + if e := ends(1, "another candidate")["step-r1"]; e.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(e.Said, "n2 does not compose and blames no module") || + len(e.Warnings) != 1 || e.Warnings[0] != "n1: the composer names no culprits" { + t.Fatalf("a step of another candidate: %+v", e) + } + + // The bound: n1's ask taken and saying progress past the bound, then silent; n2's never taken. + from = b.journal.mark() + step("step-r2", "a", "r2", false) + bound := asks(2) + var n1 asked + for _, a := range bound { + if a.machine == "n1" { + n1 = a + } + } + say("ask-taken", "b1", nodebuildgo.AskTaken{Ask: n1.ask.ID, Asker: runAsker, At: time.Now().UTC(), Node: "b1"}) + for range 3 { // past the 2 s bound, each beat within it + select { + case e := <-ended: + t.Fatalf("a run saying its progress ended: %+v", e) + case <-time.After(time.Second): + } + say("step-progress", "b1", nodebuildgo.StepProgress{Ask: n1.ask.ID, Asker: runAsker, At: time.Now().UTC(), Phase: "run"}) + } + e := ends(1, "the bounds")["step-r2"] + if e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "said no progress within 2s (composition-step-bound)") || + !strings.Contains(e.Said, "taken by no build node within 15s (run-untaken-bound)") { + t.Fatalf("past the bounds: %+v", e) + } + mu.Lock() + dropped := len(drops) + mu.Unlock() + if dropped != 2 { + t.Fatalf("%d asks dropped past their bounds, want 2", dropped) + } + answer(n1, composes) // heard after the bound: ignored and journaled + b.journal.await(t, from, "the step-ended of the run ask "+n1.ask.ID+" is ignored", "a late step-ended", 10*time.Second) + + // A rerun of a: n1 it passed keeps its verdict; n2 it failed starts one run carrying b, which moves. + step("step-a2", "a", "", true) + again := asks(1) + noMoreAsks("a rerun of a") + if again[0].machine != "n2" { + t.Fatalf("the rerun composes %s, want only the machine a did not pass", again[0].machine) + } + answer(again[0], composes) + moved := ends(2, "the rerun") + if a2, bb := moved["step-a2"], moved["step-b"]; a2.Outcome != meshwants.StepEndedOutcomePassed || bb.Outcome != meshwants.StepEndedOutcomePassed || + !bb.Rerun || bb.By != "op@n1" || len(bb.Shared) != 2 || bb.Shared[1].Ask != again[0].ask.ID { + t.Fatalf("after the rerun (asked again %s): a2 %+v; b %+v", again[0].ask.ID, moved["step-a2"].Shared, moved["step-b"]) + } + + // A superseded step leaves the runs; its other reader goes on. + step("step-r3a", "a", "r3", false) + step("step-r3b", "b", "r3", false) + sup := asks(2) + readers(runKeyOf("run-r3", "n1"), 2) + readers(runKeyOf("run-r3", "n2"), 2) + pe, _ := json.Marshal(mpr.PipelineEnded{V: 1, Pipeline: "run-r3.a", Module: "a", Branch: trunkBranch, Commit: commit, Outcome: mpr.PipelineEndedOutcomeSuperseded, Run: "r3", Steps: []mpr.StepResult{}}) + if _, err := b.js.Publish(b.ctx, "mesh.seat."+mpr.Seat+".event."+mpr.EventPipelineEnded+".n1", pe); err != nil { + t.Fatal(err) + } + for _, a := range sup { + answer(a, composes) + } + if got := ends(1, "the superseded run"); got["step-r3b"].Outcome != meshwants.StepEndedOutcomePassed { + t.Fatalf("the other reader: %+v", got) + } + select { + case e := <-ended: + t.Fatalf("a superseded step said its end: %+v", e) + case <-time.After(200 * time.Millisecond): + } + + // Restarted during a run: the new assigner takes it up by its ask. + step("step-r4", "c", "r4", false) + restart := asks(2) + stop() + for _, a := range restart { + answer(a, composes) + } + from = b.journal.mark() + b.startStoppable(t) + b.journal.await(t, from, "is taken up again by its runs (ask step-r4)", "the take-up", 30*time.Second) + if e := ends(1, "the take-up")["step-r4"]; e.Outcome != meshwants.StepEndedOutcomePassed || len(e.Shared) != 2 { + t.Fatalf("the step taken up: %+v", e) + } + + // compositions-drop: the readers ended naming the drop, the key deleted. + step("step-r5", "a", "r5", false) + asks(2) + body, _ := json.Marshal(meshwants.CompositionsDropInput{Candidate: "run-r5", Machine: "n1", Why: "stuck"}) + reply, err := nc.Request("mesh.seat.mesh-wants.call.compositions-drop.n1~mesh-delivery-pipeline", body, 10*time.Second) + if err != nil || !strings.Contains(string(reply.Data), `"ended":[`) { + t.Fatalf("compositions-drop answered %s %v", reply.Data, err) + } + if e := ends(1, "the drop")["step-r5"]; e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "dropped by hand: stuck") { + t.Fatalf("a reader of the dropped run: %+v", e) + } + if keys := runKeys("run-r5"); len(keys) != 1 { + t.Errorf("the keys of run-r5 after the drop: %v, want n2's alone", keys) + } + + // At the cap of 1 MiB no run starts: the step ends could not run, naming the fill. + for i := range 2 { + if _, err := b.buckets[buckets.CompositionRuns()].Put(b.ctx, fmt.Sprintf("fill%d", i), make([]byte, 700<<10)); err != nil { + t.Fatal(err) + } + } + step("step-r6", "a", "r6", false) + if e := ends(1, "the cap")["step-r6"]; e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "is at its cap of 1 MiB") { + t.Fatalf("at the cap: %+v", e) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index 128652db..74648708 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -69,11 +69,13 @@ import ( type bed struct { // seatListRead is closed once the assigner read the seat list (serveSeatList). seatListRead <-chan struct{} - srv *server.Server - js jetstream.JetStream - ctx context.Context - dir string - buckets map[string]jetstream.KeyValue + // settings are the files of settings a test sets, by the variable naming each (as the manifest's resources fill them). + settings map[string]string + srv *server.Server + js jetstream.JetStream + ctx context.Context + dir string + buckets map[string]jetstream.KeyValue // placed is, by node, the modules the bed's assignments names on it (place). placed map[string][]string // journal is what the programs the bed starts write in their journal. @@ -301,6 +303,13 @@ func (b *bed) startStoppable(t *testing.T) (<-chan error, func()) { t.Fatal(err) } env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_ASSIGNER_OUTSIDE_SETTLE_FILE": settle} + for name, value := range b.settings { + file := filepath.Join(b.dir, name) + if err := os.WriteFile(file, []byte(value), 0o600); err != nil { + t.Fatal(err) + } + env[name] = file + } ctx, stop := context.WithCancel(b.ctx) done := make(chan error, 1) go func() {