From 2a93cbcd9bef0ee187abd8872f957ff18b709a74 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 04:26:43 +0100 Subject: [PATCH] R17: the queue's whole test against the bus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runner started whole with pipelines-at-once 1 and pipelines-waiting-cap 1: a second take-in waits, saying wait-held, and is placed with wait-ended on the first's end; a newer waiting take-in takes its shared movers out of an older waiting one, which ends superseded; a start over the cap waits and raises queue-full, cleared on the count; a cancel places the next; a pipeline on its approval alone gives up its place and, answered while it still runs, is keyed running at once; a placing the bus refuses raises pipelines-stalled, cleared when the next start places the waiting from the runner's own state, which starts no pipeline of its own. The tools runner_rerun, runner_accept and runner_ask_again are shown to take the mover's module. Builds: design 47 H §748, §774; acceptance test 112 (the runner's half); lead answers H-A-14, H-A-36, H-A-52, H-A-53, H-A-96, H-A-101, H-A-106, H-A-107 Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../rerun_test.go | 5 + .../verbtools_test.go | 20 +- .../queue_integration_test.go | 405 ++++++++++++++++++ 3 files changed, 426 insertions(+), 4 deletions(-) create mode 100644 modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/rerun_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/rerun_test.go index 004596bd..0524f4bf 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/rerun_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/rerun_test.go @@ -14,6 +14,8 @@ func TestRerunTool(t *testing.T) { in seat.RerunInput }{ {"asks the verb", "", seat.RerunInput{Pipeline: "main.981.node-tools", Step: "composition", Why: "the composer was late"}}, + // H-A-101 (principle 4): the mover's module is named. + {"names the mover", "", seat.RerunInput{Pipeline: "main.981", Module: "node-tools", Step: "composition", Why: "the composer was late"}}, {"why missing", "names why", seat.RerunInput{Pipeline: "main.981.node-tools", Step: "composition"}}, {"step missing", "names the pipeline and the step", seat.RerunInput{Pipeline: "main.981.node-tools", Why: "x"}}, } { @@ -24,6 +26,9 @@ func TestRerunTool(t *testing.T) { asked, body = key, b.(seat.RerunInput) return json.RawMessage(`{}`), nil } + if !takes(RerunTool(ask).Input, "module") { + t.Fatal("runner_rerun's input names no module (H-A-101)") + } _, err := rerun(ask, c.in) if c.refusal != "" { if err == nil || !strings.Contains(err.Error(), c.refusal) || asked != "" { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/verbtools_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/verbtools_test.go index 9ab08415..ff163acd 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/verbtools_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/verbtools_test.go @@ -22,11 +22,12 @@ func TestVerbTools(t *testing.T) { {"run", "runner_run", "seat:mesh-pipeline-runner.run", "", map[string]any{"members": []any{map[string]any{}}, "why": "x"}, seat.RunInput{Members: []seat.Member{{}}, Why: "x"}}, {"run no members", "runner_run", "", "members", map[string]any{}, nil}, - {"accept", "runner_accept", "seat:mesh-pipeline-runner.accept", "", map[string]any{"pipeline": "p", "step": "lint", "artifact": "a", "why": "x"}, - seat.AcceptInput{Pipeline: "p", Step: "lint", Artifact: "a", Why: "x"}}, + // H-A-101 (principle 4): accept and ask-again name the mover's module. + {"accept", "runner_accept", "seat:mesh-pipeline-runner.accept", "", map[string]any{"pipeline": "p", "module": "lib", "step": "lint", "artifact": "a", "why": "x"}, + seat.AcceptInput{Pipeline: "p", Module: "lib", Step: "lint", Artifact: "a", Why: "x"}}, {"accept no step", "runner_accept", "", "pipeline and the step", map[string]any{"pipeline": "p", "why": "x"}, nil}, - {"ask-again", "runner_ask_again", "seat:mesh-pipeline-runner.ask-again", "", map[string]any{"pipeline": "p", "why": "x"}, - seat.AskAgainInput{Pipeline: "p", Why: "x"}}, + {"ask-again", "runner_ask_again", "seat:mesh-pipeline-runner.ask-again", "", map[string]any{"pipeline": "p", "module": "lib", "why": "x"}, + seat.AskAgainInput{Pipeline: "p", Module: "lib", Why: "x"}}, // R14/AC2 (§748, H-A-53): runner_place asks the verb place with the pipeline and why. {"place", "runner_place", "seat:mesh-pipeline-runner.place", "", map[string]any{"pipeline": "main.6", "why": "urgent"}, seat.PlaceInput{Pipeline: "main.6", Why: "urgent"}}, @@ -44,6 +45,10 @@ func TestVerbTools(t *testing.T) { for _, tl := range VerbTools(ask) { if tl.Name == c.tool { run = tl.Run + // H-A-101: a tool asking a verb that names the mover's module offers module in its input. + if _, named := c.args["module"]; named && !takes(tl.Input, "module") { + t.Fatalf("%s's input names no module: %v", c.tool, tl.Input) + } } } out, err := run(c.args) @@ -61,3 +66,10 @@ func TestVerbTools(t *testing.T) { }) } } + +// takes says whether a tool's input schema has the property name. +func takes(input map[string]any, name string) bool { + props, _ := input["properties"].(map[string]any) + _, ok := props[name] + return ok +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go new file mode 100644 index 00000000..e09f2b80 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go @@ -0,0 +1,405 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" + builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" + catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/seatlist" + "git.novox.be/novox/mesh-sdk/go/testbus" + + "mesh-delivery-pipeline/internal/pipeline" +) + +// R17 (design 47 H §748, §774; acceptance test 112's runner half; H-A-14, H-A-36, H-A-52, H-A-53, H-A-96, H-A-106, +// H-A-107): the queue whole against the real bus, the runner started as its process is with pipelines-at-once 1 (its +// default) and pipelines-waiting-cap 1. A second take-in is written waiting and says wait-held, and is placed with wait-ended +// on the first's pipeline-ended; a newer waiting take-in takes the movers it shares out of an older waiting one, which ends +// superseded once none is left; a start over the cap still waits and raises queue-full, cleared when an end brings the +// waiting back to the cap; a cancel places the next; a pipeline whose every open step is an approval asked gives up its +// place, and answered while it still runs is keyed running at once, over the bound, ahead of the waiting; a placing that +// fails leaves pipelines waiting with none running and raises pipelines-stalled; at the runner's next start the waiting +// one is placed from its own state, which clears it, and the start begins no pipeline of its own. +func TestTheQueueRunsPipelinesInTurn(t *testing.T) { + url := testbus.URL(t) + nc, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + + const module = "mesh-delivery-pipeline" + for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries", seat.StateQueue} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}); err != nil { + t.Fatal(err) + } + } + for _, state := range []string{"bounds", TestRunsState} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1, LimitMarkerTTL: time.Minute}); err != nil { + t.Fatal(err) + } + } + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil { + t.Fatal(err) + } + events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}) + if err != nil { + t.Fatal(err) + } + if _, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: "n1_" + module, AckPolicy: jetstream.AckExplicitPolicy}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy}); err != nil { + t.Fatal(err) + } + + // The seats the runner calls, answered as their holders do: four registered modules, gated's policy asking the + // operator's approval; main's half at revision 1 holds them all. + modules := []string{"app", "lib", "tool", "gated"} + var regs []catalog.Registration + var entries []catalog.CatalogEntry + for _, m := range modules { + steps := catalog.StepStatuses{Review: "bypass", Validate: "bypass", Composition: "bypass", Approval: "bypass"} + if m == "gated" { + steps.Approval = "required" + } + regs = append(regs, catalog.Registration{Module: m, Branches: []string{"main"}, Origin: "o/" + m, Policy: catalog.Policy{Steps: steps}}) + entries = append(entries, catalog.CatalogEntry{Module: m, Commit: "c1", Fp: "src2:" + m}) + } + answer := func(subject string, body any) { + if _, err := nc.Subscribe(subject, func(m *nats.Msg) { + raw, _ := json.Marshal(map[string]any{"result": body}) + _ = m.Respond(raw) + }); err != nil { + t.Fatal(err) + } + } + answer("mesh.seat.mesh-catalog.call.registrations.*", catalog.RegistrationsAnswer{Registrations: regs}) + answer("mesh.seat.mesh-catalog.call.revision.*", catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 1, Kind: "trunk", Branch: "main", Source: 2}, Entries: entries}) + answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{}) + // The router of the operator's asks holds the seat operator-channel; its record of each ask is in its state asks. + const router = "operator-router" + answer("mesh.seat."+seatlist.HolderSeat+".call."+seatlist.HolderVerb+".*", map[string]any{"seats": []any{ + map[string]any{"seat": oc.Seat, "holders": []any{map[string]any{"module": router}}}}}) + records, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: router + "_" + oc.StateAsks, History: 1}) + if err != nil { + t.Fatal(err) + } + def, err := os.ReadFile("../../../mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json") + if err != nil { + t.Fatal(err) + } + if _, err := nc.Subscribe("mesh.seat.mesh-catalog.call."+seatlist.Verb+".*", func(m *nats.Msg) { + var in struct { + Seat string `json:"seat"` + } + _ = json.Unmarshal(m.Data, &in) + entry := seatlist.Entry{Seat: seat.Seat, Source: seatlist.SourceTrunk, Digest: "sha256:runner"} + var result any = seatlist.Listing{Revision: 1, Seats: []seatlist.Entry{entry}} + if in.Seat != "" { + entry.Definition = def + result = seatlist.Answer{Seat: in.Seat, Found: in.Seat == seat.Seat, Entry: &entry, Revision: 1} + } + raw, _ := json.Marshal(map[string]any{"result": result}) + _ = m.Respond(raw) + }); err != nil { + t.Fatal(err) + } + // What the runner says per source revision: wait-held, wait-ended and pipeline-ended. + var saidMu sync.Mutex + var said []string + if _, err := nc.Subscribe("mesh.seat."+seat.Seat+".event.>", func(m *nats.Msg) { + saidMu.Lock() + said = append(said, strings.TrimPrefix(m.Subject, "mesh.seat."+seat.Seat+".event.")) + saidMu.Unlock() + }); err != nil { + t.Fatal(err) + } + wasSaid := func(s string) bool { saidMu.Lock(); defer saidMu.Unlock(); return slices.Contains(said, s) } + + dir := t.TempDir() + broker := filepath.Join(dir, "broker") + raw, _ := json.Marshal(process.Account{URL: url, Node: "n1", Module: module, User: "runner", Password: "x"}) + if err := os.WriteFile(broker, raw, 0o600); err != nil { + t.Fatal(err) + } + waitingCap := filepath.Join(dir, "pipelines-waiting-cap") + if err := os.WriteFile(waitingCap, []byte("1\n"), 0o600); err != nil { + t.Fatal(err) + } + logs := &syncBuffer{} + start := func() func() { + runCtx, stop := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { + done <- run(runCtx, func(k string) string { + switch k { + case process.BrokerFileEnv: + return broker + case envPipelinesWaitingCap: + return waitingCap + } + return "" + }, logs) + }() + return func() { stop(); <-done } + } + stopRunner := start() + defer func() { stopRunner() }() + + pipelines, err := js.KeyValue(ctx, module+"_pipelines") + if err != nil { + t.Fatal(err) + } + queue, err := js.KeyValue(ctx, module+"_"+seat.StateQueue) + if err != nil { + t.Fatal(err) + } + findings, err := js.KeyValue(ctx, module+"_findings") + if err != nil { + t.Fatal(err) + } + get := func(key string) seat.Pipeline { + var p seat.Pipeline + readPipeline(ctx, pipelines, key, &p) + return p + } + keyed := func(prefix string) []string { + ks, err := queue.ListKeysFiltered(ctx, prefix+">") + if err != nil { + return nil + } + var out []string + for k := range ks.Keys() { + out = append(out, k) + } + slices.Sort(out) + return out + } + running := func() []string { + var out []string + for _, k := range keyed("running.") { + out = append(out, strings.TrimPrefix(k, "running.")) + } + return out + } + finding := func(kind string) (string, bool) { + e, err := findings.Get(ctx, "delivery."+kind+"."+kind) + if err != nil { + return "", false + } + return string(e.Value()), true + } + take := func(source int64, movers ...string) { + t.Helper() + ev := catalog.TakeInTaken{Branch: "main", Source: source, Revision: 1, Call: fmt.Sprintf("%016x", source)} + for _, m := range movers { + ev.Entries = append(ev.Entries, catalog.TakeInEntry{Module: m, Commit: "c1", Fingerprint: "src2:" + m}) + } + body, _ := json.Marshal(ev) + if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", body); err != nil { + t.Fatal(err) + } + key := pipeline.Key("main", source) + waitFor(t, ctx, logs, func() bool { var p seat.Pipeline; return readPipeline(ctx, pipelines, key, &p) }) + } + // built says the build of mover m in pipeline key ended passed, as the seat mesh-builds' holder says it. + built := func(key, m string) { + t.Helper() + var b seat.StepResult + waitFor(t, ctx, logs, func() bool { + b = stepFor(get(key), m, seat.StepResultStepBuild) + return b.Outcome == seat.StepResultOutcomeRunning && b.Ask != "" + }) + ended, _ := json.Marshal(builds.StepEnded{ID: b.Ask, Pipeline: key, Step: "build", Node: "n1", Outcome: "passed", + Entries: []builds.BuildEntry{{Module: m, Commit: "c1", Fingerprint: "src2:" + m}}}) + if _, err := js.Publish(ctx, "mesh.seat.mesh-builds.event.step-ended.n1", ended); err != nil { + t.Fatal(err) + } + } + verb := func(name string, in any) error { + t.Helper() + var reply struct { + Error string `json:"error"` + } + waitFor(t, ctx, logs, func() bool { + m, err := nc.Request("mesh.seat."+seat.Seat+".call."+name+".agent~dev", mustJSON(in), 15*time.Second) + reply.Error = "" + return err == nil && json.Unmarshal(m.Data, &reply) == nil && !strings.Contains(reply.Error, "not yet read") + }) + if reply.Error != "" { + return errors.New(reply.Error) + } + return nil + } + outcome := func(key, want string) { + t.Helper() + waitFor(t, ctx, logs, func() bool { return get(key).Outcome == want }) + } + + // Two take-ins in a row: the first runs, its build asked; the second is written waiting under sequence 1 with no step, + // says pipeline-started and wait-held. + take(3, "app") + outcome("main.3", seat.PipelineOutcomeRunning) + take(4, "lib") + if p := get("main.4"); p.Outcome != seat.PipelineOutcomeWaiting || p.Seq != 1 || len(p.Steps) != 0 { + t.Fatalf("main.4 = %s seq %d with %d steps, want waiting at 1 with none", p.Outcome, p.Seq, len(p.Steps)) + } + waitFor(t, ctx, logs, func() bool { return wasSaid("wait-held.main.4") && wasSaid("pipeline-started.main.4") }) + if r := running(); !slices.Equal(r, []string{"main.3"}) { + t.Fatalf("running %v, want main.3 alone", r) + } + // The first's pipeline-ended places the second, which says wait-ended and has its build asked. + built("main.3", "app") + outcome("main.3", seat.PipelineOutcomePassed) + outcome("main.4", seat.PipelineOutcomeRunning) + waitFor(t, ctx, logs, func() bool { return wasSaid("wait-ended.main.4") }) + waitFor(t, ctx, logs, func() bool { return stepFor(get("main.4"), "lib", seat.StepResultStepBuild).Ask != "" }) + + // A newer waiting take-in takes app out of the older waiting main.5; over the cap of 1 it still waits and raises + // queue-full naming it. main.7 takes tool, main.5's last mover, so main.5 ends superseded and leaves the queue. + take(5, "tool", "app") + take(6, "app") + waitFor(t, ctx, logs, func() bool { f, ok := finding(pipeline.QueueFullKind); return ok && strings.Contains(f, "main.6") }) + if p := get("main.5"); p.Outcome != seat.PipelineOutcomeWaiting || len(p.Movers) != 1 || p.Movers[0].Module != "tool" || !slices.Equal(p.Superseded, []string{"app"}) { + t.Fatalf("main.5 = %s moving %+v, superseded %v; want waiting with tool alone, app superseded", p.Outcome, p.Movers, p.Superseded) + } + take(7, "tool") + outcome("main.5", seat.PipelineOutcomeSuperseded) + waitFor(t, ctx, logs, func() bool { return wasSaid("pipeline-ended.main.5") && len(keyed("waiting.")) == 2 }) + if p := get("main.4"); p.Outcome != seat.PipelineOutcomeRunning { + t.Fatalf("the running main.4 = %s, want it running to its end", p.Outcome) + } + // A cancel places the next, the lowest sequence; one waiting is back within the cap, so queue-full is cleared. + if err := verb(seat.VerbCancel, seat.CancelInput{Pipeline: "main.4", Why: "the queue's whole test"}); err != nil { + t.Fatal(err) + } + outcome("main.4", seat.PipelineOutcomeCancelled) + outcome("main.6", seat.PipelineOutcomeRunning) + waitFor(t, ctx, logs, func() bool { _, ok := finding(pipeline.QueueFullKind); return !ok }) + if p := get("main.7"); p.Outcome != seat.PipelineOutcomeWaiting { + t.Fatalf("main.7 = %s, want waiting behind main.6", p.Outcome) + } + + // gated waits, then runs; once its build passed its every open step is the approval asked of the operator: it gives up + // its place and main.9 is placed. + built("main.6", "app") + outcome("main.7", seat.PipelineOutcomeRunning) + take(8, "gated") + built("main.7", "tool") + outcome("main.8", seat.PipelineOutcomeRunning) + take(9, "lib") + built("main.8", "gated") + outcome("main.9", seat.PipelineOutcomeRunning) + approval := stepFor(get("main.8"), "gated", seat.StepResultStepApproval) + if p := get("main.8"); p.Outcome != seat.PipelineOutcomeRunning || approval.Outcome != seat.StepResultOutcomeRunning || approval.Ask == "" { + t.Fatalf("main.8 = %s, its approval %+v; want running on its approval asked", p.Outcome, approval) + } + waitFor(t, ctx, logs, func() bool { return slices.Equal(running(), []string{"main.9"}) }) + // gated's build is asked again, its mover named (H-A-101), and then its approval answered: still running, main.8 is + // keyed running at once beside main.9, over the bound, while main.10 waits. + if err := verb(seat.VerbRerun, seat.RerunInput{Pipeline: "main.8", Module: "gated", Step: "build", Why: "the queue's whole test"}); err != nil { + t.Fatal(err) + } + take(10, "app") + warrant := oc.Warrant{About: pipeline.About("gated", "main"), Ask: approval.Ask, Asker: module, Outcome: oc.WarrantOutcomeExpired, + Words: "the queue's whole test", At: time.Now().UTC().Format(time.RFC3339)} + if _, err := records.Put(ctx, module+"."+approval.Ask, mustJSON(map[string]any{"state": "decided", "warrant": warrant})); err != nil { + t.Fatal(err) + } + if _, err := js.Publish(ctx, oc.SubjectDecided(module), mustJSON(warrant)); err != nil { + t.Fatal(err) + } + waitFor(t, ctx, logs, func() bool { return slices.Equal(running(), []string{"main.8", "main.9"}) }) + if p := get("main.10"); p.Outcome != seat.PipelineOutcomeWaiting { + t.Fatalf("main.10 = %s, want waiting while two run", p.Outcome) + } + // main.8's end (its approval declined) places nothing while main.9 runs. + built("main.8", "gated") + outcome("main.8", seat.PipelineOutcomeFailed) + waitFor(t, ctx, logs, func() bool { return slices.Equal(running(), []string{"main.9"}) }) + + // The bus refuses the queue's next running key, so main.9's end places nothing: main.10 waits with none running, and + // pipelines-stalled names it. + setMaxMsgSize(t, ctx, js, "KV_"+module+"_"+seat.StateQueue, 40) + built("main.9", "lib") + outcome("main.9", seat.PipelineOutcomePassed) + waitFor(t, ctx, logs, func() bool { + f, ok := finding(pipeline.PipelinesStalledKind) + return ok && strings.Contains(f, "main.10") + }) + if p, r := get("main.10"), running(); p.Outcome != seat.PipelineOutcomeWaiting || len(r) != 0 { + t.Fatalf("main.10 = %s with %v running, want it waiting and none running", p.Outcome, r) + } + + // At the runner's next start the waiting main.10 is placed from its own state, saying wait-ended, which clears + // pipelines-stalled; the start begins no pipeline of its own. + stopRunner() + setMaxMsgSize(t, ctx, js, "KV_"+module+"_"+seat.StateQueue, -1) + before := pipelineKeys(t, ctx, pipelines) + stopRunner = start() + outcome("main.10", seat.PipelineOutcomeRunning) + waitFor(t, ctx, logs, func() bool { return wasSaid("wait-ended.main.10") && slices.Equal(running(), []string{"main.10"}) }) + waitFor(t, ctx, logs, func() bool { _, ok := finding(pipeline.PipelinesStalledKind); return !ok }) + if after := pipelineKeys(t, ctx, pipelines); !slices.Equal(before, after) { + t.Fatalf("the start wrote pipelines %v, want %v", after, before) + } +} + +func stepFor(p seat.Pipeline, module, step string) seat.StepResult { + for _, s := range p.Steps { + if s.Module == module && s.Step == step { + return s + } + } + return seat.StepResult{} +} + +// setMaxMsgSize sets the largest message the stream takes (-1 for none): a key's write over it is refused by the bus. +func setMaxMsgSize(t *testing.T, ctx context.Context, js jetstream.JetStream, stream string, size int32) { + t.Helper() + s, err := js.Stream(ctx, stream) + if err != nil { + t.Fatal(err) + } + cfg := s.CachedInfo().Config + cfg.MaxMsgSize = size + if _, err := js.UpdateStream(ctx, cfg); err != nil { + t.Fatal(err) + } +} + +func pipelineKeys(t *testing.T, ctx context.Context, kv jetstream.KeyValue) []string { + t.Helper() + ks, err := kv.Keys(ctx) + if err != nil { + t.Fatal(err) + } + slices.Sort(ks) + return ks +}