R17: the queue's whole test against the bus
The runner started whole with pipelines-at-once 1 and pipelines-waiting-cap 1: a second take-in waits, saying wait-held, and is placed with wait-ended on the first's end; a newer waiting take-in takes its shared movers out of an older waiting one, which ends superseded; a start over the cap waits and raises queue-full, cleared on the count; a cancel places the next; a pipeline on its approval alone gives up its place and, answered while it still runs, is keyed running at once; a placing the bus refuses raises pipelines-stalled, cleared when the next start places the waiting from the runner's own state, which starts no pipeline of its own. The tools runner_rerun, runner_accept and runner_ask_again are shown to take the mover's module. Builds: design 47 H §748, §774; acceptance test 112 (the runner's half); lead answers H-A-14, H-A-36, H-A-52, H-A-53, H-A-96, H-A-101, H-A-106, H-A-107 Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
This commit is contained in:
@@ -14,6 +14,8 @@ func TestRerunTool(t *testing.T) {
|
||||
in seat.RerunInput
|
||||
}{
|
||||
{"asks the verb", "", seat.RerunInput{Pipeline: "main.981.node-tools", Step: "composition", Why: "the composer was late"}},
|
||||
// H-A-101 (principle 4): the mover's module is named.
|
||||
{"names the mover", "", seat.RerunInput{Pipeline: "main.981", Module: "node-tools", Step: "composition", Why: "the composer was late"}},
|
||||
{"why missing", "names why", seat.RerunInput{Pipeline: "main.981.node-tools", Step: "composition"}},
|
||||
{"step missing", "names the pipeline and the step", seat.RerunInput{Pipeline: "main.981.node-tools", Why: "x"}},
|
||||
} {
|
||||
@@ -24,6 +26,9 @@ func TestRerunTool(t *testing.T) {
|
||||
asked, body = key, b.(seat.RerunInput)
|
||||
return json.RawMessage(`{}`), nil
|
||||
}
|
||||
if !takes(RerunTool(ask).Input, "module") {
|
||||
t.Fatal("runner_rerun's input names no module (H-A-101)")
|
||||
}
|
||||
_, err := rerun(ask, c.in)
|
||||
if c.refusal != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), c.refusal) || asked != "" {
|
||||
|
||||
@@ -22,11 +22,12 @@ func TestVerbTools(t *testing.T) {
|
||||
{"run", "runner_run", "seat:mesh-pipeline-runner.run", "", map[string]any{"members": []any{map[string]any{}}, "why": "x"},
|
||||
seat.RunInput{Members: []seat.Member{{}}, Why: "x"}},
|
||||
{"run no members", "runner_run", "", "members", map[string]any{}, nil},
|
||||
{"accept", "runner_accept", "seat:mesh-pipeline-runner.accept", "", map[string]any{"pipeline": "p", "step": "lint", "artifact": "a", "why": "x"},
|
||||
seat.AcceptInput{Pipeline: "p", Step: "lint", Artifact: "a", Why: "x"}},
|
||||
// H-A-101 (principle 4): accept and ask-again name the mover's module.
|
||||
{"accept", "runner_accept", "seat:mesh-pipeline-runner.accept", "", map[string]any{"pipeline": "p", "module": "lib", "step": "lint", "artifact": "a", "why": "x"},
|
||||
seat.AcceptInput{Pipeline: "p", Module: "lib", Step: "lint", Artifact: "a", Why: "x"}},
|
||||
{"accept no step", "runner_accept", "", "pipeline and the step", map[string]any{"pipeline": "p", "why": "x"}, nil},
|
||||
{"ask-again", "runner_ask_again", "seat:mesh-pipeline-runner.ask-again", "", map[string]any{"pipeline": "p", "why": "x"},
|
||||
seat.AskAgainInput{Pipeline: "p", Why: "x"}},
|
||||
{"ask-again", "runner_ask_again", "seat:mesh-pipeline-runner.ask-again", "", map[string]any{"pipeline": "p", "module": "lib", "why": "x"},
|
||||
seat.AskAgainInput{Pipeline: "p", Module: "lib", Why: "x"}},
|
||||
// R14/AC2 (§748, H-A-53): runner_place asks the verb place with the pipeline and why.
|
||||
{"place", "runner_place", "seat:mesh-pipeline-runner.place", "", map[string]any{"pipeline": "main.6", "why": "urgent"},
|
||||
seat.PlaceInput{Pipeline: "main.6", Why: "urgent"}},
|
||||
@@ -44,6 +45,10 @@ func TestVerbTools(t *testing.T) {
|
||||
for _, tl := range VerbTools(ask) {
|
||||
if tl.Name == c.tool {
|
||||
run = tl.Run
|
||||
// H-A-101: a tool asking a verb that names the mover's module offers module in its input.
|
||||
if _, named := c.args["module"]; named && !takes(tl.Input, "module") {
|
||||
t.Fatalf("%s's input names no module: %v", c.tool, tl.Input)
|
||||
}
|
||||
}
|
||||
}
|
||||
out, err := run(c.args)
|
||||
@@ -61,3 +66,10 @@ func TestVerbTools(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// takes says whether a tool's input schema has the property name.
|
||||
func takes(input map[string]any, name string) bool {
|
||||
props, _ := input["properties"].(map[string]any)
|
||||
_, ok := props[name]
|
||||
return ok
|
||||
}
|
||||
|
||||
@@ -0,0 +1,405 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
|
||||
"mesh-delivery-pipeline/internal/pipeline"
|
||||
)
|
||||
|
||||
// R17 (design 47 H §748, §774; acceptance test 112's runner half; H-A-14, H-A-36, H-A-52, H-A-53, H-A-96, H-A-106,
|
||||
// H-A-107): the queue whole against the real bus, the runner started as its process is with pipelines-at-once 1 (its
|
||||
// default) and pipelines-waiting-cap 1. A second take-in is written waiting and says wait-held, and is placed with wait-ended
|
||||
// on the first's pipeline-ended; a newer waiting take-in takes the movers it shares out of an older waiting one, which ends
|
||||
// superseded once none is left; a start over the cap still waits and raises queue-full, cleared when an end brings the
|
||||
// waiting back to the cap; a cancel places the next; a pipeline whose every open step is an approval asked gives up its
|
||||
// place, and answered while it still runs is keyed running at once, over the bound, ahead of the waiting; a placing that
|
||||
// fails leaves pipelines waiting with none running and raises pipelines-stalled; at the runner's next start the waiting
|
||||
// one is placed from its own state, which clears it, and the start begins no pipeline of its own.
|
||||
func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
nc, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
|
||||
const module = "mesh-delivery-pipeline"
|
||||
for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries", seat.StateQueue} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, state := range []string{"bounds", TestRunsState} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1, LimitMarkerTTL: time.Minute}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: "n1_" + module, AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The seats the runner calls, answered as their holders do: four registered modules, gated's policy asking the
|
||||
// operator's approval; main's half at revision 1 holds them all.
|
||||
modules := []string{"app", "lib", "tool", "gated"}
|
||||
var regs []catalog.Registration
|
||||
var entries []catalog.CatalogEntry
|
||||
for _, m := range modules {
|
||||
steps := catalog.StepStatuses{Review: "bypass", Validate: "bypass", Composition: "bypass", Approval: "bypass"}
|
||||
if m == "gated" {
|
||||
steps.Approval = "required"
|
||||
}
|
||||
regs = append(regs, catalog.Registration{Module: m, Branches: []string{"main"}, Origin: "o/" + m, Policy: catalog.Policy{Steps: steps}})
|
||||
entries = append(entries, catalog.CatalogEntry{Module: m, Commit: "c1", Fp: "src2:" + m})
|
||||
}
|
||||
answer := func(subject string, body any) {
|
||||
if _, err := nc.Subscribe(subject, func(m *nats.Msg) {
|
||||
raw, _ := json.Marshal(map[string]any{"result": body})
|
||||
_ = m.Respond(raw)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
answer("mesh.seat.mesh-catalog.call.registrations.*", catalog.RegistrationsAnswer{Registrations: regs})
|
||||
answer("mesh.seat.mesh-catalog.call.revision.*", catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 1, Kind: "trunk", Branch: "main", Source: 2}, Entries: entries})
|
||||
answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{})
|
||||
// The router of the operator's asks holds the seat operator-channel; its record of each ask is in its state asks.
|
||||
const router = "operator-router"
|
||||
answer("mesh.seat."+seatlist.HolderSeat+".call."+seatlist.HolderVerb+".*", map[string]any{"seats": []any{
|
||||
map[string]any{"seat": oc.Seat, "holders": []any{map[string]any{"module": router}}}}})
|
||||
records, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: router + "_" + oc.StateAsks, History: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
def, err := os.ReadFile("../../../mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := nc.Subscribe("mesh.seat.mesh-catalog.call."+seatlist.Verb+".*", func(m *nats.Msg) {
|
||||
var in struct {
|
||||
Seat string `json:"seat"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Data, &in)
|
||||
entry := seatlist.Entry{Seat: seat.Seat, Source: seatlist.SourceTrunk, Digest: "sha256:runner"}
|
||||
var result any = seatlist.Listing{Revision: 1, Seats: []seatlist.Entry{entry}}
|
||||
if in.Seat != "" {
|
||||
entry.Definition = def
|
||||
result = seatlist.Answer{Seat: in.Seat, Found: in.Seat == seat.Seat, Entry: &entry, Revision: 1}
|
||||
}
|
||||
raw, _ := json.Marshal(map[string]any{"result": result})
|
||||
_ = m.Respond(raw)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// What the runner says per source revision: wait-held, wait-ended and pipeline-ended.
|
||||
var saidMu sync.Mutex
|
||||
var said []string
|
||||
if _, err := nc.Subscribe("mesh.seat."+seat.Seat+".event.>", func(m *nats.Msg) {
|
||||
saidMu.Lock()
|
||||
said = append(said, strings.TrimPrefix(m.Subject, "mesh.seat."+seat.Seat+".event."))
|
||||
saidMu.Unlock()
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wasSaid := func(s string) bool { saidMu.Lock(); defer saidMu.Unlock(); return slices.Contains(said, s) }
|
||||
|
||||
dir := t.TempDir()
|
||||
broker := filepath.Join(dir, "broker")
|
||||
raw, _ := json.Marshal(process.Account{URL: url, Node: "n1", Module: module, User: "runner", Password: "x"})
|
||||
if err := os.WriteFile(broker, raw, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitingCap := filepath.Join(dir, "pipelines-waiting-cap")
|
||||
if err := os.WriteFile(waitingCap, []byte("1\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
logs := &syncBuffer{}
|
||||
start := func() func() {
|
||||
runCtx, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- run(runCtx, func(k string) string {
|
||||
switch k {
|
||||
case process.BrokerFileEnv:
|
||||
return broker
|
||||
case envPipelinesWaitingCap:
|
||||
return waitingCap
|
||||
}
|
||||
return ""
|
||||
}, logs)
|
||||
}()
|
||||
return func() { stop(); <-done }
|
||||
}
|
||||
stopRunner := start()
|
||||
defer func() { stopRunner() }()
|
||||
|
||||
pipelines, err := js.KeyValue(ctx, module+"_pipelines")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
queue, err := js.KeyValue(ctx, module+"_"+seat.StateQueue)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
findings, err := js.KeyValue(ctx, module+"_findings")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
get := func(key string) seat.Pipeline {
|
||||
var p seat.Pipeline
|
||||
readPipeline(ctx, pipelines, key, &p)
|
||||
return p
|
||||
}
|
||||
keyed := func(prefix string) []string {
|
||||
ks, err := queue.ListKeysFiltered(ctx, prefix+">")
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for k := range ks.Keys() {
|
||||
out = append(out, k)
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
running := func() []string {
|
||||
var out []string
|
||||
for _, k := range keyed("running.") {
|
||||
out = append(out, strings.TrimPrefix(k, "running."))
|
||||
}
|
||||
return out
|
||||
}
|
||||
finding := func(kind string) (string, bool) {
|
||||
e, err := findings.Get(ctx, "delivery."+kind+"."+kind)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
return string(e.Value()), true
|
||||
}
|
||||
take := func(source int64, movers ...string) {
|
||||
t.Helper()
|
||||
ev := catalog.TakeInTaken{Branch: "main", Source: source, Revision: 1, Call: fmt.Sprintf("%016x", source)}
|
||||
for _, m := range movers {
|
||||
ev.Entries = append(ev.Entries, catalog.TakeInEntry{Module: m, Commit: "c1", Fingerprint: "src2:" + m})
|
||||
}
|
||||
body, _ := json.Marshal(ev)
|
||||
if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := pipeline.Key("main", source)
|
||||
waitFor(t, ctx, logs, func() bool { var p seat.Pipeline; return readPipeline(ctx, pipelines, key, &p) })
|
||||
}
|
||||
// built says the build of mover m in pipeline key ended passed, as the seat mesh-builds' holder says it.
|
||||
built := func(key, m string) {
|
||||
t.Helper()
|
||||
var b seat.StepResult
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
b = stepFor(get(key), m, seat.StepResultStepBuild)
|
||||
return b.Outcome == seat.StepResultOutcomeRunning && b.Ask != ""
|
||||
})
|
||||
ended, _ := json.Marshal(builds.StepEnded{ID: b.Ask, Pipeline: key, Step: "build", Node: "n1", Outcome: "passed",
|
||||
Entries: []builds.BuildEntry{{Module: m, Commit: "c1", Fingerprint: "src2:" + m}}})
|
||||
if _, err := js.Publish(ctx, "mesh.seat.mesh-builds.event.step-ended.n1", ended); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
verb := func(name string, in any) error {
|
||||
t.Helper()
|
||||
var reply struct {
|
||||
Error string `json:"error"`
|
||||
}
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
m, err := nc.Request("mesh.seat."+seat.Seat+".call."+name+".agent~dev", mustJSON(in), 15*time.Second)
|
||||
reply.Error = ""
|
||||
return err == nil && json.Unmarshal(m.Data, &reply) == nil && !strings.Contains(reply.Error, "not yet read")
|
||||
})
|
||||
if reply.Error != "" {
|
||||
return errors.New(reply.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
outcome := func(key, want string) {
|
||||
t.Helper()
|
||||
waitFor(t, ctx, logs, func() bool { return get(key).Outcome == want })
|
||||
}
|
||||
|
||||
// Two take-ins in a row: the first runs, its build asked; the second is written waiting under sequence 1 with no step,
|
||||
// says pipeline-started and wait-held.
|
||||
take(3, "app")
|
||||
outcome("main.3", seat.PipelineOutcomeRunning)
|
||||
take(4, "lib")
|
||||
if p := get("main.4"); p.Outcome != seat.PipelineOutcomeWaiting || p.Seq != 1 || len(p.Steps) != 0 {
|
||||
t.Fatalf("main.4 = %s seq %d with %d steps, want waiting at 1 with none", p.Outcome, p.Seq, len(p.Steps))
|
||||
}
|
||||
waitFor(t, ctx, logs, func() bool { return wasSaid("wait-held.main.4") && wasSaid("pipeline-started.main.4") })
|
||||
if r := running(); !slices.Equal(r, []string{"main.3"}) {
|
||||
t.Fatalf("running %v, want main.3 alone", r)
|
||||
}
|
||||
// The first's pipeline-ended places the second, which says wait-ended and has its build asked.
|
||||
built("main.3", "app")
|
||||
outcome("main.3", seat.PipelineOutcomePassed)
|
||||
outcome("main.4", seat.PipelineOutcomeRunning)
|
||||
waitFor(t, ctx, logs, func() bool { return wasSaid("wait-ended.main.4") })
|
||||
waitFor(t, ctx, logs, func() bool { return stepFor(get("main.4"), "lib", seat.StepResultStepBuild).Ask != "" })
|
||||
|
||||
// A newer waiting take-in takes app out of the older waiting main.5; over the cap of 1 it still waits and raises
|
||||
// queue-full naming it. main.7 takes tool, main.5's last mover, so main.5 ends superseded and leaves the queue.
|
||||
take(5, "tool", "app")
|
||||
take(6, "app")
|
||||
waitFor(t, ctx, logs, func() bool { f, ok := finding(pipeline.QueueFullKind); return ok && strings.Contains(f, "main.6") })
|
||||
if p := get("main.5"); p.Outcome != seat.PipelineOutcomeWaiting || len(p.Movers) != 1 || p.Movers[0].Module != "tool" || !slices.Equal(p.Superseded, []string{"app"}) {
|
||||
t.Fatalf("main.5 = %s moving %+v, superseded %v; want waiting with tool alone, app superseded", p.Outcome, p.Movers, p.Superseded)
|
||||
}
|
||||
take(7, "tool")
|
||||
outcome("main.5", seat.PipelineOutcomeSuperseded)
|
||||
waitFor(t, ctx, logs, func() bool { return wasSaid("pipeline-ended.main.5") && len(keyed("waiting.")) == 2 })
|
||||
if p := get("main.4"); p.Outcome != seat.PipelineOutcomeRunning {
|
||||
t.Fatalf("the running main.4 = %s, want it running to its end", p.Outcome)
|
||||
}
|
||||
// A cancel places the next, the lowest sequence; one waiting is back within the cap, so queue-full is cleared.
|
||||
if err := verb(seat.VerbCancel, seat.CancelInput{Pipeline: "main.4", Why: "the queue's whole test"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
outcome("main.4", seat.PipelineOutcomeCancelled)
|
||||
outcome("main.6", seat.PipelineOutcomeRunning)
|
||||
waitFor(t, ctx, logs, func() bool { _, ok := finding(pipeline.QueueFullKind); return !ok })
|
||||
if p := get("main.7"); p.Outcome != seat.PipelineOutcomeWaiting {
|
||||
t.Fatalf("main.7 = %s, want waiting behind main.6", p.Outcome)
|
||||
}
|
||||
|
||||
// gated waits, then runs; once its build passed its every open step is the approval asked of the operator: it gives up
|
||||
// its place and main.9 is placed.
|
||||
built("main.6", "app")
|
||||
outcome("main.7", seat.PipelineOutcomeRunning)
|
||||
take(8, "gated")
|
||||
built("main.7", "tool")
|
||||
outcome("main.8", seat.PipelineOutcomeRunning)
|
||||
take(9, "lib")
|
||||
built("main.8", "gated")
|
||||
outcome("main.9", seat.PipelineOutcomeRunning)
|
||||
approval := stepFor(get("main.8"), "gated", seat.StepResultStepApproval)
|
||||
if p := get("main.8"); p.Outcome != seat.PipelineOutcomeRunning || approval.Outcome != seat.StepResultOutcomeRunning || approval.Ask == "" {
|
||||
t.Fatalf("main.8 = %s, its approval %+v; want running on its approval asked", p.Outcome, approval)
|
||||
}
|
||||
waitFor(t, ctx, logs, func() bool { return slices.Equal(running(), []string{"main.9"}) })
|
||||
// gated's build is asked again, its mover named (H-A-101), and then its approval answered: still running, main.8 is
|
||||
// keyed running at once beside main.9, over the bound, while main.10 waits.
|
||||
if err := verb(seat.VerbRerun, seat.RerunInput{Pipeline: "main.8", Module: "gated", Step: "build", Why: "the queue's whole test"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
take(10, "app")
|
||||
warrant := oc.Warrant{About: pipeline.About("gated", "main"), Ask: approval.Ask, Asker: module, Outcome: oc.WarrantOutcomeExpired,
|
||||
Words: "the queue's whole test", At: time.Now().UTC().Format(time.RFC3339)}
|
||||
if _, err := records.Put(ctx, module+"."+approval.Ask, mustJSON(map[string]any{"state": "decided", "warrant": warrant})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.Publish(ctx, oc.SubjectDecided(module), mustJSON(warrant)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFor(t, ctx, logs, func() bool { return slices.Equal(running(), []string{"main.8", "main.9"}) })
|
||||
if p := get("main.10"); p.Outcome != seat.PipelineOutcomeWaiting {
|
||||
t.Fatalf("main.10 = %s, want waiting while two run", p.Outcome)
|
||||
}
|
||||
// main.8's end (its approval declined) places nothing while main.9 runs.
|
||||
built("main.8", "gated")
|
||||
outcome("main.8", seat.PipelineOutcomeFailed)
|
||||
waitFor(t, ctx, logs, func() bool { return slices.Equal(running(), []string{"main.9"}) })
|
||||
|
||||
// The bus refuses the queue's next running key, so main.9's end places nothing: main.10 waits with none running, and
|
||||
// pipelines-stalled names it.
|
||||
setMaxMsgSize(t, ctx, js, "KV_"+module+"_"+seat.StateQueue, 40)
|
||||
built("main.9", "lib")
|
||||
outcome("main.9", seat.PipelineOutcomePassed)
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
f, ok := finding(pipeline.PipelinesStalledKind)
|
||||
return ok && strings.Contains(f, "main.10")
|
||||
})
|
||||
if p, r := get("main.10"), running(); p.Outcome != seat.PipelineOutcomeWaiting || len(r) != 0 {
|
||||
t.Fatalf("main.10 = %s with %v running, want it waiting and none running", p.Outcome, r)
|
||||
}
|
||||
|
||||
// At the runner's next start the waiting main.10 is placed from its own state, saying wait-ended, which clears
|
||||
// pipelines-stalled; the start begins no pipeline of its own.
|
||||
stopRunner()
|
||||
setMaxMsgSize(t, ctx, js, "KV_"+module+"_"+seat.StateQueue, -1)
|
||||
before := pipelineKeys(t, ctx, pipelines)
|
||||
stopRunner = start()
|
||||
outcome("main.10", seat.PipelineOutcomeRunning)
|
||||
waitFor(t, ctx, logs, func() bool { return wasSaid("wait-ended.main.10") && slices.Equal(running(), []string{"main.10"}) })
|
||||
waitFor(t, ctx, logs, func() bool { _, ok := finding(pipeline.PipelinesStalledKind); return !ok })
|
||||
if after := pipelineKeys(t, ctx, pipelines); !slices.Equal(before, after) {
|
||||
t.Fatalf("the start wrote pipelines %v, want %v", after, before)
|
||||
}
|
||||
}
|
||||
|
||||
func stepFor(p seat.Pipeline, module, step string) seat.StepResult {
|
||||
for _, s := range p.Steps {
|
||||
if s.Module == module && s.Step == step {
|
||||
return s
|
||||
}
|
||||
}
|
||||
return seat.StepResult{}
|
||||
}
|
||||
|
||||
// setMaxMsgSize sets the largest message the stream takes (-1 for none): a key's write over it is refused by the bus.
|
||||
func setMaxMsgSize(t *testing.T, ctx context.Context, js jetstream.JetStream, stream string, size int32) {
|
||||
t.Helper()
|
||||
s, err := js.Stream(ctx, stream)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := s.CachedInfo().Config
|
||||
cfg.MaxMsgSize = size
|
||||
if _, err := js.UpdateStream(ctx, cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func pipelineKeys(t *testing.T, ctx context.Context, kv jetstream.KeyValue) []string {
|
||||
t.Helper()
|
||||
ks, err := kv.Keys(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
slices.Sort(ks)
|
||||
return ks
|
||||
}
|
||||
Reference in New Issue
Block a user