R17: the queue's whole test against the bus

The runner started whole with pipelines-at-once 1 and pipelines-waiting-cap 1: a second take-in
waits, saying wait-held, and is placed with wait-ended on the first's end; a newer waiting take-in
takes its shared movers out of an older waiting one, which ends superseded; a start over the cap
waits and raises queue-full, cleared on the count; a cancel places the next; a pipeline on its
approval alone gives up its place and, answered while it still runs, is keyed running at once; a
placing the bus refuses raises pipelines-stalled, cleared when the next start places the waiting
from the runner's own state, which starts no pipeline of its own. The tools runner_rerun,
runner_accept and runner_ask_again are shown to take the mover's module.

Builds: design 47 H §748, §774; acceptance test 112 (the runner's half); lead answers H-A-14, H-A-36, H-A-52, H-A-53, H-A-96, H-A-101, H-A-106, H-A-107

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
This commit is contained in:
2026-11-20 04:56:31 +01:00
parent 0548dc8f80
commit 2a93cbcd9b
3 changed files with 426 additions and 4 deletions
@@ -14,6 +14,8 @@ func TestRerunTool(t *testing.T) {
in seat.RerunInput
}{
{"asks the verb", "", seat.RerunInput{Pipeline: "main.981.node-tools", Step: "composition", Why: "the composer was late"}},
// H-A-101 (principle 4): the mover's module is named.
{"names the mover", "", seat.RerunInput{Pipeline: "main.981", Module: "node-tools", Step: "composition", Why: "the composer was late"}},
{"why missing", "names why", seat.RerunInput{Pipeline: "main.981.node-tools", Step: "composition"}},
{"step missing", "names the pipeline and the step", seat.RerunInput{Pipeline: "main.981.node-tools", Why: "x"}},
} {
@@ -24,6 +26,9 @@ func TestRerunTool(t *testing.T) {
asked, body = key, b.(seat.RerunInput)
return json.RawMessage(`{}`), nil
}
if !takes(RerunTool(ask).Input, "module") {
t.Fatal("runner_rerun's input names no module (H-A-101)")
}
_, err := rerun(ask, c.in)
if c.refusal != "" {
if err == nil || !strings.Contains(err.Error(), c.refusal) || asked != "" {
@@ -22,11 +22,12 @@ func TestVerbTools(t *testing.T) {
{"run", "runner_run", "seat:mesh-pipeline-runner.run", "", map[string]any{"members": []any{map[string]any{}}, "why": "x"},
seat.RunInput{Members: []seat.Member{{}}, Why: "x"}},
{"run no members", "runner_run", "", "members", map[string]any{}, nil},
{"accept", "runner_accept", "seat:mesh-pipeline-runner.accept", "", map[string]any{"pipeline": "p", "step": "lint", "artifact": "a", "why": "x"},
seat.AcceptInput{Pipeline: "p", Step: "lint", Artifact: "a", Why: "x"}},
// H-A-101 (principle 4): accept and ask-again name the mover's module.
{"accept", "runner_accept", "seat:mesh-pipeline-runner.accept", "", map[string]any{"pipeline": "p", "module": "lib", "step": "lint", "artifact": "a", "why": "x"},
seat.AcceptInput{Pipeline: "p", Module: "lib", Step: "lint", Artifact: "a", Why: "x"}},
{"accept no step", "runner_accept", "", "pipeline and the step", map[string]any{"pipeline": "p", "why": "x"}, nil},
{"ask-again", "runner_ask_again", "seat:mesh-pipeline-runner.ask-again", "", map[string]any{"pipeline": "p", "why": "x"},
seat.AskAgainInput{Pipeline: "p", Why: "x"}},
{"ask-again", "runner_ask_again", "seat:mesh-pipeline-runner.ask-again", "", map[string]any{"pipeline": "p", "module": "lib", "why": "x"},
seat.AskAgainInput{Pipeline: "p", Module: "lib", Why: "x"}},
// R14/AC2 (§748, H-A-53): runner_place asks the verb place with the pipeline and why.
{"place", "runner_place", "seat:mesh-pipeline-runner.place", "", map[string]any{"pipeline": "main.6", "why": "urgent"},
seat.PlaceInput{Pipeline: "main.6", Why: "urgent"}},
@@ -44,6 +45,10 @@ func TestVerbTools(t *testing.T) {
for _, tl := range VerbTools(ask) {
if tl.Name == c.tool {
run = tl.Run
// H-A-101: a tool asking a verb that names the mover's module offers module in its input.
if _, named := c.args["module"]; named && !takes(tl.Input, "module") {
t.Fatalf("%s's input names no module: %v", c.tool, tl.Input)
}
}
}
out, err := run(c.args)
@@ -61,3 +66,10 @@ func TestVerbTools(t *testing.T) {
})
}
}
// takes says whether a tool's input schema has the property name.
func takes(input map[string]any, name string) bool {
props, _ := input["properties"].(map[string]any)
_, ok := props[name]
return ok
}
@@ -0,0 +1,405 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/testbus"
"mesh-delivery-pipeline/internal/pipeline"
)
// R17 (design 47 H §748, §774; acceptance test 112's runner half; H-A-14, H-A-36, H-A-52, H-A-53, H-A-96, H-A-106,
// H-A-107): the queue whole against the real bus, the runner started as its process is with pipelines-at-once 1 (its
// default) and pipelines-waiting-cap 1. A second take-in is written waiting and says wait-held, and is placed with wait-ended
// on the first's pipeline-ended; a newer waiting take-in takes the movers it shares out of an older waiting one, which ends
// superseded once none is left; a start over the cap still waits and raises queue-full, cleared when an end brings the
// waiting back to the cap; a cancel places the next; a pipeline whose every open step is an approval asked gives up its
// place, and answered while it still runs is keyed running at once, over the bound, ahead of the waiting; a placing that
// fails leaves pipelines waiting with none running and raises pipelines-stalled; at the runner's next start the waiting
// one is placed from its own state, which clears it, and the start begins no pipeline of its own.
func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
url := testbus.URL(t)
nc, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer nc.Close()
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
defer cancel()
const module = "mesh-delivery-pipeline"
for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries", seat.StateQueue} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}); err != nil {
t.Fatal(err)
}
}
for _, state := range []string{"bounds", TestRunsState} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1, LimitMarkerTTL: time.Minute}); err != nil {
t.Fatal(err)
}
}
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil {
t.Fatal(err)
}
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}})
if err != nil {
t.Fatal(err)
}
if _, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: "n1_" + module, AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy}); err != nil {
t.Fatal(err)
}
// The seats the runner calls, answered as their holders do: four registered modules, gated's policy asking the
// operator's approval; main's half at revision 1 holds them all.
modules := []string{"app", "lib", "tool", "gated"}
var regs []catalog.Registration
var entries []catalog.CatalogEntry
for _, m := range modules {
steps := catalog.StepStatuses{Review: "bypass", Validate: "bypass", Composition: "bypass", Approval: "bypass"}
if m == "gated" {
steps.Approval = "required"
}
regs = append(regs, catalog.Registration{Module: m, Branches: []string{"main"}, Origin: "o/" + m, Policy: catalog.Policy{Steps: steps}})
entries = append(entries, catalog.CatalogEntry{Module: m, Commit: "c1", Fp: "src2:" + m})
}
answer := func(subject string, body any) {
if _, err := nc.Subscribe(subject, func(m *nats.Msg) {
raw, _ := json.Marshal(map[string]any{"result": body})
_ = m.Respond(raw)
}); err != nil {
t.Fatal(err)
}
}
answer("mesh.seat.mesh-catalog.call.registrations.*", catalog.RegistrationsAnswer{Registrations: regs})
answer("mesh.seat.mesh-catalog.call.revision.*", catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 1, Kind: "trunk", Branch: "main", Source: 2}, Entries: entries})
answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{})
// The router of the operator's asks holds the seat operator-channel; its record of each ask is in its state asks.
const router = "operator-router"
answer("mesh.seat."+seatlist.HolderSeat+".call."+seatlist.HolderVerb+".*", map[string]any{"seats": []any{
map[string]any{"seat": oc.Seat, "holders": []any{map[string]any{"module": router}}}}})
records, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: router + "_" + oc.StateAsks, History: 1})
if err != nil {
t.Fatal(err)
}
def, err := os.ReadFile("../../../mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json")
if err != nil {
t.Fatal(err)
}
if _, err := nc.Subscribe("mesh.seat.mesh-catalog.call."+seatlist.Verb+".*", func(m *nats.Msg) {
var in struct {
Seat string `json:"seat"`
}
_ = json.Unmarshal(m.Data, &in)
entry := seatlist.Entry{Seat: seat.Seat, Source: seatlist.SourceTrunk, Digest: "sha256:runner"}
var result any = seatlist.Listing{Revision: 1, Seats: []seatlist.Entry{entry}}
if in.Seat != "" {
entry.Definition = def
result = seatlist.Answer{Seat: in.Seat, Found: in.Seat == seat.Seat, Entry: &entry, Revision: 1}
}
raw, _ := json.Marshal(map[string]any{"result": result})
_ = m.Respond(raw)
}); err != nil {
t.Fatal(err)
}
// What the runner says per source revision: wait-held, wait-ended and pipeline-ended.
var saidMu sync.Mutex
var said []string
if _, err := nc.Subscribe("mesh.seat."+seat.Seat+".event.>", func(m *nats.Msg) {
saidMu.Lock()
said = append(said, strings.TrimPrefix(m.Subject, "mesh.seat."+seat.Seat+".event."))
saidMu.Unlock()
}); err != nil {
t.Fatal(err)
}
wasSaid := func(s string) bool { saidMu.Lock(); defer saidMu.Unlock(); return slices.Contains(said, s) }
dir := t.TempDir()
broker := filepath.Join(dir, "broker")
raw, _ := json.Marshal(process.Account{URL: url, Node: "n1", Module: module, User: "runner", Password: "x"})
if err := os.WriteFile(broker, raw, 0o600); err != nil {
t.Fatal(err)
}
waitingCap := filepath.Join(dir, "pipelines-waiting-cap")
if err := os.WriteFile(waitingCap, []byte("1\n"), 0o600); err != nil {
t.Fatal(err)
}
logs := &syncBuffer{}
start := func() func() {
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
go func() {
done <- run(runCtx, func(k string) string {
switch k {
case process.BrokerFileEnv:
return broker
case envPipelinesWaitingCap:
return waitingCap
}
return ""
}, logs)
}()
return func() { stop(); <-done }
}
stopRunner := start()
defer func() { stopRunner() }()
pipelines, err := js.KeyValue(ctx, module+"_pipelines")
if err != nil {
t.Fatal(err)
}
queue, err := js.KeyValue(ctx, module+"_"+seat.StateQueue)
if err != nil {
t.Fatal(err)
}
findings, err := js.KeyValue(ctx, module+"_findings")
if err != nil {
t.Fatal(err)
}
get := func(key string) seat.Pipeline {
var p seat.Pipeline
readPipeline(ctx, pipelines, key, &p)
return p
}
keyed := func(prefix string) []string {
ks, err := queue.ListKeysFiltered(ctx, prefix+">")
if err != nil {
return nil
}
var out []string
for k := range ks.Keys() {
out = append(out, k)
}
slices.Sort(out)
return out
}
running := func() []string {
var out []string
for _, k := range keyed("running.") {
out = append(out, strings.TrimPrefix(k, "running."))
}
return out
}
finding := func(kind string) (string, bool) {
e, err := findings.Get(ctx, "delivery."+kind+"."+kind)
if err != nil {
return "", false
}
return string(e.Value()), true
}
take := func(source int64, movers ...string) {
t.Helper()
ev := catalog.TakeInTaken{Branch: "main", Source: source, Revision: 1, Call: fmt.Sprintf("%016x", source)}
for _, m := range movers {
ev.Entries = append(ev.Entries, catalog.TakeInEntry{Module: m, Commit: "c1", Fingerprint: "src2:" + m})
}
body, _ := json.Marshal(ev)
if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", body); err != nil {
t.Fatal(err)
}
key := pipeline.Key("main", source)
waitFor(t, ctx, logs, func() bool { var p seat.Pipeline; return readPipeline(ctx, pipelines, key, &p) })
}
// built says the build of mover m in pipeline key ended passed, as the seat mesh-builds' holder says it.
built := func(key, m string) {
t.Helper()
var b seat.StepResult
waitFor(t, ctx, logs, func() bool {
b = stepFor(get(key), m, seat.StepResultStepBuild)
return b.Outcome == seat.StepResultOutcomeRunning && b.Ask != ""
})
ended, _ := json.Marshal(builds.StepEnded{ID: b.Ask, Pipeline: key, Step: "build", Node: "n1", Outcome: "passed",
Entries: []builds.BuildEntry{{Module: m, Commit: "c1", Fingerprint: "src2:" + m}}})
if _, err := js.Publish(ctx, "mesh.seat.mesh-builds.event.step-ended.n1", ended); err != nil {
t.Fatal(err)
}
}
verb := func(name string, in any) error {
t.Helper()
var reply struct {
Error string `json:"error"`
}
waitFor(t, ctx, logs, func() bool {
m, err := nc.Request("mesh.seat."+seat.Seat+".call."+name+".agent~dev", mustJSON(in), 15*time.Second)
reply.Error = ""
return err == nil && json.Unmarshal(m.Data, &reply) == nil && !strings.Contains(reply.Error, "not yet read")
})
if reply.Error != "" {
return errors.New(reply.Error)
}
return nil
}
outcome := func(key, want string) {
t.Helper()
waitFor(t, ctx, logs, func() bool { return get(key).Outcome == want })
}
// Two take-ins in a row: the first runs, its build asked; the second is written waiting under sequence 1 with no step,
// says pipeline-started and wait-held.
take(3, "app")
outcome("main.3", seat.PipelineOutcomeRunning)
take(4, "lib")
if p := get("main.4"); p.Outcome != seat.PipelineOutcomeWaiting || p.Seq != 1 || len(p.Steps) != 0 {
t.Fatalf("main.4 = %s seq %d with %d steps, want waiting at 1 with none", p.Outcome, p.Seq, len(p.Steps))
}
waitFor(t, ctx, logs, func() bool { return wasSaid("wait-held.main.4") && wasSaid("pipeline-started.main.4") })
if r := running(); !slices.Equal(r, []string{"main.3"}) {
t.Fatalf("running %v, want main.3 alone", r)
}
// The first's pipeline-ended places the second, which says wait-ended and has its build asked.
built("main.3", "app")
outcome("main.3", seat.PipelineOutcomePassed)
outcome("main.4", seat.PipelineOutcomeRunning)
waitFor(t, ctx, logs, func() bool { return wasSaid("wait-ended.main.4") })
waitFor(t, ctx, logs, func() bool { return stepFor(get("main.4"), "lib", seat.StepResultStepBuild).Ask != "" })
// A newer waiting take-in takes app out of the older waiting main.5; over the cap of 1 it still waits and raises
// queue-full naming it. main.7 takes tool, main.5's last mover, so main.5 ends superseded and leaves the queue.
take(5, "tool", "app")
take(6, "app")
waitFor(t, ctx, logs, func() bool { f, ok := finding(pipeline.QueueFullKind); return ok && strings.Contains(f, "main.6") })
if p := get("main.5"); p.Outcome != seat.PipelineOutcomeWaiting || len(p.Movers) != 1 || p.Movers[0].Module != "tool" || !slices.Equal(p.Superseded, []string{"app"}) {
t.Fatalf("main.5 = %s moving %+v, superseded %v; want waiting with tool alone, app superseded", p.Outcome, p.Movers, p.Superseded)
}
take(7, "tool")
outcome("main.5", seat.PipelineOutcomeSuperseded)
waitFor(t, ctx, logs, func() bool { return wasSaid("pipeline-ended.main.5") && len(keyed("waiting.")) == 2 })
if p := get("main.4"); p.Outcome != seat.PipelineOutcomeRunning {
t.Fatalf("the running main.4 = %s, want it running to its end", p.Outcome)
}
// A cancel places the next, the lowest sequence; one waiting is back within the cap, so queue-full is cleared.
if err := verb(seat.VerbCancel, seat.CancelInput{Pipeline: "main.4", Why: "the queue's whole test"}); err != nil {
t.Fatal(err)
}
outcome("main.4", seat.PipelineOutcomeCancelled)
outcome("main.6", seat.PipelineOutcomeRunning)
waitFor(t, ctx, logs, func() bool { _, ok := finding(pipeline.QueueFullKind); return !ok })
if p := get("main.7"); p.Outcome != seat.PipelineOutcomeWaiting {
t.Fatalf("main.7 = %s, want waiting behind main.6", p.Outcome)
}
// gated waits, then runs; once its build passed its every open step is the approval asked of the operator: it gives up
// its place and main.9 is placed.
built("main.6", "app")
outcome("main.7", seat.PipelineOutcomeRunning)
take(8, "gated")
built("main.7", "tool")
outcome("main.8", seat.PipelineOutcomeRunning)
take(9, "lib")
built("main.8", "gated")
outcome("main.9", seat.PipelineOutcomeRunning)
approval := stepFor(get("main.8"), "gated", seat.StepResultStepApproval)
if p := get("main.8"); p.Outcome != seat.PipelineOutcomeRunning || approval.Outcome != seat.StepResultOutcomeRunning || approval.Ask == "" {
t.Fatalf("main.8 = %s, its approval %+v; want running on its approval asked", p.Outcome, approval)
}
waitFor(t, ctx, logs, func() bool { return slices.Equal(running(), []string{"main.9"}) })
// gated's build is asked again, its mover named (H-A-101), and then its approval answered: still running, main.8 is
// keyed running at once beside main.9, over the bound, while main.10 waits.
if err := verb(seat.VerbRerun, seat.RerunInput{Pipeline: "main.8", Module: "gated", Step: "build", Why: "the queue's whole test"}); err != nil {
t.Fatal(err)
}
take(10, "app")
warrant := oc.Warrant{About: pipeline.About("gated", "main"), Ask: approval.Ask, Asker: module, Outcome: oc.WarrantOutcomeExpired,
Words: "the queue's whole test", At: time.Now().UTC().Format(time.RFC3339)}
if _, err := records.Put(ctx, module+"."+approval.Ask, mustJSON(map[string]any{"state": "decided", "warrant": warrant})); err != nil {
t.Fatal(err)
}
if _, err := js.Publish(ctx, oc.SubjectDecided(module), mustJSON(warrant)); err != nil {
t.Fatal(err)
}
waitFor(t, ctx, logs, func() bool { return slices.Equal(running(), []string{"main.8", "main.9"}) })
if p := get("main.10"); p.Outcome != seat.PipelineOutcomeWaiting {
t.Fatalf("main.10 = %s, want waiting while two run", p.Outcome)
}
// main.8's end (its approval declined) places nothing while main.9 runs.
built("main.8", "gated")
outcome("main.8", seat.PipelineOutcomeFailed)
waitFor(t, ctx, logs, func() bool { return slices.Equal(running(), []string{"main.9"}) })
// The bus refuses the queue's next running key, so main.9's end places nothing: main.10 waits with none running, and
// pipelines-stalled names it.
setMaxMsgSize(t, ctx, js, "KV_"+module+"_"+seat.StateQueue, 40)
built("main.9", "lib")
outcome("main.9", seat.PipelineOutcomePassed)
waitFor(t, ctx, logs, func() bool {
f, ok := finding(pipeline.PipelinesStalledKind)
return ok && strings.Contains(f, "main.10")
})
if p, r := get("main.10"), running(); p.Outcome != seat.PipelineOutcomeWaiting || len(r) != 0 {
t.Fatalf("main.10 = %s with %v running, want it waiting and none running", p.Outcome, r)
}
// At the runner's next start the waiting main.10 is placed from its own state, saying wait-ended, which clears
// pipelines-stalled; the start begins no pipeline of its own.
stopRunner()
setMaxMsgSize(t, ctx, js, "KV_"+module+"_"+seat.StateQueue, -1)
before := pipelineKeys(t, ctx, pipelines)
stopRunner = start()
outcome("main.10", seat.PipelineOutcomeRunning)
waitFor(t, ctx, logs, func() bool { return wasSaid("wait-ended.main.10") && slices.Equal(running(), []string{"main.10"}) })
waitFor(t, ctx, logs, func() bool { _, ok := finding(pipeline.PipelinesStalledKind); return !ok })
if after := pipelineKeys(t, ctx, pipelines); !slices.Equal(before, after) {
t.Fatalf("the start wrote pipelines %v, want %v", after, before)
}
}
func stepFor(p seat.Pipeline, module, step string) seat.StepResult {
for _, s := range p.Steps {
if s.Module == module && s.Step == step {
return s
}
}
return seat.StepResult{}
}
// setMaxMsgSize sets the largest message the stream takes (-1 for none): a key's write over it is refused by the bus.
func setMaxMsgSize(t *testing.T, ctx context.Context, js jetstream.JetStream, stream string, size int32) {
t.Helper()
s, err := js.Stream(ctx, stream)
if err != nil {
t.Fatal(err)
}
cfg := s.CachedInfo().Config
cfg.MaxMsgSize = size
if _, err := js.UpdateStream(ctx, cfg); err != nil {
t.Fatal(err)
}
}
func pipelineKeys(t *testing.T, ctx context.Context, kv jetstream.KeyValue) []string {
t.Helper()
ks, err := kv.Keys(ctx)
if err != nil {
t.Fatal(err)
}
slices.Sort(ks)
return ks
}