diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go new file mode 100644 index 00000000..6ec49142 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go @@ -0,0 +1,536 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" + catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/seatlist" + "git.novox.be/novox/mesh-sdk/go/testbus" + "git.novox.be/novox/mesh-sdk/go/views" + "git.novox.be/novox/mesh-sdk/go/views/viewstest" + + "mesh-delivery-pipeline/internal/pipeline" +) + +// Test 109 (design 47 G §733 *Whole (runner…)*, §724 to §728, §535 and §647 as A-466y and A-466z amend them; A-468f: in +// mesh-delivery with a fixture build seat holder; A-470f: no reader-only refusal). The runner whole against the real bus: a +// module whose program and tools bundle test one directory asks one unit and one integration run, both artifacts' steps +// ending with its result and shared; a failing test fails once and is counted once; a rerun of one artifact's failed step +// that passes moves the other's, the pipeline ending again, and an accepted step does not move; a runner restarted during a +// run takes it up by its ask, its clock holding that ask (A-470n); a test run waiting with a holder present is not failed; +// a replaced reader leaves the ask to the other readers; a run that keeps saying its progress past its bound is not ended, +// and a taken ask silent for its bound ends could not run with its readers, its ask dropped; a second revision with the +// same digests asks its own runs; a seat the seat-holders view names no holder for ends its pipeline failed at once, its +// other steps not asked (A-470g, A-470h); the process answers test-runs with its fill against test-runs-cap-mib and +// test-runs-drop deletes the key and its output object (A-470i); at the cap a step ends could not run, naming the fill. +// The step bound is two seconds here, as the setting test-step-bound names it, where the mesh's is 30 minutes. +func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { + url := testbus.URL(t) + nc, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + const module = "mesh-delivery-pipeline" + for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries"} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}); err != nil { + t.Fatal(err) + } + } + for _, state := range []string{"bounds", TestRunsState} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1, LimitMarkerTTL: time.Minute}); err != nil { + t.Fatal(err) + } + } + // test-runs' own object store, made with the state (A-469e). + if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: catalogue.BusName(module, catalogue.StateObjectsName(TestRunsState))}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil { + t.Fatal(err) + } + events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>", "mesh.stream.*.event.>"}}) + if err != nil { + t.Fatal(err) + } + if _, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: "n1_" + module, AckPolicy: jetstream.AckExplicitPolicy}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy}); err != nil { + t.Fatal(err) + } + + // The seat-holders view of the seat mesh-module-assigner's claimant (§647, A-466z), served as its owner does: node-build-go + // has a holder, node-build-node none. + holders := viewstest.Serve(t, ctx, js, nc, "assigner", "seat-holders", "assigner.seat-holders", "n1_"+module) + if _, err := holders.Put(ctx, "node-build-go", []byte(`{"holders":[{"module":"mesh-build-go"}]}`)); err != nil { + t.Fatal(err) + } + + // The registrations: app tests its program and tools bundle from one directory, its unit step required and its + // integration step optional; web is built by node-build-node. + policy := catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "required", Integration: "optional", Composition: "bypass", Review: "bypass", Approval: "bypass"}} + regs := []catalog.Registration{{Module: "app", Branches: []string{"main"}, Origin: "o/app", Policy: policy}, + {Module: "web", Branches: []string{"main"}, Origin: "o/web", Policy: policy}} + answer := func(subject string, body any) { + if _, err := nc.Subscribe(subject, func(m *nats.Msg) { + raw, _ := json.Marshal(map[string]any{"result": body}) + _ = m.Respond(raw) + }); err != nil { + t.Fatal(err) + } + } + answer("mesh.seat.mesh-catalog.call.registrations.*", catalog.RegistrationsAnswer{Registrations: regs}) + answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{}) + entries := []catalog.CatalogEntry{ + {Module: "app", Commit: "c1", Fp: "src2:aa", Path: "modules/app", Repository: "o/app", + Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, + {Module: "web", Commit: "w1", Fp: "src2:ww", Path: "modules/web", Repository: "o/web", Artifacts: []catalog.Artifact{{Ref: "web:program", Language: "node"}}}, + {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, + {Module: "mesh-build-node", Acts: []catalog.ActingRule{{Seat: "node-build-node", Step: "build", On: "node"}}}, + {Module: "assigner", Manifest: map[string]any{"module": "assigner", "version": "0.1", "claims": []any{map[string]any{"name": seatlist.HolderSeat, "scope": "mesh"}}}}, + } + var revision atomic.Int64 + revision.Store(1) + if _, err := nc.Subscribe("mesh.seat.mesh-catalog.call.revision.*", func(m *nats.Msg) { + r := revision.Load() + raw, _ := json.Marshal(map[string]any{"result": catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: r, Kind: "trunk", Branch: "main", Source: 2 + r}, Entries: entries}}) + _ = m.Respond(raw) + }); err != nil { + t.Fatal(err) + } + def, err := os.ReadFile("../../../mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json") + if err != nil { + t.Fatal(err) + } + if _, err := nc.Subscribe("mesh.seat.mesh-catalog.call."+seatlist.Verb+".*", func(m *nats.Msg) { + var in struct { + Seat string `json:"seat"` + } + _ = json.Unmarshal(m.Data, &in) + entry := seatlist.Entry{Seat: seat.Seat, Source: seatlist.SourceTrunk, Digest: "sha256:runner"} + var result any = seatlist.Listing{Revision: 1, Seats: []seatlist.Entry{entry}} + if in.Seat != "" { + entry.Definition = def + result = seatlist.Answer{Seat: in.Seat, Found: in.Seat == seat.Seat, Entry: &entry, Revision: 1} + } + raw, _ := json.Marshal(map[string]any{"result": result}) + _ = m.Respond(raw) + }); err != nil { + t.Fatal(err) + } + // Every pipeline-ended said, and every drop asked of a build seat. + var mu sync.Mutex + var endings []seat.PipelineEnded + var drops []string + if _, err := nc.Subscribe("mesh.seat.mesh-pipeline-runner.event.pipeline-ended.>", func(m *nats.Msg) { + var e seat.PipelineEnded + if json.Unmarshal(m.Data, &e) == nil { + mu.Lock() + endings = append(endings, e) + mu.Unlock() + } + }); err != nil { + t.Fatal(err) + } + if _, err := nc.Subscribe("mesh.seat.*.call.drop.>", func(m *nats.Msg) { + mu.Lock() + drops = append(drops, string(m.Data)) + mu.Unlock() + raw, _ := json.Marshal(map[string]any{"result": map[string]any{}}) + _ = m.Respond(raw) + }); err != nil { + t.Fatal(err) + } + ended := func(key string) []seat.PipelineEnded { + mu.Lock() + defer mu.Unlock() + var out []seat.PipelineEnded + for _, e := range endings { + if e.Pipeline == key { + out = append(out, e) + } + } + return out + } + + // The settings, as the manifest's resources write them (A-469x): a cap of 1 MiB, a step bound of two seconds. + dir := t.TempDir() + files := map[string]string{envRunsCap: "1", envStepBound: "2s"} + env := map[string]string{} + for name, content := range files { + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + env[name] = path + } + broker := filepath.Join(dir, "broker") + raw, _ := json.Marshal(process.Account{URL: url, Node: "n1", Module: module, User: "runner", Password: "x"}) + if err := os.WriteFile(broker, raw, 0o600); err != nil { + t.Fatal(err) + } + env[process.BrokerFileEnv] = broker + logs := &syncBuffer{} + start := func() func() { + runCtx, stop := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { done <- run(runCtx, func(k string) string { return env[k] }, logs) }() + return func() { stop(); <-done } + } + stop := start() + defer func() { stop() }() + + pipelines, err := js.KeyValue(ctx, module+"_pipelines") + if err != nil { + t.Fatal(err) + } + runsKV, err := js.KeyValue(ctx, module+"_"+TestRunsState) + if err != nil { + t.Fatal(err) + } + firstViews, err := js.KeyValue(ctx, module+"_"+views.StatusState) + if err != nil { + t.Fatal(err) + } + // The seat-holders view's first view is read before any pipeline is placed, so a seat is judged from it (A-466y). + waitFor(t, ctx, logs, func() bool { + e, err := firstViews.Get(ctx, views.StatusKey("assigner", "seat-holders")) + var s views.Status + return err == nil && json.Unmarshal(e.Value(), &s) == nil && !s.Answered.IsZero() + }) + + takeIn := func(module, commit, fp string, source, rev int64) { + taken, _ := json.Marshal(catalog.TakeInTaken{Key: "m." + commit, Kind: "taken", Branch: "main", Source: source, Revision: rev, + Entries: []catalog.TakeInEntry{{Module: module, Commit: commit, Fingerprint: fp}}}) + if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", taken); err != nil { + t.Fatal(err) + } + } + placed := func(key string) seat.Pipeline { + var p seat.Pipeline + waitFor(t, ctx, logs, func() bool { return readPipeline(ctx, pipelines, key, &p) && len(p.Steps) > 1 }) + return p + } + read := func(key string) seat.Pipeline { + var p seat.Pipeline + readPipeline(ctx, pipelines, key, &p) + return p + } + step := func(p seat.Pipeline, artifact, name string) seat.StepResult { + for _, s := range p.Steps { + if s.Artifact == artifact && s.Step == name { + return s + } + } + return seat.StepResult{} + } + runOf := func(key string) (pipeline.Run, bool) { + e, err := runsKV.Get(ctx, key) + var r pipeline.Run + return r, err == nil && json.Unmarshal(e.Value(), &r) == nil + } + runKeys := func(candidate string) []string { + ks, _ := runsKV.Keys(ctx) + var out []string + for _, k := range ks { + if !pipeline.IsRunBound(k) && strings.HasPrefix(k, candidate+".") { + out = append(out, k) + } + } + return out + } + runOfStep := func(key, artifact, name string) (string, pipeline.Run) { + var run string + var r pipeline.Run + waitFor(t, ctx, logs, func() bool { + s := step(read(key), artifact, name) + if len(s.Shared) == 0 { + return false + } + run = s.Shared[0].Run + got, ok := runOf(run) + r = got + return ok && r.Ask != "" + }) + return run, r + } + publish := func(subject string, body any) { + raw, _ := json.Marshal(body) + if _, err := js.Publish(ctx, subject, raw); err != nil { + t.Fatal(err) + } + } + taken := func(ask string) { + publish(nodebuildgo.SubjectAskTaken("n1"), nodebuildgo.AskTaken{Ask: ask, Asker: module, Node: "n1", At: time.Now()}) + } + endAsk := func(key, ask, name, outcome string, tests *nodebuildgo.TestCounts) { + now := time.Now() + publish(nodebuildgo.SubjectStepEnded("n1"), nodebuildgo.StepEnded{ID: ask, Ask: ask, Asker: module, Pipeline: key, Module: "app", Commit: "c1", + Fingerprint: "src2:aa", Step: name, Node: "n1", Outcome: outcome, Tests: tests, At: now, Ended: &now}) + } + passBuild := func(key, module, commit, fp string) { + ask := step(read(key), "", seat.StepResultStepBuild).Ask + publish("mesh.seat.mesh-builds.event.step-ended.n1", builds.StepEnded{ID: ask, Pipeline: key, Step: "build", Node: "n1", Outcome: "passed", + Entries: []builds.BuildEntry{{Module: module, Commit: commit, Fingerprint: fp}}}) + } + call := func(verb string, in, out any) string { + var reply struct { + Result json.RawMessage `json:"result"` + Error string `json:"error"` + } + waitFor(t, ctx, logs, func() bool { + m, err := nc.Request("mesh.seat.mesh-pipeline-runner.call."+verb+".agent~dev", mustJSON(in), 2*time.Second) + reply.Error = "" + return err == nil && json.Unmarshal(m.Data, &reply) == nil && !strings.Contains(reply.Error, "not yet read") + }) + if out != nil && reply.Error == "" { + _ = json.Unmarshal(reply.Result, out) + } + return reply.Error + } + + // One unit run and one integration run for both artifacts (§724), each read by both, waiting for a build node until its + // ask is taken (A-469b), its holder present and nothing failing it (A-466y). + const p3 = "main.3.app" + takeIn("app", "c1", "src2:aa", 3, 1) + passBuild(p3, "app", "c1", "src2:aa") + unitRun, unit := runOfStep(p3, "app:program", "unit") + integRun, integ := runOfStep(p3, "app:program", "integration") + if got := runKeys("main.1"); len(got) != 2 || len(unit.Readers) != 2 || len(integ.Readers) != 2 { + t.Fatalf("runs %v, unit readers %+v, integration readers %+v; want one run per level read by both artifacts", got, unit.Readers, integ.Readers) + } + if s := step(read(p3), "app:tools", "unit"); s.Outcome != seat.StepResultOutcomeWaitingForABuildNode || s.Ask != "" || s.Shared[0].Ask != unit.Ask { + t.Fatalf("the tools' unit step = %+v; want it waiting for a build node, reading the program's run", s) + } + + // A runner restarted during a run takes it up again by its ask (§725): the run's end, heard after the restart, ends both. + taken(unit.Ask) + waitFor(t, ctx, logs, func() bool { r, _ := runOf(unitRun); return r.Node == "n1" }) + // Its clock holds the run's ask (A-470n). + waitFor(t, ctx, logs, func() bool { + e, err := runsKV.Get(ctx, pipeline.BoundPrefix+unitRun) + return err == nil && strings.Contains(string(e.Value()), `"ask":"`+unit.Ask+`"`) + }) + stop() + stop = start() + // It keeps saying its progress past its bound and is not ended (§725, §720). + // Each progress is heard before the next is said: the run's clock is written again (its bound key's revision rises). + boundRev := func() uint64 { + e, err := runsKV.Get(ctx, pipeline.BoundPrefix+unitRun) + if err != nil { + return 0 + } + return e.Revision() + } + for at := time.Now(); time.Since(at) < 3*time.Second; { + before := boundRev() + publish(nodebuildgo.SubjectStepProgress("n1"), nodebuildgo.StepProgress{Ask: unit.Ask, Asker: module, Phase: "run", At: time.Now()}) + waitFor(t, ctx, logs, func() bool { return boundRev() > before }) + } + if r, _ := runOf(unitRun); r.Ended != nil { + t.Fatalf("a run saying its progress past its bound ended: %+v", r) + } + // A failing test fails once and is counted once (§726). + endAsk(p3, unit.Ask, "unit", "failed", &nodebuildgo.TestCounts{Unit: 5}) + taken(integ.Ask) + endAsk(p3, integ.Ask, "integration", "failed", &nodebuildgo.TestCounts{Integration: 2}) + waitFor(t, ctx, logs, func() bool { return len(ended(p3)) == 1 }) + p := read(p3) + for _, a := range []string{"app:program", "app:tools"} { + if s := step(p, a, "unit"); s.Outcome != seat.StepResultOutcomeFailed || len(s.Shared) == 0 || s.Shared[0].Run != unitRun { + t.Errorf("%s's unit step = %+v; want failed with the run in shared", a, s) + } + } + for _, k := range []string{unitRun, integRun} { + if r, _ := runOf(k); r.Ended == nil || r.Outcome != seat.StepResultOutcomeFailed { + t.Errorf("the run %s at its end = %+v; want it ended failed", k, r) + } + } + if e := ended(p3)[0]; e.Outcome != seat.PipelineEndedOutcomeFailed || e.Tests == nil || e.Tests.Unit != 5 || e.Tests.Integration != 2 { + t.Fatalf("pipeline-ended = %+v; want failed, the tests counted once", e) + } + + // An accepted step does not move; a rerun of the program's unit step that passes moves the tools' to passed, and the + // pipeline says pipeline-ended again (§727, A-470d). + if e := call(seat.VerbAccept, seat.AcceptInput{Pipeline: p3, Step: "integration", Artifact: "app:program", Why: "flaky"}, nil); e != "" { + t.Fatalf("accept: %s", e) + } + if e := call(seat.VerbRerun, seat.RerunInput{Pipeline: p3, Step: "unit", Artifact: "app:program", Why: "fixed"}, nil); e != "" { + t.Fatalf("rerun: %s", e) + } + var again pipeline.Run + waitFor(t, ctx, logs, func() bool { r, ok := runOf(unitRun); again = r; return ok && r.Ended == nil && r.Ask != unit.Ask }) + taken(again.Ask) + endAsk(p3, again.Ask, "unit", "passed", &nodebuildgo.TestCounts{Unit: 5}) + if e := call(seat.VerbRerun, seat.RerunInput{Pipeline: p3, Step: "integration", Artifact: "app:tools", Why: "fixed"}, nil); e != "" { + t.Fatalf("rerun: %s", e) + } + var integAgain pipeline.Run + waitFor(t, ctx, logs, func() bool { + r, ok := runOf(integRun) + integAgain = r + return ok && r.Ended == nil && r.Ask != integ.Ask + }) + taken(integAgain.Ask) + endAsk(p3, integAgain.Ask, "integration", "passed", nil) + waitFor(t, ctx, logs, func() bool { + e := ended(p3) + return len(e) >= 2 && e[len(e)-1].Outcome == seat.PipelineEndedOutcomePassed + }) + p = read(p3) + if s := step(p, "app:tools", "unit"); s.Outcome != seat.StepResultOutcomePassed || !s.Rerun { + t.Errorf("the tools' unit step = %+v; want moved to passed, marked a rerun", s) + } + if s := step(p, "app:program", "integration"); s.Outcome != seat.StepResultOutcomeAccepted { + t.Errorf("the accepted step = %+v; want it accepted still", s) + } + for _, k := range []string{unitRun, integRun} { + if r, _ := runOf(k); r.Ended == nil || r.Outcome != seat.StepResultOutcomePassed { + t.Errorf("the run %s after its rerun = %+v; want it ended passed", k, r) + } + } + if e := ended(p3); e[len(e)-1].EndCount <= e[0].EndCount { + t.Errorf("pipeline-ended counts %d then %d; want a higher end count", e[0].EndCount, e[len(e)-1].EndCount) + } + + // A second revision with the same digests asks its own runs, so a failed test runs again (§724). + revision.Store(2) + publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 2, Kind: "trunk", Branch: "main", Source: 4}) + const p4, p5 = "main.4.app", "main.5.app" + takeIn("app", "c1", "src2:aa", 4, 2) + placed(p4) + run4, r4 := runOfStep(p4, "app:program", "integration") + if !strings.HasPrefix(run4, "main.2.") || r4.Ask == integ.Ask { + t.Fatalf("revision 2's integration run %s (%s); want its own", run4, r4.Ask) + } + // A replaced reader leaves the ask to the other readers (§726): a second pipeline of the same candidate and digest reads + // the run, the first is cancelled and leaves it, and the run's ask is not dropped. + takeIn("app", "c1", "src2:aa", 5, 2) + placed(p5) + waitFor(t, ctx, logs, func() bool { r, _ := runOf(run4); return len(r.Readers) == 4 }) + if e := call(seat.VerbCancel, seat.CancelInput{Pipeline: p4, Why: "replaced"}, nil); e != "" { + t.Fatalf("cancel: %s", e) + } + waitFor(t, ctx, logs, func() bool { + r, _ := runOf(run4) + return len(r.Readers) == 2 && r.Readers[0].Pipeline == p5 && r.Readers[1].Pipeline == p5 + }) + if r, _ := runOf(run4); r.Ended != nil || r.Ask != r4.Ask { + t.Fatalf("the run of the replaced readers = %+v; want it running under its ask", r) + } + mu.Lock() + dropped := append([]string(nil), drops...) + mu.Unlock() + if len(dropped) != 0 { + t.Fatalf("drops asked %v; want none while a reader is left\n%s", dropped, logs.String()) + } + // A taken ask silent for its bound ends could not run with its readers, its ask dropped (§725, §720). + taken(r4.Ask) + waitFor(t, ctx, logs, func() bool { + return step(read(p5), "app:tools", "integration").Outcome == seat.StepResultOutcomeCouldNotRun && + step(read(p5), "app:program", "integration").Outcome == seat.StepResultOutcomeCouldNotRun + }) + if r, _ := runOf(run4); r.Ended == nil || r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, "test-step-bound") { + t.Errorf("the silent run = %+v; want it ended could not run, naming its bound", r) + } + waitFor(t, ctx, logs, func() bool { + mu.Lock() + defer mu.Unlock() + return len(drops) == 1 && strings.Contains(drops[0], r4.Ask) + }) + + // A seat the seat-holders view names no holder for ends the pipeline failed at once, naming it, its other steps not + // asked (§535, §647; A-470g, A-470h). + const web = "main.5.web" + takeIn("web", "w1", "src2:ww", 5, 2) + waitFor(t, ctx, logs, func() bool { return len(ended(web)) == 1 }) + if e := ended(web)[0]; e.Outcome != seat.PipelineEndedOutcomeFailed || e.Why != "the seat node-build-node has no holder: the seat-holders view names none" { + t.Fatalf("pipeline-ended = %+v; want failed at once naming the seat", e) + } + // The steps of the seat with no holder end could not run, a bypassed step stays bypassed, every other ends not asked, + // and none is asked (A-470h). + for _, s := range read(web).Steps { + want := seat.StepResultOutcomeNotAsked + switch { + case s.Status == seat.StepResultStatusBypass: + want = seat.StepResultOutcomeBypassed + case s.Seat == "node-build-node": + want = seat.StepResultOutcomeCouldNotRun + } + if s.Outcome != want || s.Ask != "" { + t.Errorf("the step %s %s of the pipeline with no holder = %s (ask %q); want %s", s.Artifact, s.Step, s.Outcome, s.Ask, want) + } + } + + // test-runs is answered by the process, its fill against test-runs-cap-mib (A-470i), each run's packages from its output + // object; test-runs-drop deletes the key and its object (§728). + var runs seat.TestRunsAnswer + if e := call(seat.VerbTestRuns, seat.TestRunsInput{Candidate: "main.1"}, &runs); e != "" || len(runs.Runs) != 2 || runs.Fill.Cap != 1<<20 || runs.Fill.Store != TestRunsState { + t.Fatalf("test-runs = %+v (%s); want main.1's two runs, the fill against 1 MiB", runs, e) + } + objects, err := js.ObjectStore(ctx, catalogue.BusName(module, catalogue.StateObjectsName(TestRunsState))) + if err != nil { + t.Fatal(err) + } + if _, err := objects.GetInfo(ctx, unitRun); err != nil { + t.Fatalf("the unit run's output object: %v", err) + } + var drop seat.TestRunsDropAnswer + if e := call(seat.VerbTestRunsDrop, seat.TestRunsDropInput{Key: unitRun, Why: "test 109"}, &drop); e != "" || drop.Run.Key != unitRun { + t.Fatalf("test-runs-drop = %+v (%s)", drop, e) + } + if _, ok := runOf(unitRun); ok { + t.Error("the dropped run's key is kept") + } + if _, err := objects.GetInfo(ctx, unitRun); err == nil { + t.Error("the dropped run's output object is kept") + } + + // At the cap a step ends could not run, naming the fill (§725, A-469d). + pad := make([]byte, 600<<10) + for _, k := range []string{"pad.a", "pad.b"} { + if _, err := runsKV.Put(ctx, k, pad); err != nil { + t.Fatal(err) + } + } + revision.Store(3) + publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 3, Kind: "trunk", Branch: "main", Source: 5}) + const p6 = "main.6.app" + takeIn("app", "c1", "src2:aa", 6, 3) + waitFor(t, ctx, logs, func() bool { + s := step(read(p6), "app:program", "unit") + return s.Outcome == seat.StepResultOutcomeCouldNotRun && strings.Contains(s.Why, "test-runs-cap-mib") + }) +} + +func mustJSON(v any) []byte { + raw, _ := json.Marshal(v) + return raw +}