diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go index eb5f47fb..9523c3ce 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go @@ -46,8 +46,8 @@ func as[T any](args map[string]any) (T, error) { return in, nil } -// Tools are the runner's read verbs over its states pipelines and kept. -func Tools(r pipeline.Reads) []stdio.Tool { +// Tools are the runner's read verbs over its states pipelines, kept and test-runs. +func Tools(r pipeline.Reads, testRuns ...pipeline.RunsRead) []stdio.Tool { runs := map[string]func(map[string]any) (any, error){ seat.VerbShow: func(a map[string]any) (any, error) { in, err := as[seat.ShowInput](a) @@ -79,6 +79,15 @@ func Tools(r pipeline.Reads) []stdio.Tool { return r.KeptResults(in) }, } + for _, t := range testRuns { + runs[seat.VerbTestRuns] = func(a map[string]any) (any, error) { + in, err := as[seat.TestRunsInput](a) + if err != nil { + return nil, err + } + return t.TestRuns(in) + } + } var out []stdio.Tool for _, v := range seat.Verbs() { if run, ok := runs[v]; ok { @@ -148,10 +157,12 @@ func main() { func(n string) (buscalls.Fill, error) { return busstate.ObjectsOf(n).Fill() }, func(n string) (*stdio.LogInfo, error) { return stdio.Log(n).Info() }, time.Now) r := pipeline.Reads{Pipelines: kept{stdio.State(seat.StatePipelines)}, Kept: kept{stdio.State(seat.StateKept)}, Fill: fill} - tools := append(Tools(r), FirstViewTools(os.Getenv(stdio.ServedAs), stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...) + runs := RunsRead(stdio.State(testRunsState), stdio.State(testRunsState).Info, busstate.ObjectsOf(testRunsObjects)) + tools := append(Tools(r, runs), FirstViewTools(os.Getenv(stdio.ServedAs), stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...) tools = append(tools, UnloggedTools(stdio.State(seat.StatePipelines), stdio.Log(seat.StateStepLog), meshhandactsclient.Recorder{Ask: stdio.Ask}, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) })...) tools = append(tools, CancelTool(r.List, stdio.Ask), RerunTool(stdio.Ask)) + tools = append(tools, TestRunsTools(stdio.Ask)...) tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now)) if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] %v\n", err) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/testruns.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/testruns.go new file mode 100644 index 00000000..de196cab --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/testruns.go @@ -0,0 +1,71 @@ +package main + +import ( + "encoding/json" + "errors" + "strings" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/busstate" + "git.novox.be/novox/mesh-sdk/go/catalogue" + + "mesh-delivery-pipeline/internal/pipeline" +) + +// testRunsState is the runner's state of its test runs (§725), and testRunsObjects its own object store (A-469e). +const testRunsState = "test-runs" + +var testRunsObjects = catalogue.StateObjectsName(testRunsState) + +// RunsRead is the verb test-runs' reader over the runner's state test-runs and its object store, through the bundle's calls. +func RunsRead(state stdio.KeptState, info func() (*stdio.StateInfo, error), objects busstate.Objects) pipeline.RunsRead { + return pipeline.RunsRead{Runs: kept{state}, + Object: func(name string) ([]byte, error) { + o, err := objects.Get(name) + if err != nil && strings.Contains(err.Error(), "not found") { + return nil, pipeline.ErrNoRun + } + return o.Data, err + }, + Fill: func() (seat.Fill, error) { + i, err := info() + if err != nil { + return seat.Fill{}, err + } + return seat.Fill{Store: testRunsState, Bytes: int64(i.Bytes), Cap: i.MaxBytes}, nil + }} +} + +// TestRunsTools are runner_test_runs and runner_test_runs_drop (design 47 G §728; A-469f, A-469u): each calls its seat verb +// over the bus, test-runs answered by this bundle and test-runs-drop by the runner's process (A-469g). +func TestRunsTools(ask func(key string, body any) (json.RawMessage, error)) []stdio.Tool { + return []stdio.Tool{ + {Name: Prefix + "_test_runs", + Description: "The pipeline runner's test runs, one per candidate, level and tested packages (the state test-runs): each with its packages read from its output object, its ask, node, readers, outcome and counts, and the state's fill against its cap; filtered by a candidate, a level, a digest or a module, or none.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "candidate": map[string]any{"type": "string"}, "level": map[string]any{"type": "string"}, + "digest": map[string]any{"type": "string"}, "module": map[string]any{"type": "string"}}}, + Run: func(args map[string]any) (any, error) { + in, err := as[seat.TestRunsInput](args) + if err != nil { + return nil, err + } + return ask("seat:"+seat.Seat+"."+seat.VerbTestRuns, in) + }}, + {Name: Prefix + "_test_runs_drop", + Description: "Drop one test run by the seat verb test-runs-drop, a hand act recorded first: its ask is dropped, each reading step ended could not run naming the drop, and its key and output object deleted. It takes the run's key and why.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "key": map[string]any{"type": "string"}, "why": map[string]any{"type": "string"}}, "required": []string{"key", "why"}}, + Run: func(args map[string]any) (any, error) { + in, err := as[seat.TestRunsDropInput](args) + if err != nil { + return nil, err + } + if strings.TrimSpace(in.Key) == "" || strings.TrimSpace(in.Why) == "" { + return nil, errors.New("test-runs-drop names the run's key and why") + } + return ask("seat:"+seat.Seat+"."+seat.VerbTestRunsDrop, in) + }}, + } +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go index 3224a5a9..bda4f80b 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go @@ -48,6 +48,8 @@ func writeHandlers(v pipeline.Verbs) map[string]process.SeatHandler { seat.VerbCancel: verbOf(func(c string, in seat.CancelInput) (any, error) { return v.Cancel(c, in) }), seat.VerbStart: verbOf(func(c string, in seat.StartInput) (any, error) { return v.Start(c, in) }), seat.VerbAskAgain: verbOf(func(c string, in seat.AskAgainInput) (any, error) { return v.AskAgain(c, in) }), + // §728, A-469g: a test run's drop is answered by the process, the only writer of pipelines and test-runs. + seat.VerbTestRunsDrop: verbOf(func(c string, in seat.TestRunsDropInput) (any, error) { return v.TestRunsDrop(c, in) }), } } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go index 9e2d5d28..a0993929 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go @@ -23,7 +23,7 @@ func TestEveryWriteVerbIsServed(t *testing.T) { got = append(got, v) } slices.Sort(got) - want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbRerun, seat.VerbReview, seat.VerbStart, seat.VerbRun} + want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbRerun, seat.VerbReview, seat.VerbStart, seat.VerbRun, seat.VerbTestRunsDrop} slices.Sort(want) if !slices.Equal(got, want) { t.Errorf("the process answers %v, want %v", got, want) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index c31f6639..db8f1f1c 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -340,8 +340,17 @@ func (h *Hearer) endRunOf(p seat.Pipeline, s seat.StepResult, ev StepEnded) erro if ev.Tests != nil { end.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration} } - if ev.Output != "" && h.RunOutputs != nil { - if err := h.RunOutputs.Put(run, []byte(ev.Output)); err != nil { + if h.RunOutputs != nil { + // The run's output object holds the packages it ran with their digests and the build seat's output (§725). + out := RunOutput{Output: ev.Output, Packages: []seat.TestRunPackage{}} + for _, k := range append(append([]nodebuildgo.KeptResult(nil), ev.Kept...), ev.Carried...) { + out.Packages = append(out.Packages, seat.TestRunPackage{Package: k.Package, Digest: k.Digest}) + } + raw, err := json.Marshal(out) + if err == nil { + err = h.RunOutputs.Put(run, raw) + } + if err != nil { return fmt.Errorf("the output of the test run %s cannot be kept: %w", run, err) } end.Output = run diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_read.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_read.go new file mode 100644 index 00000000..0e863d17 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_read.go @@ -0,0 +1,160 @@ +package pipeline + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" +) + +// RunOutput is a test run's output object in test-runs-objects (§725): the packages the run ran with their digests, the +// passing ones it kept and the carried ones it skipped, and the build seat's output. +type RunOutput struct { + Packages []seat.TestRunPackage `json:"packages"` + Output string `json:"output,omitempty"` +} + +// Gone is what a run whose output object is gone answers in place of its packages (§725, §733). +const Gone = "the run's output is gone" + +// RunsRead answers the seat verb test-runs (design 47 G §728; A-469u) from the runner's state test-runs and its object +// store: each run with its packages read from its output object, and the state's fill. +type RunsRead struct { + Runs Kept + // Object reads an object of test-runs-objects; ErrNoRun when it is gone. + Object func(name string) ([]byte, error) + Fill func() (seat.Fill, error) +} + +// TestRuns answers each test run of a candidate, a level, a digest or a module, or every one. +func (r RunsRead) TestRuns(in seat.TestRunsInput) (seat.TestRunsAnswer, error) { + all, err := r.Runs.Keys() + if err != nil { + return seat.TestRunsAnswer{}, fmt.Errorf("the test runs are not listed: %w", err) + } + sort.Strings(all) + a := seat.TestRunsAnswer{Runs: []seat.TestRun{}} + for _, key := range all { + if IsRunBound(key) || (in.Candidate != "" && !strings.HasPrefix(key, in.Candidate+".")) { + continue + } + raw, err := r.Runs.Value(key) + if err != nil || raw == nil { + continue + } + var run Run + if err := json.Unmarshal(raw, &run); err != nil { + continue + } + if (in.Level != "" && run.Level != in.Level) || (in.Digest != "" && run.Digest != in.Digest) || (in.Module != "" && !readsModule(run, in.Module)) { + continue + } + t := run.Answer() + t.Key = key + r.packages(&t) + a.Runs = append(a.Runs, t) + } + if r.Fill != nil { + f, err := r.Fill() + if err != nil { + return seat.TestRunsAnswer{}, fmt.Errorf("the fill of test-runs cannot be read: %w", err) + } + a.Fill = f + } + return a, nil +} + +// packages names t's packages from its output object, or says it gone (§725, A-466v). +func (r RunsRead) packages(t *seat.TestRun) { + if t.Output == "" || r.Object == nil { + return + } + data, err := r.Object(t.Output) + if errors.Is(err, ErrNoRun) { + t.Gone = Gone + return + } + var out RunOutput + if err == nil { + err = json.Unmarshal(data, &out) + } + if err != nil { + t.Gone = "the run's output is not read: " + err.Error() + return + } + t.Packages = out.Packages +} + +// readsModule says whether a reader of run is a step of module, its artifact `:`. +func readsModule(run Run, module string) bool { + for _, o := range run.Readers { + if strings.HasPrefix(o.Artifact, module+":") { + return true + } + } + return false +} + +// TestRunsDrop is the seat verb test-runs-drop (design 47 G §728; A-469g, A-469v), answered by the runner's process: the +// hand act recorded first, the run's ask dropped, each reader ended could not run naming the drop, then the run's output +// object and its key deleted. It answers the run deleted and the readers it ended. +func (v Verbs) TestRunsDrop(caller string, in seat.TestRunsDropInput) (seat.TestRunsDropAnswer, error) { + h := v.Hearer + if h.TestRuns == nil { + return seat.TestRunsDropAnswer{}, errors.New("the runner keeps no test runs") + } + r, _, err := h.TestRuns.Get(in.Key) + if errors.Is(err, ErrNoRun) { + return seat.TestRunsDropAnswer{}, fmt.Errorf("no test run %q in test-runs", in.Key) + } + if err != nil { + return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not read: %w", in.Key, err) + } + if err := v.record(seat.VerbTestRunsDrop, "dropped the test run "+in.Key, in.Why); err != nil { + return seat.TestRunsDropAnswer{}, err + } + why := fmt.Sprintf("the test run %s was dropped by %s: %s", in.Key, caller, in.Why) + var ended []seat.TestRunReader + if r.Ended == nil { + if h.DropRun != nil && r.Ask != "" { + if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil { + h.logf("the drop of ask %s of the dropped test run %s is not answered: %v", r.Ask, in.Key, err) + } + } + if err := h.EndRun(in.Key, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil); err != nil { + return seat.TestRunsDropAnswer{}, err + } + for _, o := range r.Readers { + ended = append(ended, o.Of()) + } + } + if h.RunOutputs != nil { + // The run's object is named by its key (endRunOf); one the run named otherwise is deleted too. + for _, name := range []string{in.Key, r.Output} { + if name == "" { + continue + } + if err := h.RunOutputs.Delete(name); err != nil { + return seat.TestRunsDropAnswer{}, fmt.Errorf("the output object %s of the test run is not deleted, so its key is kept: %w", name, err) + } + } + } + if err := h.TestRuns.Delete(in.Key); err != nil { + return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not deleted: %w", in.Key, err) + } + if h.Clock != nil { + for _, k := range []string{BoundPrefix + in.Key, UntakenPrefix + in.Key} { + _ = h.Clock.Unbound(k) + } + } + gone := r.Answer() + gone.Key = in.Key + now := h.now() + if gone.Ended == nil { + gone.Ended, gone.Outcome, gone.Said = &now, seat.StepResultOutcomeCouldNotRun, why + } + return seat.TestRunsDropAnswer{Run: gone, Ended: ended}, nil +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go index 41185109..f37047d2 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -248,7 +248,7 @@ func TestATestRunsEndMovesEveryReaderAndIsCountedOnce(t *testing.T) { t.Errorf("%s counts %+v, want the run's 5 unit tests once", p.Pipeline, c) } } - if r, _, _ := f.runs.Get(unit.Shared[0].Run); r.Outcome != "failed" || r.Output != unit.Shared[0].Run || string(f.objects[r.Output]) != "--- FAIL: TestX" { + if r, _, _ := f.runs.Get(unit.Shared[0].Run); r.Outcome != "failed" || r.Output != unit.Shared[0].Run || !strings.Contains(string(f.objects[r.Output]), `"output":"--- FAIL: TestX"`) { t.Errorf("the ended run = %+v, objects %v; want its output kept under its key", r, f.objects) } saidReaders := 0 @@ -358,3 +358,49 @@ func TestARerunOfATestRunMovesEveryReaderThatDidNotPass(t *testing.T) { } } } + +// The test runs' tools (design 47 G §728, §733 unit; A-466v, A-469g, A-469u, A-469v): test-runs answers each run with its +// packages read from its output object, and a run whose object is gone without them, saying so; test-runs-drop records the +// hand act first, ends the readers could not run naming the drop, and deletes the key and its object; with no hand act +// recorded it does nothing. +func TestTheTestRunsAreReadAndDroppedByTheirVerbs(t *testing.T) { + f := newRunFixture() + p := f.start(t, 4, 9) + unit, lint := stepNamed(p, "app:program", "unit"), stepNamed(p, "app:program", "lint") + ended := time.Now() + if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask, + Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "passed", + Kept: []nodebuildgo.KeptResult{{Package: "app/x", Digest: "d1", Level: "unit"}}, Ended: &ended}); err != nil { + t.Fatal(err) + } + pipes := memKept{} + for k, e := range f.runs { + put(t, pipes, k, e.r) + } + read := RunsRead{Runs: pipes, Object: f.objects.Get, Fill: func() (seat.Fill, error) { return seat.Fill{Store: "test-runs", Bytes: 9, Cap: 16 << 20}, nil }} + a, err := read.TestRuns(seat.TestRunsInput{Level: "unit", Module: "app"}) + if err != nil || len(a.Runs) != 1 || len(a.Runs[0].Packages) != 1 || a.Runs[0].Packages[0] != (seat.TestRunPackage{Package: "app/x", Digest: "d1"}) || + a.Runs[0].Key != unit.Shared[0].Run || len(a.Runs[0].Readers) != 2 || a.Fill.Cap != 16<<20 { + t.Fatalf("test-runs = %+v, %v; want the unit run with its package and readers, and the fill", a, err) + } + delete(f.objects, unit.Shared[0].Run) // gone, whatever step-outputs' age: the run answers without its packages + if a, _ := read.TestRuns(seat.TestRunsInput{Level: "unit"}); len(a.Runs) != 1 || a.Runs[0].Gone != Gone || len(a.Runs[0].Packages) != 0 { + t.Errorf("a run whose output is gone = %+v", a.Runs) + } + acts := &actLog{} + v := Verbs{Hearer: f.h, Starter: f.s, Acts: acts} + if _, err := (Verbs{Hearer: f.h, Starter: f.s}).TestRunsDrop("op@shanks", seat.TestRunsDropInput{Key: lint.Shared[0].Run, Why: "stuck"}); err == nil { + t.Fatal("a drop with no hand act recorded was done") + } + f.objects[lint.Shared[0].Run] = []byte("{}") + d, err := v.TestRunsDrop("op@shanks", seat.TestRunsDropInput{Key: lint.Shared[0].Run, Why: "stuck"}) + if err != nil || len(*acts) != 1 || len(d.Ended) != 2 || d.Run.Key != lint.Shared[0].Run || len(f.drops) != 1 { + t.Fatalf("drop = %+v, %v; acts %v, drops %v", d, err, *acts, f.drops) + } + if _, _, err := f.runs.Get(lint.Shared[0].Run); err != ErrNoRun || f.objects[lint.Shared[0].Run] != nil { + t.Errorf("after the drop the key is %v and the object %q, want both gone", err, f.objects[lint.Shared[0].Run]) + } + if s := stepNamed(f.pipeline(p), "app:tools", "lint"); s.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(s.Why, "dropped by op@shanks") { + t.Errorf("a reader of the dropped run = %+v", s) + } +} diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index 99523c6d..9b3cb607 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -113,7 +113,8 @@ func TestManifest(t *testing.T) { "seat:node-build-node.accept.delivery", "seat:node-build-image.accept.check", "seat:node-build-image.accept.delivery"})}, {"uses operator-channel", equalSets(m.Uses, []string{"operator-channel"})}, // RN25 (§539): the runner's tools carry the prefix runner_. - {"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_unlogged", "runner_unlogged_append"})}, + {"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_unlogged", "runner_unlogged_append", + "runner_test_runs", "runner_test_runs_drop"})}, // A-305 (ADR 0227 rule 8, ADR 0240 rule 2): the process is judged by its unit, as every process of mesh-delivery, // never by a tool of its own; runner_health stays a tool. {"health is the unit", healthTool(raw) == "unit"}, diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index 46ed603b..893dcac0 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -18,7 +18,9 @@ "rerun", "accept", "cancel", - "ask-again" + "ask-again", + "test-runs", + "test-runs-drop" ] } ], @@ -27,7 +29,9 @@ "runner_first_view_again", "runner_health", "runner_unlogged", - "runner_unlogged_append" + "runner_unlogged_append", + "runner_test_runs", + "runner_test_runs_drop" ], "consumes": [ "mesh-delivery-indexer.take-in.taken",