diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go index ee7d7ded..4e01276d 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go @@ -114,7 +114,11 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any)) _ = m.Term() return } - _, err = h.OnStepEnded(bs, m.Subject(), ev) + var shared []seat.Shared + if bs.Shared != nil { + shared = bs.Shared(m.Data()) + } + _, err = h.OnStepEndedShared(bs, m.Subject(), ev, shared) switch { case errors.Is(err, pipeline.ErrRefused): logf("%v", err) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go index 7bfd4a94..ade028a9 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go @@ -324,3 +324,57 @@ func (h *Hearer) endReader(reader seat.TestRunReader, run string, r seat.TestRun return h.said(p, asked, ended) } } + +// NoReaderLeft is what a test run whose last reader left ends with (A-469z): its ask is dropped, and a later step of the +// candidate starts a new run. +const NoReaderLeft = "no reader is left" + +// leaveRun takes the step of pipeline key that read or asked a test run out of the run's readers (§726): a step superseded +// or cancelled. When no reader is left the run ends `could not run`, naming that (A-469z), and its ask is dropped (§119); +// while one is left the ask stays, so a superseded pipeline never takes away another's run. A step of no test run leaves +// nothing. +func (h *Hearer) leaveRun(key string, s seat.StepResult, why string) { + run := sharedRun(s) + if run == "" || h.TestRuns == nil { + return + } + reader := seat.TestRunReader{Pipeline: key, Artifact: s.Artifact} + for { + r, rev, err := h.TestRuns.Get(run) + if errors.Is(err, ErrNoRun) { + return + } + if err != nil { + h.logf("the test run %s is not read, so the step %s of %s does not leave it: %v", run, s.Step, key, err) + return + } + left := r.Readers[:0:0] + for _, o := range r.Readers { + if o != reader { + left = append(left, o) + } + } + if len(left) == len(r.Readers) { + return + } + r.Readers = left + if _, err := h.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + h.logf("the step %s of %s does not leave the test run %s: %v", s.Step, key, run, err) + return + } + if len(left) > 0 || r.Ended != nil { + return + } + if h.DropRun != nil { + if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil { + h.logf("the drop of ask %s of the test run %s, which no reader is left of, is not answered: %v", r.Ask, run, err) + } + } + if err := h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: NoReaderLeft}, nil); err != nil { + h.logf("the test run %s, which no reader is left of, is not ended: %v", run, err) + } + return + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 654bea6b..88bbacc7 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -26,6 +26,9 @@ type StepSeat struct { Prefix string // Decode reads the seat's step-ended body; nil is the build seats' one form. Decode func([]byte) (StepEnded, error) + // Shared reads the runs a step-ended names in `shared`, carried into the step's result as they came (A-468d); nil for + // a seat whose ends name none. + Shared func([]byte) []seat.Shared } // StepSeats are the three build seats, from their contracts. @@ -36,7 +39,25 @@ var StepSeats = []StepSeat{ // The seat mesh-builds answers the build step of a candidate, one entry named by its fingerprint (A-97, A-99). {Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded}, // The seat mesh-wants answers the composition step (§22, §24). - {Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded}, + {Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded, Shared: wantsShared}, +} + +// wantsShared is the composition runs a mesh-wants step-ended names, one per machine (design 47 G §723, §729; A-468d, +// A-469t): the same form, under the same name. +func wantsShared(raw []byte) []seat.Shared { + var w wants.StepEnded + if json.Unmarshal(raw, &w) != nil || len(w.Shared) == 0 { + return nil + } + out := make([]seat.Shared, 0, len(w.Shared)) + for _, x := range w.Shared { + e := seat.Shared{Machine: x.Machine, Run: x.Run, Ask: x.Ask} + if x.AskedBy != nil { + e.AskedBy = &seat.SharedAskedBy{Pipeline: x.AskedBy.Pipeline, Artifact: x.AskedBy.Artifact} + } + out = append(out, e) + } + return out } // decodeWantsEnded reads the composition step's end of the seat mesh-wants as the form the hearer holds each end to; a @@ -152,6 +173,13 @@ func (h *Hearer) now() time.Time { // another seat, commit, fingerprint, module or step (ADR 0666 §21; the bus grants step-ended. to that node's // holder alone, so the subject's node is the account's). A step already ended takes nothing (the bus gives an event again). func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pipeline, error) { + return h.OnStepEndedShared(bs, subject, ev, nil) +} + +// OnStepEndedShared is OnStepEnded of an end naming the runs it read in shared (A-468d): kept on the step's result. The end +// of a test run's ask also ends the run, its output kept in test-runs-objects, and every other step reading it (§725, §726, +// A-469m). +func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared) (seat.Pipeline, error) { node := strings.TrimPrefix(subject, bs.Prefix) if node == subject || node == "" || strings.Contains(node, ".") { return seat.Pipeline{}, fmt.Errorf("%w: %s is not a subject of %s's step-ended", ErrRefused, subject, bs.Seat) @@ -175,6 +203,10 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Commit, p.Fingerprint) case p.Steps[i].Outcome != seat.StepResultOutcomeRunning: if run := sharedRun(p.Steps[i]); run != "" && h.TestRuns != nil { + if r, _, err := h.TestRuns.Get(run); err == nil && r.Ended == nil && r.Ask == ev.ID { + // The asking step's end was written and its run's was not (a write failed): the run ends now, heard again. + return p, h.endRunOf(p, p.Steps[i], ev) + } // The run ended before its ask's end came (its bound, a drop): the end is ignored and said (§720, §725). h.logf("the step-ended of ask %s is ignored: its test run %s ended before it (%s)", ev.ID, run, p.Steps[i].Why) } @@ -185,6 +217,9 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi now := h.now() s := &p.Steps[i] s.Outcome, s.Node, s.Why, s.Rerun, s.By, s.Ended = ev.Outcome, node, ev.Said, ev.Rerun, ev.By, &now + if len(shared) > 0 { + s.Shared = shared + } if ev.Tests != nil { s.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration} } @@ -237,10 +272,34 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi h.appendUnlogged(ev.Pipeline) h.Ager.Age() h.Fills.Check() - return p, h.said(p, asked, ended) + if err := h.said(p, asked, ended); err != nil { + return p, err + } + return p, h.endRunOf(p, *s, ev) } } +// endRunOf ends the test run step s of p asked, with the end ev of its ask (§725, §726): the run's output kept in +// test-runs-objects under the run's key, its outcome, words, counts and node written, and every other reader ended with it. +// A step that asked no run ends nothing more. +func (h *Hearer) endRunOf(p seat.Pipeline, s seat.StepResult, ev StepEnded) error { + run := sharedRun(s) + if run == "" || h.TestRuns == nil || s.Ask != ev.ID { + return nil + } + end := seat.TestRun{Outcome: ev.Outcome, Said: ev.Said, Node: ev.Node} + if ev.Tests != nil { + end.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration} + } + if ev.Output != "" && h.RunOutputs != nil { + if err := h.RunOutputs.Put(run, []byte(ev.Output)); err != nil { + return fmt.Errorf("the output of the test run %s cannot be kept: %w", run, err) + } + end.Output = run + } + return h.EndRun(run, end, &seat.TestRunReader{Pipeline: p.Pipeline, Artifact: s.Artifact}) +} + // said says what a written pipeline became: an approval asked is a wait held on the operator (ADR 0283), never a stall. func (h *Hearer) said(p seat.Pipeline, asked, ended bool) error { if asked { @@ -344,11 +403,19 @@ const ( budgetIntegration = 50 ) -// testCounts is the module's test counts, the sum of its test steps' own counts; nil when no step counted any. +// testCounts is the module's test counts, the sum of its test steps' own counts, each test run counted once however many +// steps read it (§726, A-466i); nil when no step counted any. func testCounts(steps []seat.StepResult) *seat.TestCounts { var t seat.TestCounts counted := false + runs := map[string]bool{} for _, s := range steps { + if run := sharedRun(s); run != "" && s.Seat != wants.Seat { + if runs[run] { + continue + } + runs[run] = true + } if s.Tests != nil { t.Unit, t.Integration, counted = t.Unit+s.Tests.Unit, t.Integration+s.Tests.Integration, true } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go index a729b552..0a307702 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go @@ -113,7 +113,8 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting { continue } - if s.Ask != "" && BoundSeat(s.Seat) { + // A step of a test run leaves it: its ask is dropped only with its last reader (§726, leaveRun). + if s.Ask != "" && BoundSeat(s.Seat) && sharedRun(*s) == "" { drops = append(drops, *s) } s.Outcome, s.Why, s.Ended = seat.StepResultOutcomeSuperseded, why, ptrTime(now) @@ -136,6 +137,9 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop if len(changed) > 0 { h.appendUnlogged(key) } + for _, i := range changed { + h.leaveRun(key, p.Steps[i], why) + } for _, s := range drops { if drop != nil { if err := drop(s.Seat, s.Node, s.Ask, why); err != nil { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go index 1b1b787b..f382d652 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go @@ -151,6 +151,21 @@ func (s *Starter) share(key string, asks []Asked) []Asked { } if len(joined) > 0 { s.readOthers(key, joined) + // A run that ended within the candidate is read at once (§724): its readers end with its result. + for run, r := range joined { + if r.Ended == nil || s.EndReaders == nil { + continue + } + var readers []seat.TestRunReader + for _, a := range asks { + if a.Run == run { + readers = a.Readers + } + } + if err := s.EndReaders(run, readers, r); err != nil { + s.logf("the steps of %s reading the ended test run %s are not ended: %v", key, run, err) + } + } } return out } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go index aee903a9..ab79a96b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -8,6 +8,7 @@ import ( catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" ) // clockMem is the bound keys of test-runs in memory, each with the time to live it was last written with. @@ -219,3 +220,75 @@ func TestATestRunHasOneClockACapAndAnAge(t *testing.T) { type agedDeleter struct{ memStore } func (d agedDeleter) Delete(key string) error { delete(d.memStore, key); return nil } + +// How a test run ends and is said (design 47 G §726; A-466i, A-468d, A-469m, A-469z): the run's ask's end ends every step +// reading it with its outcome, counts and shared, its output kept in test-runs-objects; a pipeline counts the run's tests +// once; a step superseded leaves the run to its other readers, and the last to leave drops its ask and ends it could not +// run, no reader left; a composition step's end keeps the runs mesh-wants names. +func TestATestRunsEndMovesEveryReaderAndIsCountedOnce(t *testing.T) { + f := newRunFixture() + a := f.start(t, 4, 9) + b := f.start(t, 6, 9) // the same candidate: it reads a's runs + unit := stepNamed(a, "app:program", "unit") + ended := time.Now() + if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask, + Pipeline: a.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed", + Said: "TestX failed", Output: "--- FAIL: TestX", Tests: &nodebuildgo.TestCounts{Unit: 5}, Ended: &ended}); err != nil { + t.Fatal(err) + } + for _, p := range []seat.Pipeline{f.pipeline(a), f.pipeline(b)} { + for _, art := range []string{"app:program", "app:tools"} { + s := stepNamed(p, art, "unit") + if s.Outcome != seat.StepResultOutcomeFailed || s.Why != "TestX failed" || s.Tests == nil || s.Tests.Unit != 5 || + len(s.Shared) != 1 || s.Shared[0].Run != unit.Shared[0].Run || s.Shared[0].Ask != unit.Ask { + t.Errorf("%s's %s unit step = %+v, want failed with the run's result and shared", p.Pipeline, art, s) + } + } + if c := testCounts(p.Steps); c == nil || c.Unit != 5 { + t.Errorf("%s counts %+v, want the run's 5 unit tests once", p.Pipeline, c) + } + } + if r, _, _ := f.runs.Get(unit.Shared[0].Run); r.Outcome != "failed" || r.Output != unit.Shared[0].Run || string(f.objects[r.Output]) != "--- FAIL: TestX" { + t.Errorf("the ended run = %+v, objects %v; want its output kept under its key", r, f.objects) + } + saidReaders := 0 + for _, e := range f.said { + if e.Result.Step == "unit" && e.Result.Outcome == "failed" { + saidReaders++ + } + } + if saidReaders != 4 { + t.Errorf("said %d unit step-ended, want one per reading step", saidReaders) + } + // a, superseded, leaves the lint run to b's readers; b, superseded after, leaves none: the ask is dropped and the run ends. + lint := stepNamed(a, "app:program", "lint") + if _, err := f.h.Supersede(seat.Pipeline{Pipeline: "main.5.app", Branch: "main", Source: 5, Module: "app", Commit: "c2", Fingerprint: "src2:bb"}, nil, nil); err != nil { + t.Fatal(err) + } + if r, _, _ := f.runs.Get(lint.Shared[0].Run); r.Ended != nil || len(r.Readers) != 2 || len(f.drops) != 0 { + t.Fatalf("after a left the lint run = %+v, drops %v; want it running for b's two readers", r, f.drops) + } + if _, err := f.h.Supersede(seat.Pipeline{Pipeline: "main.7.app", Branch: "main", Source: 7, Module: "app", Commit: "c3", Fingerprint: "src2:cc"}, nil, nil); err != nil { + t.Fatal(err) + } + if r, _, _ := f.runs.Get(lint.Shared[0].Run); r.Outcome != seat.StepResultOutcomeCouldNotRun || r.Said != NoReaderLeft || len(f.drops) != 2 || f.drops[0] != "node-build-go "+lint.Ask { + t.Errorf("after the last reader left the lint run = %+v, drops %v; want it ended, no reader left, its ask and the integration run's dropped", r, f.drops) + } + // A composition step's end keeps the runs mesh-wants names, as they came (A-468d). + c := newRunFixture() + p := c.start(t, 4, 9) + comp := stepNamed(p, "", "composition") + bs := stepSeatOf(t, "mesh-wants") + raw := []byte(`{"id":"` + comp.Ask + `","pipeline":"main.4.app","step":"composition","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"passed",` + + `"shared":[{"machine":"ace","run":"main.9.ace","ask":"r1"}]}`) + ev, err := bs.Decode(raw) + if err != nil { + t.Fatal(err) + } + if _, err := c.h.OnStepEndedShared(bs, bs.Prefix+"n1", ev, bs.Shared(raw)); err != nil { + t.Fatal(err) + } + if s := stepNamed(c.pipeline(p), "", "composition"); len(s.Shared) != 1 || s.Shared[0] != (seat.Shared{Machine: "ace", Run: "main.9.ace", Ask: "r1"}) { + t.Errorf("the composition step's shared = %+v, want mesh-wants' as it came", s.Shared) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go index b12a19f7..91cbdf6e 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -206,11 +206,15 @@ func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, er } now := h.now() var drop []string + var leave []seat.StepResult for i := range p.Steps { s := &p.Steps[i] if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting { continue } + if sharedRun(*s) != "" && s.Seat != wants.Seat { + leave = append(leave, *s) // it leaves its test run, dropped only with the last reader (§726) + } if s.Ask != "" { drop = append(drop, s.Ask) } @@ -223,6 +227,9 @@ func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, er } else if err != nil { return seat.CancelAnswer{}, fmt.Errorf("pipeline %s cannot be written: %w", in.Pipeline, err) } + for _, s := range leave { + h.leaveRun(in.Pipeline, s, "the pipeline "+in.Pipeline+" is cancelled: "+in.Why) + } if v.Drop != nil { for _, ask := range drop { if err := v.Drop(ask); err != nil {