From 2b3a9e4a6abc43eef013ccb8fd2b05fc9f0273fc Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 11:42:09 +0100 Subject: [PATCH] runner: asking a test run again moves every reader that did not pass (R5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A rerun of one artifact's failed test step left the other artifact's step failed on a result no longer true; the rerun now joins, reads or starts the key's run, carrying the former readers marked in the key, and the new run's end moves the marked ones, said again as a rerun by its caller and naming the new ask, the pipeline reopened; an accepted step does not move. Builds: design 47 G §727 (test part); A-469l, A-470d, A-470e --- .../cmd/mesh-delivery-pipeline/testruns.go | 14 +- .../internal/pipeline/place_test.go | 16 +- .../internal/pipeline/runends.go | 36 +++-- .../internal/pipeline/steps.go | 51 +++++- .../internal/pipeline/testruns.go | 85 ++++++++-- .../internal/pipeline/testruns_test.go | 64 ++++++++ .../internal/pipeline/verbs.go | 149 +++++++++++++++++- 7 files changed, 371 insertions(+), 44 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go index ca1091ea..e26358bb 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go @@ -34,7 +34,7 @@ type testRunsKV struct { kv jetstream.KeyValue } -func (t testRunsKV) Create(key string, r seat.TestRun) error { +func (t testRunsKV) Create(key string, r pipeline.Run) error { raw, err := json.Marshal(r) if err != nil { return err @@ -47,22 +47,22 @@ func (t testRunsKV) Create(key string, r seat.TestRun) error { return nil } -func (t testRunsKV) Get(key string) (seat.TestRun, uint64, error) { +func (t testRunsKV) Get(key string) (pipeline.Run, uint64, error) { e, err := t.kv.Get(t.ctx, key) if errors.Is(err, jetstream.ErrKeyNotFound) { - return seat.TestRun{}, 0, pipeline.ErrNoRun + return pipeline.Run{}, 0, pipeline.ErrNoRun } if err != nil { - return seat.TestRun{}, 0, err + return pipeline.Run{}, 0, err } - var r seat.TestRun + var r pipeline.Run if err := json.Unmarshal(e.Value(), &r); err != nil { - return seat.TestRun{}, 0, fmt.Errorf("the test run %s is no test run: %w", key, err) + return pipeline.Run{}, 0, fmt.Errorf("the test run %s is no test run: %w", key, err) } return r, e.Revision(), nil } -func (t testRunsKV) Update(key string, r seat.TestRun, revision uint64) (uint64, error) { +func (t testRunsKV) Update(key string, r pipeline.Run, revision uint64) (uint64, error) { raw, err := json.Marshal(r) if err != nil { return 0, err diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index d4347310..f516c158 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -331,38 +331,38 @@ func TestAnArtifactWithoutALanguageIsPlacedByItsKind(t *testing.T) { // testRunsMem is the state test-runs in memory, a write by compare-and-set on the revision read. type testRunsMem map[string]struct { - r seat.TestRun + r Run rev uint64 } -func (m testRunsMem) Create(key string, r seat.TestRun) error { +func (m testRunsMem) Create(key string, r Run) error { if _, ok := m[key]; ok { return ErrExists } m[key] = struct { - r seat.TestRun + r Run rev uint64 }{r, 1} return nil } -func (m testRunsMem) Get(key string) (seat.TestRun, uint64, error) { +func (m testRunsMem) Get(key string) (Run, uint64, error) { e, ok := m[key] if !ok { - return seat.TestRun{}, 0, ErrNoRun + return Run{}, 0, ErrNoRun } - e.r.Readers = append([]seat.TestRunReader(nil), e.r.Readers...) + e.r.Readers = append([]RunReader(nil), e.r.Readers...) return e.r, e.rev, nil } func (m testRunsMem) Delete(key string) error { delete(m, key); return nil } -func (m testRunsMem) Update(key string, r seat.TestRun, rev uint64) (uint64, error) { +func (m testRunsMem) Update(key string, r Run, rev uint64) (uint64, error) { if m[key].rev != rev { return 0, ErrChanged } m[key] = struct { - r seat.TestRun + r Run rev uint64 }{r, rev + 1} return rev + 1, nil diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go index ade028a9..8d98ba7e 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go @@ -231,10 +231,10 @@ func (h *Hearer) runSettings() RunSettings { } // EndRun ends the test run under run with end's outcome, words, counts, node and output, written once by compare-and-set, -// and ends each of its readers with it but skip, the step whose own end ended it (§725, §726). A run ended already is -// not ended again. +// and ends each of its readers with it but skip, the step whose own end ended it (§725, §726). A run ended already is not +// ended again. func (h *Hearer) EndRun(run string, end seat.TestRun, skip *seat.TestRunReader) error { - var r seat.TestRun + var r Run for { got, rev, err := h.TestRuns.Get(run) if errors.Is(err, ErrNoRun) { @@ -271,7 +271,7 @@ func (h *Hearer) EndRun(run string, end seat.TestRun, skip *seat.TestRunReader) } var first error for _, reader := range r.Readers { - if skip != nil && reader == *skip { + if skip != nil && reader.Of() == *skip { continue } if err := h.endReader(reader, run, r); err != nil && first == nil { @@ -282,17 +282,24 @@ func (h *Hearer) EndRun(run string, end seat.TestRun, skip *seat.TestRunReader) } // endReader ends reader's step of run with the run's result (§726): its outcome, words, counts and node, the step naming -// the run in Shared as it did. A step ended already takes nothing. -func (h *Hearer) endReader(reader seat.TestRunReader, run string, r seat.TestRun) error { +// the run in Shared as it did. A step ended already takes nothing, but one the rerun that asked the run marked (A-470d, +// A-470e): it read another ask of the key, and it moves with this one, said again marked a rerun by that caller, naming +// the new ask, its pipeline reopened (§727). +func (h *Hearer) endReader(reader RunReader, run string, r Run) error { for { p, rev, err := h.Store.Get(reader.Pipeline) if err != nil { return nil // a pipeline gone has no reader left to end } - i := -1 + i, moved := -1, false for j, s := range p.Steps { - if s.Artifact == reader.Artifact && sharedRun(s) == run && s.Outcome == seat.StepResultOutcomeRunning { - i = j + if s.Artifact != reader.Artifact || sharedRun(s) != run { + continue + } + if s.Outcome == seat.StepResultOutcomeRunning { + i, moved = j, false + } else if reader.Rerun && s.Shared[0].Ask != r.Ask && s.Accepted == nil { + i, moved = j, true } } if i < 0 { @@ -300,7 +307,14 @@ func (h *Hearer) endReader(reader seat.TestRunReader, run string, r seat.TestRun } now := h.now() s := &p.Steps[i] - s.Outcome, s.Why, s.Node, s.Ended = r.Outcome, r.Said, r.Node, &now + if moved { + s.Rerun, s.By = true, reader.By + s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: s.Shared[0].AskedBy}} + if p.Outcome != seat.PipelineOutcomeRunning { + p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeRunning, "", nil + } + } + s.Outcome, s.Why, s.Node, s.Ended, s.Tests = r.Outcome, r.Said, r.Node, &now, nil if r.Tests != nil { t := *r.Tests s.Tests = &t @@ -350,7 +364,7 @@ func (h *Hearer) leaveRun(key string, s seat.StepResult, why string) { } left := r.Readers[:0:0] for _, o := range r.Readers { - if o != reader { + if o.Of() != reader { left = append(left, o) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 88bbacc7..4f32dbb1 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -154,7 +154,7 @@ type Hearer struct { func (h *Hearer) EndReaders(run string, readers []seat.TestRunReader, r seat.TestRun) error { var first error for _, reader := range readers { - if err := h.endReader(reader, run, r); err != nil && first == nil { + if err := h.endReader(RunReader{Pipeline: reader.Pipeline, Artifact: reader.Artifact}, run, Run{TestRun: r}); err != nil && first == nil { first = err } } @@ -236,6 +236,8 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh } unlogged := endEntry(ev.Pipeline, *s, now, keptKeys...) p.Unlogged = append(p.Unlogged, unlogged) + // The steps of this pipeline reading the run end in the same write, so the pipeline ends once (§726, §727). + readers := h.sameRunReaders(&p, i, ev, now) from := now if ev.Ended != nil { from = ev.Ended.UTC() @@ -268,6 +270,11 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh if err := h.say(seat.EventStepEnded, p, s); err != nil { return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", ev.Pipeline, seat.EventStepEnded, err) } + for _, j := range readers { + if err := h.say(seat.EventStepEnded, p, &p.Steps[j]); err != nil { + return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", ev.Pipeline, seat.EventStepEnded, err) + } + } // The step's end is written and said before its log entry is appended, so a log that refuses ends no step (§486, §606). h.appendUnlogged(ev.Pipeline) h.Ager.Age() @@ -279,6 +286,48 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh } } +// sameRunReaders ends, in p, every other step reading the test run step i asked with ev, its ask's end: a running reader, +// and one the rerun that asked the run marked, which moves (A-470d, A-470e), said again marked a rerun by that caller. It +// answers their indexes. +func (h *Hearer) sameRunReaders(p *seat.Pipeline, i int, ev StepEnded, now time.Time) []int { + asker := p.Steps[i] + run := sharedRun(asker) + if run == "" || h.TestRuns == nil || asker.Ask != ev.ID { + return nil + } + marks := map[string]RunReader{} + if r, _, err := h.TestRuns.Get(run); err == nil { + for _, o := range r.Readers { + if o.Pipeline == p.Pipeline { + marks[o.Artifact] = o + } + } + } + var out []int + for j := range p.Steps { + s := &p.Steps[j] + if j == i || sharedRun(*s) != run { + continue + } + switch m := marks[s.Artifact]; { + case s.Outcome == seat.StepResultOutcomeRunning && s.Ask == "": + case m.Rerun && s.Shared[0].Ask != ev.ID && s.Accepted == nil: + s.Rerun, s.By = true, m.By + s.Shared = []seat.Shared{{Run: run, Ask: ev.ID, AskedBy: &seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: asker.Artifact}}} + default: + continue + } + s.Outcome, s.Why, s.Node, s.Ended, s.Tests = asker.Outcome, asker.Why, asker.Node, &now, nil + if asker.Tests != nil { + t := *asker.Tests + s.Tests = &t + } + p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *s, now)) + out = append(out, j) + } + return out +} + // endRunOf ends the test run step s of p asked, with the end ev of its ask (§725, §726): the run's output kept in // test-runs-objects under the run's key, its outcome, words, counts and node written, and every other reader ended with it. // A step that asked no run ends nothing more. diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go index f382d652..be004716 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go @@ -18,15 +18,46 @@ import ( // packages, written create-only by the step that asks the run and changed by compare-and-set. type TestRuns interface { // Create writes key when it is absent; ErrExists when it is written already. - Create(key string, r seat.TestRun) error + Create(key string, r Run) error // Get reads key and its revision; ErrNoRun when it is absent. - Get(key string) (seat.TestRun, uint64, error) + Get(key string) (Run, uint64, error) // Update writes key at the revision read; ErrChanged when another write came between. - Update(key string, r seat.TestRun, revision uint64) (uint64, error) + Update(key string, r Run, revision uint64) (uint64, error) // Delete deletes key; a key gone is no fault. Delete(key string) error } +// Run is a key's value in test-runs (§725): the test run, its readers each marked whether a rerun's new run moves it and by +// whom (A-470d), so the marks are in the key and a restart takes them up. +type Run struct { + seat.TestRun + Readers []RunReader `json:"readers,omitempty"` +} + +// RunReader is a step reading a test run, its pipeline key and artifact (§725), marked rerun with the caller of the rerun +// whose new run moves it (A-470d). +type RunReader struct { + Pipeline string `json:"pipeline"` + Artifact string `json:"artifact"` + Rerun bool `json:"rerun,omitempty"` + By string `json:"by,omitempty"` +} + +// Of is the reader as the seat names it. +func (r RunReader) Of() seat.TestRunReader { + return seat.TestRunReader{Pipeline: r.Pipeline, Artifact: r.Artifact} +} + +// Answer is the run as the seat's read answers it, its readers named by pipeline and artifact. +func (r Run) Answer() seat.TestRun { + t := r.TestRun + t.Readers = nil + for _, o := range r.Readers { + t.Readers = append(t.Readers, o.Of()) + } + return t +} + // ErrNoRun is a Get of a test-runs key that is not written. var ErrNoRun = errors.New("no test run under the key") @@ -79,14 +110,14 @@ func readsRun(s seat.StepResult, run string) bool { } // addReaders adds each reader not in r's readers yet, in order. -func addReaders(r *seat.TestRun, readers []seat.TestRunReader) { +func addReaders(r *Run, readers []seat.TestRunReader) { for _, n := range readers { have := false for _, o := range r.Readers { - have = have || o == n + have = have || o.Of() == n } if !have { - r.Readers = append(r.Readers, n) + r.Readers = append(r.Readers, RunReader{Pipeline: n.Pipeline, Artifact: n.Artifact}) } } } @@ -122,7 +153,7 @@ func (s *Starter) share(key string, asks []Asked) []Asked { return asks } var out []Asked - joined := map[string]seat.TestRun{} + joined := map[string]Run{} for _, a := range asks { if a.Run == "" { out = append(out, a) @@ -162,7 +193,7 @@ func (s *Starter) share(key string, asks []Asked) []Asked { readers = a.Readers } } - if err := s.EndReaders(run, readers, r); err != nil { + if err := s.EndReaders(run, readers, r.TestRun); err != nil { s.logf("the steps of %s reading the ended test run %s are not ended: %v", key, run, err) } } @@ -219,9 +250,10 @@ func (s *Starter) started(a Asked, write func() error) error { } // claimRun writes a's test run create-only; ok says the ask is put (the key is a's own), else run is the run it reads. -func (s *Starter) claimRun(a Asked) (seat.TestRun, bool, error) { +func (s *Starter) claimRun(a Asked) (Run, bool, error) { now := s.now() - value := seat.TestRun{Level: a.Level, Digest: a.Digest, Seat: a.Seat, Ask: a.ID, Started: &now, Readers: a.Readers} + value := Run{TestRun: seat.TestRun{Level: a.Level, Digest: a.Digest, Seat: a.Seat, Ask: a.ID, Started: &now}} + addReaders(&value, a.Readers) for { err := s.started(a, func() error { return s.TestRuns.Create(a.Run, value) }) if err == nil { @@ -229,17 +261,17 @@ func (s *Starter) claimRun(a Asked) (seat.TestRun, bool, error) { } var full *fullErr if errors.As(err, &full) { - return seat.TestRun{}, false, err + return Run{}, false, err } if !errors.Is(err, ErrExists) { - return seat.TestRun{}, false, err + return Run{}, false, err } r, rev, err := s.TestRuns.Get(a.Run) if errors.Is(err, ErrNoRun) { continue // deleted meanwhile: written again } if err != nil { - return seat.TestRun{}, false, err + return Run{}, false, err } if r.Ask == a.ID { return r, true, nil @@ -252,7 +284,7 @@ func (s *Starter) claimRun(a Asked) (seat.TestRun, bool, error) { if errors.Is(err, ErrChanged) { continue } else if err != nil { - return seat.TestRun{}, false, err + return Run{}, false, err } return value, true, nil } @@ -260,7 +292,7 @@ func (s *Starter) claimRun(a Asked) (seat.TestRun, bool, error) { if _, err := s.TestRuns.Update(a.Run, r, rev); errors.Is(err, ErrChanged) { continue } else if err != nil { - return seat.TestRun{}, false, err + return Run{}, false, err } return r, false, nil } @@ -268,7 +300,7 @@ func (s *Starter) claimRun(a Asked) (seat.TestRun, bool, error) { // readOthers writes pipeline key's steps that read another step's run: no ask of their own, the run's ask and its asker // in Shared (A-469m, A-469q). -func (s *Starter) readOthers(key string, joined map[string]seat.TestRun) { +func (s *Starter) readOthers(key string, joined map[string]Run) { if s.Store == nil { return } @@ -303,3 +335,24 @@ func (s *Starter) readOthers(key string, joined map[string]seat.TestRun) { return } } + +// joinRun adds reader to run's readers by compare-and-set. +func (s *Starter) joinRun(run string, reader seat.TestRunReader) { + if s.TestRuns == nil { + return + } + for { + r, rev, err := s.TestRuns.Get(run) + if err != nil { + s.logf("the test run %s is not read, so %s does not join it: %v", run, reader.Pipeline, err) + return + } + addReaders(&r, []seat.TestRunReader{reader}) + if _, err := s.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + s.logf("%s does not join the test run %s: %v", reader.Pipeline, run, err) + } + return + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go index 7d60ec64..e80d6291 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -292,3 +292,67 @@ func TestATestRunsEndMovesEveryReaderAndIsCountedOnce(t *testing.T) { t.Errorf("the composition step's shared = %+v, want mesh-wants' as it came", s.Shared) } } + +// Asking a test run again moves every reader (design 47 G §727; A-469l, A-470d, A-470e): a rerun of one artifact's failed +// step starts a new run under the same key, carrying the former readers, the one that did not pass marked; the new run's +// passing end moves the other artifact's step to passed, marked a rerun by the caller and naming the new ask, and the +// pipeline says pipeline-ended again; a step a person accepted does not move. +func TestARerunOfATestRunMovesEveryReaderThatDidNotPass(t *testing.T) { + for _, accepted := range []bool{false, true} { + f := newRunFixture() + reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Policy: catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "required", + Integration: "bypass", Composition: "bypass", Review: "bypass", Approval: "bypass"}}} + f.s.Registration = func(string) (catalog.Registration, bool) { return reg, true } + p := f.start(t, 4, 9) + e := f.store[p.Pipeline] + e.p.Steps[0].Outcome = seat.StepResultOutcomePassed // the build + f.store[p.Pipeline] = e + unit := stepNamed(p, "app:program", "unit") + end := func(ask string, outcome string, rerun bool) { + t.Helper() + ended := time.Now() + if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: ask, + Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: outcome, + Rerun: rerun, By: map[bool]string{true: "op@shanks"}[rerun], Ended: &ended}); err != nil { + t.Fatal(err) + } + } + end(unit.Ask, "failed", false) + if q := f.pipeline(p); q.Outcome != seat.PipelineOutcomeFailed || q.EndCount != 1 || stepNamed(q, "app:tools", "unit").Outcome != "failed" { + t.Fatalf("after the failing run the pipeline = %s (%d), tools %s", q.Outcome, q.EndCount, stepNamed(q, "app:tools", "unit").Outcome) + } + if accepted { + e := f.store[p.Pipeline] + i := stepIndex(e.p, "unit", "app:program") + e.p.Steps[i].Outcome, e.p.Steps[i].Accepted = seat.StepResultOutcomeAccepted, &seat.Accepted{By: "op@shanks", Why: "known"} + f.store[p.Pipeline] = e + } + v := Verbs{Hearer: f.h, Starter: f.s, View: func(string) (Half, bool) { return Half{Entries: map[string]catalog.CatalogEntry{ + "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, + "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}, true }, Acts: &actLog{}} + a, err := v.Rerun("op@shanks", seat.RerunInput{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:tools", Why: "flaky"}) + if err != nil { + t.Fatal(err) + } + again := a.Result.Ask + r, _, _ := f.runs.Get(unit.Shared[0].Run) + if again == "" || again == unit.Ask || r.Ask != again || r.Ended != nil || len(r.Readers) != 2 || r.Readers[0].Rerun == accepted || (!accepted && r.Readers[0].By != "op@shanks") { + t.Fatalf("the rerun's run = %+v (ask %s), want a new run under the same key carrying the program, marked unless accepted", r, again) + } + end(again, "passed", true) + q := f.pipeline(p) + prog := stepNamed(q, "app:program", "unit") + if accepted { + if prog.Outcome != seat.StepResultOutcomeAccepted { + t.Errorf("an accepted step moved to %s", prog.Outcome) + } + continue + } + if prog.Outcome != seat.StepResultOutcomePassed || !prog.Rerun || prog.By != "op@shanks" || prog.Shared[0].Ask != again { + t.Errorf("the program's step = %+v, want passed, a rerun by the caller, naming the new ask", prog) + } + if q.Outcome != seat.PipelineOutcomePassed || q.EndCount != 2 { + t.Errorf("the pipeline = %s (%d), want passed at its second end", q.Outcome, q.EndCount) + } + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go index 0c4ff543..554bf8f3 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -236,6 +236,11 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error if in.Step == seat.StepResultStepApproval { return seat.RerunAnswer{}, errors.New("an approval is asked again by ask-again, not rerun") } + if p, _, err := v.Hearer.Store.Get(in.Pipeline); err == nil { + if i, err := stepOf(p, in.Step, in.Artifact); err == nil && v.Hearer.TestRuns != nil && sharedRun(p.Steps[i]) != "" && BoundSeat(p.Steps[i].Seat) { + return v.rerunTest(caller, in, p.Steps[i]) + } + } var ask *Asked p, _, err := v.change(in.Pipeline, seat.EventStepStarted, func() error { return v.record(seat.VerbRerun, fmt.Sprintf("asked the %s step of %s again", in.Step, in.Pipeline), in.Why) @@ -295,6 +300,148 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error return seat.RerunAnswer{Pipeline: in.Pipeline, Result: &r}, nil } +// rerunTest asks a test step of a test run again (design 47 G §727, A-469l): it joins its key's run while that runs, reads one +// that started after the step's last end, or else starts a new run under a new ask, replacing the ended key by compare-and-set +// and carrying its readers into the new value, so the new run's end moves every former reader that did not pass. +func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult) (seat.RerunAnswer, error) { + h := v.Hearer + run := sharedRun(st) + if st.Outcome == seat.StepResultOutcomeRunning { + return seat.RerunAnswer{}, fmt.Errorf("the %s step of %s is running: it is not asked again", in.Step, in.Pipeline) + } + r, rev, err := h.TestRuns.Get(run) + if err != nil && !errors.Is(err, ErrNoRun) { + return seat.RerunAnswer{}, fmt.Errorf("the test run %s is not read: %w", run, err) + } + exists := err == nil + reader := seat.TestRunReader{Pipeline: in.Pipeline, Artifact: st.Artifact} + read := exists && r.Ended != nil && r.Outcome != seat.StepResultOutcomeCouldNotRun && st.Ended != nil && r.Started != nil && r.Started.After(*st.Ended) + join := exists && r.Ended == nil + var ask *Asked + if !join && !read { + p, _, err := h.Store.Get(in.Pipeline) + if err != nil { + return seat.RerunAnswer{}, err + } + if ask, err = v.askOf(p, st, caller, h.now()); err != nil { + return seat.RerunAnswer{}, err + } + } + var asker *seat.SharedAskedBy + if join || read { + if asker, err = askerOf(h.Store, r.Ask); err != nil { + return seat.RerunAnswer{}, err + } + if asker != nil && *asker == (seat.SharedAskedBy{Pipeline: reader.Pipeline, Artifact: reader.Artifact}) { + asker = nil + } + } + event := seat.EventStepStarted + if read { + event = seat.EventStepEnded + } + p, _, err := v.change(in.Pipeline, event, func() error { + return v.record(seat.VerbRerun, fmt.Sprintf("asked the %s step of %s again", in.Step, in.Pipeline), in.Why) + }, func(p *seat.Pipeline, now time.Time) (int, error) { + switch p.Outcome { + case seat.PipelineOutcomeCancelled, seat.PipelineOutcomeSuperseded: + return -1, fmt.Errorf("the pipeline %s is %s: a step of it is not asked again", p.Pipeline, p.Outcome) + } + i, err := stepOf(*p, in.Step, in.Artifact) + if err != nil { + return -1, err + } + s := &p.Steps[i] + *s = seat.StepResult{Step: s.Step, Artifact: s.Artifact, Status: s.Status, Fingerprint: s.Fingerprint, Seat: s.Seat, + Outcome: seat.StepResultOutcomeRunning, By: caller, Rerun: true, Started: &now} + switch { + case ask != nil: + s.Ask, s.Shared = ask.ID, []seat.Shared{{Run: run, Ask: ask.ID}} + case read: + s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: asker}} + s.Outcome, s.Why, s.Node, s.Ended = r.Outcome, r.Said, r.Node, &now + if r.Tests != nil { + t := *r.Tests + s.Tests = &t + } + default: + s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: asker}} + } + if p.Outcome != seat.PipelineOutcomeRunning { + p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeRunning, "", nil + } + for j := range p.Steps { + if p.Steps[j].Step == seat.StepResultStepApproval && p.Steps[j].Outcome == seat.StepResultOutcomeNotAsked { + p.Steps[j].Outcome, p.Steps[j].Why = seat.StepResultOutcomeWaiting, "" + } + } + return i, nil + }) + if err != nil { + return seat.RerunAnswer{}, err + } + h.pin(p) + switch { + case join || read: + v.Starter.joinRun(run, reader) + default: + now := h.now() + value := Run{TestRun: seat.TestRun{Level: st.Step, Digest: r.Digest, Seat: st.Seat, Ask: ask.ID, Started: &now}} + // Every former reader is carried, each that did not pass marked to move with the new run by this caller (A-470d). + for _, o := range r.Readers { + if o.Of() == reader { + continue + } + o.Rerun, o.By = !v.passedReader(o, run), "" + if o.Rerun { + o.By = caller + } + value.Readers = append(value.Readers, o) + } + addReaders(&value, []seat.TestRunReader{reader}) + a := *ask + a.Run = run + if err := v.Starter.started(a, func() error { + if !exists { + return h.TestRuns.Create(run, value) + } + _, err := h.TestRuns.Update(run, value, rev) + return err + }); err != nil { + return seat.RerunAnswer{}, fmt.Errorf("the step is written running but its test run %s is not: %w", run, err) + } + v.Starter.carry(in.Pipeline, []Asked{*ask}) + if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil { + return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err) + } + } + i, err := stepOf(p, in.Step, in.Artifact) + if err != nil { + return seat.RerunAnswer{}, err + } + res := p.Steps[i] + return seat.RerunAnswer{Pipeline: in.Pipeline, Result: &res}, nil +} + +// passedReader says whether reader's step of run passed, or a person accepted it (§30): a step that moves with no new run. +func (v Verbs) passedReader(reader RunReader, run string) bool { + p, _, err := v.Hearer.Store.Get(reader.Pipeline) + if err != nil { + return true // a pipeline gone has no step to move + } + for _, s := range p.Steps { + if s.Artifact == reader.Artifact && sharedRun(s) == run { + switch s.Outcome { + case seat.StepResultOutcomePassed, seat.StepResultOutcomeCarried, seat.StepResultOutcomeAccepted, + seat.StepResultOutcomeSuperseded, seat.StepResultOutcomeCancelled: + return true + } + return s.Accepted != nil + } + } + return true +} + // askOf is the ask to put on the serving seat for step s again: the one Place makes for it, under a new id (an ask put twice // under one id is kept once), saying it is a rerun and by whom. func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time.Time) (*Asked, error) { @@ -319,7 +466,7 @@ func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time } else { return nil, fmt.Errorf("the catalog half of %s is not held yet: the step is asked when it is", p.Branch) } - _, placed := Place(p, reg, half.Entries, now, v.Starter.Name) + _, placed := place(p, reg, half.Entries, now, v.Starter.Name, false) base := AskID(startOf(p, now), p.Pipeline, s.Artifact, s.Step) for _, a := range placed { if a.ID != base {