diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go index b8a6634c..da948880 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go @@ -74,7 +74,19 @@ func (t testRunsKV) Update(key string, r pipeline.Run, revision uint64) (uint64, return rev, err } -func (t testRunsKV) Delete(key string) error { +func (t testRunsKV) Delete(key string, revision uint64) error { + err := t.kv.Delete(t.ctx, key, jetstream.LastRevision(revision)) + if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) { + return pipeline.ErrChanged + } + if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + return err + } + return nil +} + +// deleteBound deletes a bound key, whose value is no run: a key gone is no fault. +func (t testRunsKV) deleteBound(key string) error { if err := t.kv.Delete(t.ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { return err } @@ -84,7 +96,7 @@ func (t testRunsKV) Delete(key string) error { // Bound writes key again with its time to live: deleted, then created with ttl, so its expiry is heard only ttl after // the last write (a delete is no expiry). func (t testRunsKV) Bound(key string, ttl time.Duration) error { - if err := t.Delete(key); err != nil { + if err := t.deleteBound(key); err != nil { return err } _, err := t.kv.Create(t.ctx, key, []byte(`{"bound":"`+key+`"}`), jetstream.KeyTTL(ttl)) @@ -94,7 +106,7 @@ func (t testRunsKV) Bound(key string, ttl time.Duration) error { return err } -func (t testRunsKV) Unbound(key string) error { return t.Delete(key) } +func (t testRunsKV) Unbound(key string) error { return t.deleteBound(key) } func (t testRunsKV) Has(key string) (bool, error) { _, err := t.kv.Get(t.ctx, key) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/aged.go b/modules/mesh-delivery-pipeline/internal/pipeline/aged.go index 948ac76a..bf9b7dda 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/aged.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/aged.go @@ -1,6 +1,7 @@ package pipeline import ( + "errors" "fmt" "time" @@ -139,23 +140,29 @@ func (a *Ager) ageRun(r Retention, e seat.LogEntry) { if run == "" { return } - t, _, err := a.Runs.Get(run) - // The entry is past the retention (ageAt), and a run ends no later than the reader's end it was read at: a run not - // ended yet is aged at a later reader's end. - if err != nil || t.Ended == nil { - return - } - if t.Output != "" && a.RunOutputs != nil { - if err := a.RunOutputs(t.Output); err != nil { - a.logf("aged: the output %s of the test run %s is not deleted, so the run is kept: %v", t.Output, run, err) + for { + t, rev, err := a.Runs.Get(run) + // The entry is past the retention (ageAt), and a run ends no later than the reader's end it was read at: a run not + // ended yet is aged at a later reader's end. + if err != nil || t.Ended == nil { return } - } - if err := a.Runs.Delete(run); err != nil { - a.logf("aged: the test run %s is not deleted: %v", run, err) + if t.Output != "" && a.RunOutputs != nil { + if err := a.RunOutputs(t.Output); err != nil { + a.logf("aged: the output %s of the test run %s is not deleted, so the run is kept: %v", t.Output, run, err) + return + } + } + // Deleted at the revision read: a rerun that wrote the key again since is read again, and a run it started is kept. + if err := a.Runs.Delete(run, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + a.logf("aged: the test run %s is not deleted: %v", run, err) + return + } + a.logf("aged: deleted the test run %s and its output (%s after its end)", run, r.Age) return } - a.logf("aged: deleted the test run %s and its output (%s after its end)", run, r.Age) } func (a *Ager) ageEntry(r Retention, e seat.LogEntry, now time.Time) { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index 5c933433..eec07992 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -355,7 +355,13 @@ func (m testRunsMem) Get(key string) (Run, uint64, error) { return e.r, e.rev, nil } -func (m testRunsMem) Delete(key string) error { delete(m, key); return nil } +func (m testRunsMem) Delete(key string, rev uint64) error { + if e, ok := m[key]; ok && e.rev != rev { + return ErrChanged + } + delete(m, key) + return nil +} func (m testRunsMem) Update(key string, r Run, rev uint64) (uint64, error) { if m[key].rev != rev { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go index 122936ae..d694bbb2 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go @@ -119,12 +119,29 @@ func (h *Hearer) RunTaken(ask, node string) error { h.runStarted(run, r) break } - if err := h.Clock.Bound(BoundPrefix+run, h.runSettings().StepBound); err != nil { + if err := h.bound(run, ask); err != nil { return fmt.Errorf("the clock of the test run %s is not begun: %w", run, err) } return h.Clock.Unbound(UntakenPrefix + run) } +// bound writes the clock of the run under ask, judged from a read of the run, then reads the run again: a run ended since, +// or asked again and not taken, has its clock deleted, so no bound written from the earlier read ends the run after it. +func (h *Hearer) bound(run, ask string) error { + if err := h.Clock.Bound(BoundPrefix+run, h.runSettings().StepBound); err != nil { + return err + } + r, _, err := h.TestRuns.Get(run) + if err != nil && !errors.Is(err, ErrNoRun) { + return err + } + // A run asked again and taken since keeps the clock: it is its own as well (its ask-taken writes it). + if errors.Is(err, ErrNoRun) || r.Ended != nil || (r.Ask != ask && r.Node == "") { + return h.Clock.Unbound(BoundPrefix + run) + } + return nil +} + // RunProgress takes a build seat's step-progress of a test run's ask (§725, A-466p): its clock begins again, so a run that // keeps saying its progress is never ended by it. An ask of no running, taken test run takes nothing. func (h *Hearer) RunProgress(ask string) error { @@ -139,7 +156,7 @@ func (h *Hearer) RunProgress(ask string) error { if err != nil { return fmt.Errorf("the test run %s is not read: %w", run, err) } - return h.Clock.Bound(BoundPrefix+run, h.runSettings().StepBound) + return h.bound(run, ask) } // RunExpired takes the expiry of a test run's bound key (§725, §720 *When the clock ends*, A-466s): the run ends `could not @@ -157,7 +174,8 @@ func (h *Hearer) RunExpired(key string) error { if err != nil { return fmt.Errorf("the test run %s is not read: %w", run, err) } - if r.Ended != nil || (untaken && r.Node != "") { + // An untaken bound of a taken run, and a clock of a run not taken (a clock left from an ask before a rerun's), end nothing. + if r.Ended != nil || (untaken && r.Node != "") || (!untaken && r.Node == "") { return nil } s := h.runSettings() @@ -170,7 +188,7 @@ func (h *Hearer) RunExpired(key string) error { h.logf("the drop of ask %s of the test run %s is not answered: %v", r.Ask, run, err) } } - return h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil) + return h.EndRun(run, seat.TestRun{Ask: r.Ask, Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil) } // TakeUp takes up again, at the runner's start, each test run with no end (§725): a taken run whose clock is gone, and an @@ -233,7 +251,8 @@ func (h *Hearer) runSettings() RunSettings { // EndRun ends the test run under run with end's outcome, words, counts, node and output, written once by compare-and-set, // and ends each of its readers with it but skip, the step whose own end ended it (§725, §726). A run ended already is not -// ended again. +// ended again, and end's ask, the one its caller judged, ends only a run under that ask: a run a rerun asked since is not +// ended by a judgement of the one before. func (h *Hearer) EndRun(run string, end seat.TestRun, skip *seat.TestRunReader) error { var r Run for { @@ -244,7 +263,7 @@ func (h *Hearer) EndRun(run string, end seat.TestRun, skip *seat.TestRunReader) if err != nil { return fmt.Errorf("the test run %s is not read: %w", run, err) } - if got.Ended != nil { + if got.Ended != nil || (end.Ask != "" && got.Ask != end.Ask) { return nil } now := h.now() @@ -373,23 +392,34 @@ func (h *Hearer) leaveRun(key string, s seat.StepResult, why string) { return } r.Readers = left + // The last to leave ends the run in the same write, so a step joining it after that read finds it ended and asks + // its own, never joining a run then ended for having no reader. + last := len(left) == 0 && r.Ended == nil + if last { + now := h.now() + r.Ended, r.Outcome, r.Said = &now, seat.StepResultOutcomeCouldNotRun, NoReaderLeft + } if _, err := h.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) { continue } else if err != nil { h.logf("the step %s of %s does not leave the test run %s: %v", s.Step, key, run, err) return } - if len(left) > 0 || r.Ended != nil { + if !last { return } + if h.Clock != nil { + for _, k := range []string{BoundPrefix + run, UntakenPrefix + run} { + if err := h.Clock.Unbound(k); err != nil { + h.logf("the bound %s of the ended test run is not deleted: %v", k, err) + } + } + } if h.DropRun != nil { if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil { h.logf("the drop of ask %s of the test run %s, which no reader is left of, is not answered: %v", r.Ask, run, err) } } - if err := h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: NoReaderLeft}, nil); err != nil { - h.logf("the test run %s, which no reader is left of, is not ended: %v", run, err) - } return } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 718e322b..6a1ce8ca 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -336,7 +336,7 @@ func (h *Hearer) endRunOf(p seat.Pipeline, s seat.StepResult, ev StepEnded) erro if run == "" || h.TestRuns == nil || s.Ask != ev.ID { return nil } - end := seat.TestRun{Outcome: ev.Outcome, Said: ev.Said, Node: ev.Node} + end := seat.TestRun{Ask: ev.ID, Outcome: ev.Outcome, Said: ev.Said, Node: ev.Node} if ev.Tests != nil { end.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration} } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go index 23cf0685..5287d4ae 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go @@ -23,8 +23,9 @@ type TestRuns interface { Get(key string) (Run, uint64, error) // Update writes key at the revision read; ErrChanged when another write came between. Update(key string, r Run, revision uint64) (uint64, error) - // Delete deletes key; a key gone is no fault. - Delete(key string) error + // Delete deletes key at the revision read; ErrChanged when another write came between, a key gone is no fault. No + // write of a run puts a value judged from an earlier read over a later one (the kind of the stale write). + Delete(key string, revision uint64) error } // Run is a key's value in test-runs (§725): the test run, its readers each marked whether a rerun's new run moves it and by diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_read.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_read.go index 0e863d17..b725c219 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_read.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_read.go @@ -124,7 +124,7 @@ func (v Verbs) TestRunsDrop(caller string, in seat.TestRunsDropInput) (seat.Test h.logf("the drop of ask %s of the dropped test run %s is not answered: %v", r.Ask, in.Key, err) } } - if err := h.EndRun(in.Key, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil); err != nil { + if err := h.EndRun(in.Key, seat.TestRun{Ask: r.Ask, Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil); err != nil { return seat.TestRunsDropAnswer{}, err } for _, o := range r.Readers { @@ -142,8 +142,24 @@ func (v Verbs) TestRunsDrop(caller string, in seat.TestRunsDropInput) (seat.Test } } } - if err := h.TestRuns.Delete(in.Key); err != nil { - return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not deleted: %w", in.Key, err) + // The key is deleted at the revision of the run dropped: a run a rerun asked under it since is not deleted with it. + for { + now, rev, err := h.TestRuns.Get(in.Key) + if errors.Is(err, ErrNoRun) { + break + } + if err != nil { + return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not read again, so it is not deleted: %w", in.Key, err) + } + if now.Ask != r.Ask { + return seat.TestRunsDropAnswer{}, fmt.Errorf("the key %s holds the run of ask %s, asked since the run of ask %s was dropped: it is not deleted", in.Key, now.Ask, r.Ask) + } + if err := h.TestRuns.Delete(in.Key, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not deleted: %w", in.Key, err) + } + break } if h.Clock != nil { for _, k := range []string{BoundPrefix + in.Key, UntakenPrefix + in.Key} { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go index 6000bd0b..cbbd531b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -15,11 +15,20 @@ import ( type clockMem struct { runs testRunsMem bound map[string]time.Duration + // before is called once at the next Bound, a write that comes between its caller's read and its write. + before func() } -func (c *clockMem) Bound(key string, ttl time.Duration) error { c.bound[key] = ttl; return nil } -func (c *clockMem) Unbound(key string) error { delete(c.bound, key); return nil } -func (c *clockMem) Has(key string) (bool, error) { _, ok := c.bound[key]; return ok, nil } +func (c *clockMem) Bound(key string, ttl time.Duration) error { + if b := c.before; b != nil { + c.before = nil + b() + } + c.bound[key] = ttl + return nil +} +func (c *clockMem) Unbound(key string) error { delete(c.bound, key); return nil } +func (c *clockMem) Has(key string) (bool, error) { _, ok := c.bound[key]; return ok, nil } func (c *clockMem) Keys() ([]string, error) { var out []string for k := range c.runs { @@ -453,3 +462,140 @@ func TestTheTestRunsAreReadAndDroppedByTheirVerbs(t *testing.T) { t.Errorf("a reader of the dropped run = %+v", s) } } + +// racingRuns is test-runs with a write coming between a caller's read and its next update or delete: each hook is called +// once, just before that write. +type racingRuns struct { + testRunsMem + beforeUpdate, beforeDelete func() +} + +func (r *racingRuns) Update(key string, v Run, rev uint64) (uint64, error) { + if b := r.beforeUpdate; b != nil { + r.beforeUpdate = nil + b() + } + return r.testRunsMem.Update(key, v, rev) +} + +func (r *racingRuns) Delete(key string, rev uint64) error { + if b := r.beforeDelete; b != nil { + r.beforeDelete = nil + b() + } + return r.testRunsMem.Delete(key, rev) +} + +// No write of test-runs or its bound keys puts back, ends or deletes a run from a read made before its latest change (design +// 47 G §725, §726, §727, §728; the kind test 109 met): each write is a compare-and-set at the revision read, retried from the +// latest read, and a judgement of a run ends only the run under the ask it judged. +func TestNoWriteOfATestRunActsOnAReadBeforeItsLatestChange(t *testing.T) { + asked := func(f *runFixture, run string) { + t.Helper() + r, rev, _ := f.runs.Get(run) + r.TestRun = seat.TestRun{Level: r.Level, Digest: r.Digest, Seat: r.Seat, Ask: "a2"} + if _, err := f.runs.Update(run, r, rev); err != nil { + t.Fatal(err) + } + } + for _, c := range []struct { + name string + race func(t *testing.T, f *runFixture, p seat.Pipeline, run string) + }{ + {"an expiry does not end a run asked again while its ask was dropped", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) { + unit := stepNamed(p, "app:program", "unit") + _ = f.h.RunTaken(unit.Ask, "ace") + f.h.DropRun = func(string, string, string, string) error { asked(f, run); return nil } + _ = f.h.RunExpired(BoundPrefix + run) + }}, + {"the last reader to leave ends the run in its own write, so a step joining then asks its own", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) { + f.h.DropRun = func(string, string, string, string) error { + _, _, _ = f.s.claimRun(Asked{Run: run, ID: "a2", Level: "unit", Seat: "node-build-go", + Readers: []seat.TestRunReader{{Pipeline: "main.5.app", Artifact: "app:program"}}}) + return nil + } + for _, a := range []string{"app:program", "app:tools"} { + f.h.leaveRun(p.Pipeline, stepNamed(p, a, "unit"), "cancelled") + } + }}, + {"the age does not delete a run asked again since it was read", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) { + _ = f.h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomePassed}, nil) + runs := &racingRuns{testRunsMem: f.runs, beforeDelete: func() { asked(f, run) }} + a := &Ager{Runs: runs, RunsKept: time.Hour, Pipelines: agedDeleter{f.store}, Aged: agedState{}, Now: func() time.Time { return agedNow }, + Log: func(from uint64, limit int) ([]LoggedEntry, error) { + if from > 1 { + return nil, nil + } + return []LoggedEntry{{Sequence: 1, Entry: seat.LogEntry{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:tools", + What: seat.LogEntryWhatEnded, At: agedNow.Add(-2 * time.Hour)}}}, nil + }} + a.Age() + }}, + {"test-runs-drop does not delete a run asked again under its key", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) { + f.h.TestRuns = &racingRuns{testRunsMem: f.runs, beforeDelete: func() { asked(f, run) }} + v := Verbs{Hearer: f.h, Starter: f.s, Acts: &actLog{}} + if _, err := v.TestRunsDrop("op@shanks", seat.TestRunsDropInput{Key: run, Why: "stuck"}); err == nil { + t.Error("a drop whose key holds a run asked since answers no error") + } + }}, + {"a step-progress heard as the run ends leaves no clock", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) { + unit := stepNamed(p, "app:program", "unit") + _ = f.h.RunTaken(unit.Ask, "ace") + f.clock.before = func() { _ = f.h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomePassed}, nil) } + _ = f.h.RunProgress(unit.Ask) + if _, ok := f.clock.bound[BoundPrefix+run]; ok { + t.Error("a clock written from a read before the run's end is kept") + } + asked(f, run) // the key's run is asked again, so the check below holds + }}, + {"a clock of a run not taken ends nothing", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) { + asked(f, run) + _ = f.h.RunExpired(BoundPrefix + run) + }}, + } { + t.Run(c.name, func(t *testing.T) { + f := newRunFixture() + p := f.start(t, 4, 9) + run := stepNamed(p, "app:program", "unit").Shared[0].Run + c.race(t, f, p, run) + if r, _, err := f.runs.Get(run); err != nil || r.Ask != "a2" || r.Ended != nil { + t.Errorf("the key's run = %+v, %v; want the run asked again under a2, running", r, err) + } + }) + } + // A rerun whose read of the key is changed before its write (a step joining the ended run) writes from the latest read, + // carrying that reader, never answering the change as an error nor putting back the earlier readers. + t.Run("a rerun writes its run from the latest read", func(t *testing.T) { + f := newRunFixture() + p := f.start(t, 4, 9) + unit := stepNamed(p, "app:program", "unit") + run := unit.Shared[0].Run + ended := time.Now() + if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask, + Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed", Ended: &ended}); err != nil { + t.Fatal(err) + } + f.h.TestRuns = &racingRuns{testRunsMem: f.runs, beforeUpdate: func() { + r, rev, _ := f.runs.Get(run) + r.Readers = append(r.Readers, RunReader{Pipeline: "main.9.app", Artifact: "app:program"}) + _, _ = f.runs.Update(run, r, rev) + }} + v := Verbs{Hearer: f.h, Starter: f.s, View: func(string) (Half, bool) { + return Half{Entries: map[string]catalog.CatalogEntry{ + "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, + "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}, true + }, Acts: &actLog{}} + a, err := v.Rerun("op@shanks", seat.RerunInput{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:program", Why: "fixed"}) + if err != nil { + t.Fatalf("a rerun whose read was changed before its write: %v", err) + } + r, _, _ := f.runs.Get(run) + joined := false + for _, o := range r.Readers { + joined = joined || o.Pipeline == "main.9.app" + } + if r.Ask != a.Result.Ask || r.Ended != nil || !joined || len(r.Readers) != 3 { + t.Errorf("the rerun's run = %+v; want the new ask running, carrying the reader that joined", r) + } + }) +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go index 4626d260..5ca58405 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -392,29 +392,50 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult) v.Starter.joinRun(run, reader) default: now := h.now() - value := Run{TestRun: seat.TestRun{Level: st.Step, Digest: r.Digest, Seat: st.Seat, Ask: ask.ID, Started: &now}} - // Every former reader is carried, each that did not pass marked to move with the new run by this caller (A-470d). - for _, o := range r.Readers { - if o.Of() == reader { - continue - } - o.Rerun, o.By = !v.passedReader(o, run), "" - if o.Rerun { - o.By = caller - } - value.Readers = append(value.Readers, o) - } - addReaders(&value, []seat.TestRunReader{reader}) a := *ask a.Run = run - if err := v.Starter.started(a, func() error { - if !exists { - return h.TestRuns.Create(run, value) + // The new run is written from the latest read of the key: a write between (a reader leaving, a join) is read again and + // its readers carried, never put back; a run another step started since is said, not replaced. + for { + value := Run{TestRun: seat.TestRun{Level: st.Step, Digest: r.Digest, Seat: st.Seat, Ask: ask.ID, Started: &now}} + // Every former reader is carried, each that did not pass marked to move with the new run by this caller (A-470d). + for _, o := range r.Readers { + if o.Of() == reader { + continue + } + o.Rerun, o.By = !v.passedReader(o, run), "" + if o.Rerun { + o.By = caller + } + value.Readers = append(value.Readers, o) } - _, err := h.TestRuns.Update(run, value, rev) - return err - }); err != nil { - return seat.RerunAnswer{}, fmt.Errorf("the step is written running but its test run %s is not: %w", run, err) + addReaders(&value, []seat.TestRunReader{reader}) + err := v.Starter.started(a, func() error { + if !exists { + return h.TestRuns.Create(run, value) + } + _, err := h.TestRuns.Update(run, value, rev) + return err + }) + if errors.Is(err, ErrChanged) || errors.Is(err, ErrExists) { + got, gotRev, gerr := h.TestRuns.Get(run) + switch { + case errors.Is(gerr, ErrNoRun): + r, rev, exists = Run{TestRun: seat.TestRun{Digest: r.Digest}}, 0, false + continue + case gerr != nil: + err = gerr + case got.Ended == nil: + err = fmt.Errorf("the run of ask %s was asked under the key since it was read", got.Ask) + default: + r, rev, exists = got, gotRev, true + continue + } + } + if err != nil { + return seat.RerunAnswer{}, fmt.Errorf("the step is written running but its test run %s is not: %w", run, err) + } + break } v.Starter.carry(in.Pipeline, []Asked{*ask}) if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil {