runner: no write of a test run acts on a read made before its latest change

The age and test-runs-drop deleted a run's key whatever was written after their read, an expiry or the
last reader's leave ended whatever run the key held by then, a rerun answered a write between its read
and its write as an error, and a step-progress heard as its run ended wrote a clock that later ended
the run asked next. A delete of test-runs is now a compare-and-set at the revision read, retried from
the latest read; a run's end ends only the run under the ask its caller judged; the last reader ends
the run in its own leave; a rerun writes its new run from the latest read; a clock is read against the
run after it is written, and a clock of a run not taken ends nothing.

Builds: design 47 G §725, §726, §727, §728; A-469l, A-469z, A-470d
This commit is contained in:
2026-11-17 12:23:40 +01:00
parent a439b8da47
commit 2a90323af3
9 changed files with 295 additions and 56 deletions
@@ -74,7 +74,19 @@ func (t testRunsKV) Update(key string, r pipeline.Run, revision uint64) (uint64,
return rev, err
}
func (t testRunsKV) Delete(key string) error {
func (t testRunsKV) Delete(key string, revision uint64) error {
err := t.kv.Delete(t.ctx, key, jetstream.LastRevision(revision))
if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) {
return pipeline.ErrChanged
}
if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
return err
}
return nil
}
// deleteBound deletes a bound key, whose value is no run: a key gone is no fault.
func (t testRunsKV) deleteBound(key string) error {
if err := t.kv.Delete(t.ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
return err
}
@@ -84,7 +96,7 @@ func (t testRunsKV) Delete(key string) error {
// Bound writes key again with its time to live: deleted, then created with ttl, so its expiry is heard only ttl after
// the last write (a delete is no expiry).
func (t testRunsKV) Bound(key string, ttl time.Duration) error {
if err := t.Delete(key); err != nil {
if err := t.deleteBound(key); err != nil {
return err
}
_, err := t.kv.Create(t.ctx, key, []byte(`{"bound":"`+key+`"}`), jetstream.KeyTTL(ttl))
@@ -94,7 +106,7 @@ func (t testRunsKV) Bound(key string, ttl time.Duration) error {
return err
}
func (t testRunsKV) Unbound(key string) error { return t.Delete(key) }
func (t testRunsKV) Unbound(key string) error { return t.deleteBound(key) }
func (t testRunsKV) Has(key string) (bool, error) {
_, err := t.kv.Get(t.ctx, key)
@@ -1,6 +1,7 @@
package pipeline
import (
"errors"
"fmt"
"time"
@@ -139,23 +140,29 @@ func (a *Ager) ageRun(r Retention, e seat.LogEntry) {
if run == "" {
return
}
t, _, err := a.Runs.Get(run)
// The entry is past the retention (ageAt), and a run ends no later than the reader's end it was read at: a run not
// ended yet is aged at a later reader's end.
if err != nil || t.Ended == nil {
return
}
if t.Output != "" && a.RunOutputs != nil {
if err := a.RunOutputs(t.Output); err != nil {
a.logf("aged: the output %s of the test run %s is not deleted, so the run is kept: %v", t.Output, run, err)
for {
t, rev, err := a.Runs.Get(run)
// The entry is past the retention (ageAt), and a run ends no later than the reader's end it was read at: a run not
// ended yet is aged at a later reader's end.
if err != nil || t.Ended == nil {
return
}
}
if err := a.Runs.Delete(run); err != nil {
a.logf("aged: the test run %s is not deleted: %v", run, err)
if t.Output != "" && a.RunOutputs != nil {
if err := a.RunOutputs(t.Output); err != nil {
a.logf("aged: the output %s of the test run %s is not deleted, so the run is kept: %v", t.Output, run, err)
return
}
}
// Deleted at the revision read: a rerun that wrote the key again since is read again, and a run it started is kept.
if err := a.Runs.Delete(run, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
a.logf("aged: the test run %s is not deleted: %v", run, err)
return
}
a.logf("aged: deleted the test run %s and its output (%s after its end)", run, r.Age)
return
}
a.logf("aged: deleted the test run %s and its output (%s after its end)", run, r.Age)
}
func (a *Ager) ageEntry(r Retention, e seat.LogEntry, now time.Time) {
@@ -355,7 +355,13 @@ func (m testRunsMem) Get(key string) (Run, uint64, error) {
return e.r, e.rev, nil
}
func (m testRunsMem) Delete(key string) error { delete(m, key); return nil }
func (m testRunsMem) Delete(key string, rev uint64) error {
if e, ok := m[key]; ok && e.rev != rev {
return ErrChanged
}
delete(m, key)
return nil
}
func (m testRunsMem) Update(key string, r Run, rev uint64) (uint64, error) {
if m[key].rev != rev {
@@ -119,12 +119,29 @@ func (h *Hearer) RunTaken(ask, node string) error {
h.runStarted(run, r)
break
}
if err := h.Clock.Bound(BoundPrefix+run, h.runSettings().StepBound); err != nil {
if err := h.bound(run, ask); err != nil {
return fmt.Errorf("the clock of the test run %s is not begun: %w", run, err)
}
return h.Clock.Unbound(UntakenPrefix + run)
}
// bound writes the clock of the run under ask, judged from a read of the run, then reads the run again: a run ended since,
// or asked again and not taken, has its clock deleted, so no bound written from the earlier read ends the run after it.
func (h *Hearer) bound(run, ask string) error {
if err := h.Clock.Bound(BoundPrefix+run, h.runSettings().StepBound); err != nil {
return err
}
r, _, err := h.TestRuns.Get(run)
if err != nil && !errors.Is(err, ErrNoRun) {
return err
}
// A run asked again and taken since keeps the clock: it is its own as well (its ask-taken writes it).
if errors.Is(err, ErrNoRun) || r.Ended != nil || (r.Ask != ask && r.Node == "") {
return h.Clock.Unbound(BoundPrefix + run)
}
return nil
}
// RunProgress takes a build seat's step-progress of a test run's ask (§725, A-466p): its clock begins again, so a run that
// keeps saying its progress is never ended by it. An ask of no running, taken test run takes nothing.
func (h *Hearer) RunProgress(ask string) error {
@@ -139,7 +156,7 @@ func (h *Hearer) RunProgress(ask string) error {
if err != nil {
return fmt.Errorf("the test run %s is not read: %w", run, err)
}
return h.Clock.Bound(BoundPrefix+run, h.runSettings().StepBound)
return h.bound(run, ask)
}
// RunExpired takes the expiry of a test run's bound key (§725, §720 *When the clock ends*, A-466s): the run ends `could not
@@ -157,7 +174,8 @@ func (h *Hearer) RunExpired(key string) error {
if err != nil {
return fmt.Errorf("the test run %s is not read: %w", run, err)
}
if r.Ended != nil || (untaken && r.Node != "") {
// An untaken bound of a taken run, and a clock of a run not taken (a clock left from an ask before a rerun's), end nothing.
if r.Ended != nil || (untaken && r.Node != "") || (!untaken && r.Node == "") {
return nil
}
s := h.runSettings()
@@ -170,7 +188,7 @@ func (h *Hearer) RunExpired(key string) error {
h.logf("the drop of ask %s of the test run %s is not answered: %v", r.Ask, run, err)
}
}
return h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil)
return h.EndRun(run, seat.TestRun{Ask: r.Ask, Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil)
}
// TakeUp takes up again, at the runner's start, each test run with no end (§725): a taken run whose clock is gone, and an
@@ -233,7 +251,8 @@ func (h *Hearer) runSettings() RunSettings {
// EndRun ends the test run under run with end's outcome, words, counts, node and output, written once by compare-and-set,
// and ends each of its readers with it but skip, the step whose own end ended it (§725, §726). A run ended already is not
// ended again.
// ended again, and end's ask, the one its caller judged, ends only a run under that ask: a run a rerun asked since is not
// ended by a judgement of the one before.
func (h *Hearer) EndRun(run string, end seat.TestRun, skip *seat.TestRunReader) error {
var r Run
for {
@@ -244,7 +263,7 @@ func (h *Hearer) EndRun(run string, end seat.TestRun, skip *seat.TestRunReader)
if err != nil {
return fmt.Errorf("the test run %s is not read: %w", run, err)
}
if got.Ended != nil {
if got.Ended != nil || (end.Ask != "" && got.Ask != end.Ask) {
return nil
}
now := h.now()
@@ -373,23 +392,34 @@ func (h *Hearer) leaveRun(key string, s seat.StepResult, why string) {
return
}
r.Readers = left
// The last to leave ends the run in the same write, so a step joining it after that read finds it ended and asks
// its own, never joining a run then ended for having no reader.
last := len(left) == 0 && r.Ended == nil
if last {
now := h.now()
r.Ended, r.Outcome, r.Said = &now, seat.StepResultOutcomeCouldNotRun, NoReaderLeft
}
if _, err := h.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
h.logf("the step %s of %s does not leave the test run %s: %v", s.Step, key, run, err)
return
}
if len(left) > 0 || r.Ended != nil {
if !last {
return
}
if h.Clock != nil {
for _, k := range []string{BoundPrefix + run, UntakenPrefix + run} {
if err := h.Clock.Unbound(k); err != nil {
h.logf("the bound %s of the ended test run is not deleted: %v", k, err)
}
}
}
if h.DropRun != nil {
if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil {
h.logf("the drop of ask %s of the test run %s, which no reader is left of, is not answered: %v", r.Ask, run, err)
}
}
if err := h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: NoReaderLeft}, nil); err != nil {
h.logf("the test run %s, which no reader is left of, is not ended: %v", run, err)
}
return
}
}
@@ -336,7 +336,7 @@ func (h *Hearer) endRunOf(p seat.Pipeline, s seat.StepResult, ev StepEnded) erro
if run == "" || h.TestRuns == nil || s.Ask != ev.ID {
return nil
}
end := seat.TestRun{Outcome: ev.Outcome, Said: ev.Said, Node: ev.Node}
end := seat.TestRun{Ask: ev.ID, Outcome: ev.Outcome, Said: ev.Said, Node: ev.Node}
if ev.Tests != nil {
end.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration}
}
@@ -23,8 +23,9 @@ type TestRuns interface {
Get(key string) (Run, uint64, error)
// Update writes key at the revision read; ErrChanged when another write came between.
Update(key string, r Run, revision uint64) (uint64, error)
// Delete deletes key; a key gone is no fault.
Delete(key string) error
// Delete deletes key at the revision read; ErrChanged when another write came between, a key gone is no fault. No
// write of a run puts a value judged from an earlier read over a later one (the kind of the stale write).
Delete(key string, revision uint64) error
}
// Run is a key's value in test-runs (§725): the test run, its readers each marked whether a rerun's new run moves it and by
@@ -124,7 +124,7 @@ func (v Verbs) TestRunsDrop(caller string, in seat.TestRunsDropInput) (seat.Test
h.logf("the drop of ask %s of the dropped test run %s is not answered: %v", r.Ask, in.Key, err)
}
}
if err := h.EndRun(in.Key, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil); err != nil {
if err := h.EndRun(in.Key, seat.TestRun{Ask: r.Ask, Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil); err != nil {
return seat.TestRunsDropAnswer{}, err
}
for _, o := range r.Readers {
@@ -142,8 +142,24 @@ func (v Verbs) TestRunsDrop(caller string, in seat.TestRunsDropInput) (seat.Test
}
}
}
if err := h.TestRuns.Delete(in.Key); err != nil {
return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not deleted: %w", in.Key, err)
// The key is deleted at the revision of the run dropped: a run a rerun asked under it since is not deleted with it.
for {
now, rev, err := h.TestRuns.Get(in.Key)
if errors.Is(err, ErrNoRun) {
break
}
if err != nil {
return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not read again, so it is not deleted: %w", in.Key, err)
}
if now.Ask != r.Ask {
return seat.TestRunsDropAnswer{}, fmt.Errorf("the key %s holds the run of ask %s, asked since the run of ask %s was dropped: it is not deleted", in.Key, now.Ask, r.Ask)
}
if err := h.TestRuns.Delete(in.Key, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not deleted: %w", in.Key, err)
}
break
}
if h.Clock != nil {
for _, k := range []string{BoundPrefix + in.Key, UntakenPrefix + in.Key} {
@@ -15,11 +15,20 @@ import (
type clockMem struct {
runs testRunsMem
bound map[string]time.Duration
// before is called once at the next Bound, a write that comes between its caller's read and its write.
before func()
}
func (c *clockMem) Bound(key string, ttl time.Duration) error { c.bound[key] = ttl; return nil }
func (c *clockMem) Unbound(key string) error { delete(c.bound, key); return nil }
func (c *clockMem) Has(key string) (bool, error) { _, ok := c.bound[key]; return ok, nil }
func (c *clockMem) Bound(key string, ttl time.Duration) error {
if b := c.before; b != nil {
c.before = nil
b()
}
c.bound[key] = ttl
return nil
}
func (c *clockMem) Unbound(key string) error { delete(c.bound, key); return nil }
func (c *clockMem) Has(key string) (bool, error) { _, ok := c.bound[key]; return ok, nil }
func (c *clockMem) Keys() ([]string, error) {
var out []string
for k := range c.runs {
@@ -453,3 +462,140 @@ func TestTheTestRunsAreReadAndDroppedByTheirVerbs(t *testing.T) {
t.Errorf("a reader of the dropped run = %+v", s)
}
}
// racingRuns is test-runs with a write coming between a caller's read and its next update or delete: each hook is called
// once, just before that write.
type racingRuns struct {
testRunsMem
beforeUpdate, beforeDelete func()
}
func (r *racingRuns) Update(key string, v Run, rev uint64) (uint64, error) {
if b := r.beforeUpdate; b != nil {
r.beforeUpdate = nil
b()
}
return r.testRunsMem.Update(key, v, rev)
}
func (r *racingRuns) Delete(key string, rev uint64) error {
if b := r.beforeDelete; b != nil {
r.beforeDelete = nil
b()
}
return r.testRunsMem.Delete(key, rev)
}
// No write of test-runs or its bound keys puts back, ends or deletes a run from a read made before its latest change (design
// 47 G §725, §726, §727, §728; the kind test 109 met): each write is a compare-and-set at the revision read, retried from the
// latest read, and a judgement of a run ends only the run under the ask it judged.
func TestNoWriteOfATestRunActsOnAReadBeforeItsLatestChange(t *testing.T) {
asked := func(f *runFixture, run string) {
t.Helper()
r, rev, _ := f.runs.Get(run)
r.TestRun = seat.TestRun{Level: r.Level, Digest: r.Digest, Seat: r.Seat, Ask: "a2"}
if _, err := f.runs.Update(run, r, rev); err != nil {
t.Fatal(err)
}
}
for _, c := range []struct {
name string
race func(t *testing.T, f *runFixture, p seat.Pipeline, run string)
}{
{"an expiry does not end a run asked again while its ask was dropped", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
unit := stepNamed(p, "app:program", "unit")
_ = f.h.RunTaken(unit.Ask, "ace")
f.h.DropRun = func(string, string, string, string) error { asked(f, run); return nil }
_ = f.h.RunExpired(BoundPrefix + run)
}},
{"the last reader to leave ends the run in its own write, so a step joining then asks its own", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
f.h.DropRun = func(string, string, string, string) error {
_, _, _ = f.s.claimRun(Asked{Run: run, ID: "a2", Level: "unit", Seat: "node-build-go",
Readers: []seat.TestRunReader{{Pipeline: "main.5.app", Artifact: "app:program"}}})
return nil
}
for _, a := range []string{"app:program", "app:tools"} {
f.h.leaveRun(p.Pipeline, stepNamed(p, a, "unit"), "cancelled")
}
}},
{"the age does not delete a run asked again since it was read", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
_ = f.h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomePassed}, nil)
runs := &racingRuns{testRunsMem: f.runs, beforeDelete: func() { asked(f, run) }}
a := &Ager{Runs: runs, RunsKept: time.Hour, Pipelines: agedDeleter{f.store}, Aged: agedState{}, Now: func() time.Time { return agedNow },
Log: func(from uint64, limit int) ([]LoggedEntry, error) {
if from > 1 {
return nil, nil
}
return []LoggedEntry{{Sequence: 1, Entry: seat.LogEntry{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:tools",
What: seat.LogEntryWhatEnded, At: agedNow.Add(-2 * time.Hour)}}}, nil
}}
a.Age()
}},
{"test-runs-drop does not delete a run asked again under its key", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
f.h.TestRuns = &racingRuns{testRunsMem: f.runs, beforeDelete: func() { asked(f, run) }}
v := Verbs{Hearer: f.h, Starter: f.s, Acts: &actLog{}}
if _, err := v.TestRunsDrop("op@shanks", seat.TestRunsDropInput{Key: run, Why: "stuck"}); err == nil {
t.Error("a drop whose key holds a run asked since answers no error")
}
}},
{"a step-progress heard as the run ends leaves no clock", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
unit := stepNamed(p, "app:program", "unit")
_ = f.h.RunTaken(unit.Ask, "ace")
f.clock.before = func() { _ = f.h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomePassed}, nil) }
_ = f.h.RunProgress(unit.Ask)
if _, ok := f.clock.bound[BoundPrefix+run]; ok {
t.Error("a clock written from a read before the run's end is kept")
}
asked(f, run) // the key's run is asked again, so the check below holds
}},
{"a clock of a run not taken ends nothing", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
asked(f, run)
_ = f.h.RunExpired(BoundPrefix + run)
}},
} {
t.Run(c.name, func(t *testing.T) {
f := newRunFixture()
p := f.start(t, 4, 9)
run := stepNamed(p, "app:program", "unit").Shared[0].Run
c.race(t, f, p, run)
if r, _, err := f.runs.Get(run); err != nil || r.Ask != "a2" || r.Ended != nil {
t.Errorf("the key's run = %+v, %v; want the run asked again under a2, running", r, err)
}
})
}
// A rerun whose read of the key is changed before its write (a step joining the ended run) writes from the latest read,
// carrying that reader, never answering the change as an error nor putting back the earlier readers.
t.Run("a rerun writes its run from the latest read", func(t *testing.T) {
f := newRunFixture()
p := f.start(t, 4, 9)
unit := stepNamed(p, "app:program", "unit")
run := unit.Shared[0].Run
ended := time.Now()
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed", Ended: &ended}); err != nil {
t.Fatal(err)
}
f.h.TestRuns = &racingRuns{testRunsMem: f.runs, beforeUpdate: func() {
r, rev, _ := f.runs.Get(run)
r.Readers = append(r.Readers, RunReader{Pipeline: "main.9.app", Artifact: "app:program"})
_, _ = f.runs.Update(run, r, rev)
}}
v := Verbs{Hearer: f.h, Starter: f.s, View: func(string) (Half, bool) {
return Half{Entries: map[string]catalog.CatalogEntry{
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}, true
}, Acts: &actLog{}}
a, err := v.Rerun("op@shanks", seat.RerunInput{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:program", Why: "fixed"})
if err != nil {
t.Fatalf("a rerun whose read was changed before its write: %v", err)
}
r, _, _ := f.runs.Get(run)
joined := false
for _, o := range r.Readers {
joined = joined || o.Pipeline == "main.9.app"
}
if r.Ask != a.Result.Ask || r.Ended != nil || !joined || len(r.Readers) != 3 {
t.Errorf("the rerun's run = %+v; want the new ask running, carrying the reader that joined", r)
}
})
}
@@ -392,29 +392,50 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult)
v.Starter.joinRun(run, reader)
default:
now := h.now()
value := Run{TestRun: seat.TestRun{Level: st.Step, Digest: r.Digest, Seat: st.Seat, Ask: ask.ID, Started: &now}}
// Every former reader is carried, each that did not pass marked to move with the new run by this caller (A-470d).
for _, o := range r.Readers {
if o.Of() == reader {
continue
}
o.Rerun, o.By = !v.passedReader(o, run), ""
if o.Rerun {
o.By = caller
}
value.Readers = append(value.Readers, o)
}
addReaders(&value, []seat.TestRunReader{reader})
a := *ask
a.Run = run
if err := v.Starter.started(a, func() error {
if !exists {
return h.TestRuns.Create(run, value)
// The new run is written from the latest read of the key: a write between (a reader leaving, a join) is read again and
// its readers carried, never put back; a run another step started since is said, not replaced.
for {
value := Run{TestRun: seat.TestRun{Level: st.Step, Digest: r.Digest, Seat: st.Seat, Ask: ask.ID, Started: &now}}
// Every former reader is carried, each that did not pass marked to move with the new run by this caller (A-470d).
for _, o := range r.Readers {
if o.Of() == reader {
continue
}
o.Rerun, o.By = !v.passedReader(o, run), ""
if o.Rerun {
o.By = caller
}
value.Readers = append(value.Readers, o)
}
_, err := h.TestRuns.Update(run, value, rev)
return err
}); err != nil {
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but its test run %s is not: %w", run, err)
addReaders(&value, []seat.TestRunReader{reader})
err := v.Starter.started(a, func() error {
if !exists {
return h.TestRuns.Create(run, value)
}
_, err := h.TestRuns.Update(run, value, rev)
return err
})
if errors.Is(err, ErrChanged) || errors.Is(err, ErrExists) {
got, gotRev, gerr := h.TestRuns.Get(run)
switch {
case errors.Is(gerr, ErrNoRun):
r, rev, exists = Run{TestRun: seat.TestRun{Digest: r.Digest}}, 0, false
continue
case gerr != nil:
err = gerr
case got.Ended == nil:
err = fmt.Errorf("the run of ask %s was asked under the key since it was read", got.Ask)
default:
r, rev, exists = got, gotRev, true
continue
}
}
if err != nil {
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but its test run %s is not: %w", run, err)
}
break
}
v.Starter.carry(in.Pipeline, []Asked{*ask})
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil {