runner: a test run's end moves every step reading it, counted once, dropped only with its last reader (R3)

The run's ask's end now ends each reading step with the run's outcome, counts and shared, its
output kept in test-runs-objects; a pipeline counts a run's tests once; a superseded or cancelled
step leaves the run, the last to leave dropping its ask and ending it; a composition step keeps
the runs mesh-wants names.

Builds: design 47 G §726, §725 (output object), §724 (an ended run read), §729; A-466i, A-468d, A-469m, A-469t, A-469z
This commit is contained in:
2026-11-17 11:33:54 +01:00
parent cd08f1e1c2
commit 97f68b04ea
7 changed files with 229 additions and 5 deletions
@@ -114,7 +114,11 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any))
_ = m.Term()
return
}
_, err = h.OnStepEnded(bs, m.Subject(), ev)
var shared []seat.Shared
if bs.Shared != nil {
shared = bs.Shared(m.Data())
}
_, err = h.OnStepEndedShared(bs, m.Subject(), ev, shared)
switch {
case errors.Is(err, pipeline.ErrRefused):
logf("%v", err)
@@ -324,3 +324,57 @@ func (h *Hearer) endReader(reader seat.TestRunReader, run string, r seat.TestRun
return h.said(p, asked, ended)
}
}
// NoReaderLeft is what a test run whose last reader left ends with (A-469z): its ask is dropped, and a later step of the
// candidate starts a new run.
const NoReaderLeft = "no reader is left"
// leaveRun takes the step of pipeline key that read or asked a test run out of the run's readers (§726): a step superseded
// or cancelled. When no reader is left the run ends `could not run`, naming that (A-469z), and its ask is dropped (§119);
// while one is left the ask stays, so a superseded pipeline never takes away another's run. A step of no test run leaves
// nothing.
func (h *Hearer) leaveRun(key string, s seat.StepResult, why string) {
run := sharedRun(s)
if run == "" || h.TestRuns == nil {
return
}
reader := seat.TestRunReader{Pipeline: key, Artifact: s.Artifact}
for {
r, rev, err := h.TestRuns.Get(run)
if errors.Is(err, ErrNoRun) {
return
}
if err != nil {
h.logf("the test run %s is not read, so the step %s of %s does not leave it: %v", run, s.Step, key, err)
return
}
left := r.Readers[:0:0]
for _, o := range r.Readers {
if o != reader {
left = append(left, o)
}
}
if len(left) == len(r.Readers) {
return
}
r.Readers = left
if _, err := h.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
h.logf("the step %s of %s does not leave the test run %s: %v", s.Step, key, run, err)
return
}
if len(left) > 0 || r.Ended != nil {
return
}
if h.DropRun != nil {
if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil {
h.logf("the drop of ask %s of the test run %s, which no reader is left of, is not answered: %v", r.Ask, run, err)
}
}
if err := h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: NoReaderLeft}, nil); err != nil {
h.logf("the test run %s, which no reader is left of, is not ended: %v", run, err)
}
return
}
}
@@ -26,6 +26,9 @@ type StepSeat struct {
Prefix string
// Decode reads the seat's step-ended body; nil is the build seats' one form.
Decode func([]byte) (StepEnded, error)
// Shared reads the runs a step-ended names in `shared`, carried into the step's result as they came (A-468d); nil for
// a seat whose ends name none.
Shared func([]byte) []seat.Shared
}
// StepSeats are the three build seats, from their contracts.
@@ -36,7 +39,25 @@ var StepSeats = []StepSeat{
// The seat mesh-builds answers the build step of a candidate, one entry named by its fingerprint (A-97, A-99).
{Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded},
// The seat mesh-wants answers the composition step (§22, §24).
{Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded},
{Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded, Shared: wantsShared},
}
// wantsShared is the composition runs a mesh-wants step-ended names, one per machine (design 47 G §723, §729; A-468d,
// A-469t): the same form, under the same name.
func wantsShared(raw []byte) []seat.Shared {
var w wants.StepEnded
if json.Unmarshal(raw, &w) != nil || len(w.Shared) == 0 {
return nil
}
out := make([]seat.Shared, 0, len(w.Shared))
for _, x := range w.Shared {
e := seat.Shared{Machine: x.Machine, Run: x.Run, Ask: x.Ask}
if x.AskedBy != nil {
e.AskedBy = &seat.SharedAskedBy{Pipeline: x.AskedBy.Pipeline, Artifact: x.AskedBy.Artifact}
}
out = append(out, e)
}
return out
}
// decodeWantsEnded reads the composition step's end of the seat mesh-wants as the form the hearer holds each end to; a
@@ -152,6 +173,13 @@ func (h *Hearer) now() time.Time {
// another seat, commit, fingerprint, module or step (ADR 0666 §21; the bus grants step-ended.<node> to that node's
// holder alone, so the subject's node is the account's). A step already ended takes nothing (the bus gives an event again).
func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pipeline, error) {
return h.OnStepEndedShared(bs, subject, ev, nil)
}
// OnStepEndedShared is OnStepEnded of an end naming the runs it read in shared (A-468d): kept on the step's result. The end
// of a test run's ask also ends the run, its output kept in test-runs-objects, and every other step reading it (§725, §726,
// A-469m).
func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared) (seat.Pipeline, error) {
node := strings.TrimPrefix(subject, bs.Prefix)
if node == subject || node == "" || strings.Contains(node, ".") {
return seat.Pipeline{}, fmt.Errorf("%w: %s is not a subject of %s's step-ended", ErrRefused, subject, bs.Seat)
@@ -175,6 +203,10 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Commit, p.Fingerprint)
case p.Steps[i].Outcome != seat.StepResultOutcomeRunning:
if run := sharedRun(p.Steps[i]); run != "" && h.TestRuns != nil {
if r, _, err := h.TestRuns.Get(run); err == nil && r.Ended == nil && r.Ask == ev.ID {
// The asking step's end was written and its run's was not (a write failed): the run ends now, heard again.
return p, h.endRunOf(p, p.Steps[i], ev)
}
// The run ended before its ask's end came (its bound, a drop): the end is ignored and said (§720, §725).
h.logf("the step-ended of ask %s is ignored: its test run %s ended before it (%s)", ev.ID, run, p.Steps[i].Why)
}
@@ -185,6 +217,9 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
now := h.now()
s := &p.Steps[i]
s.Outcome, s.Node, s.Why, s.Rerun, s.By, s.Ended = ev.Outcome, node, ev.Said, ev.Rerun, ev.By, &now
if len(shared) > 0 {
s.Shared = shared
}
if ev.Tests != nil {
s.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration}
}
@@ -237,10 +272,34 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
h.appendUnlogged(ev.Pipeline)
h.Ager.Age()
h.Fills.Check()
return p, h.said(p, asked, ended)
if err := h.said(p, asked, ended); err != nil {
return p, err
}
return p, h.endRunOf(p, *s, ev)
}
}
// endRunOf ends the test run step s of p asked, with the end ev of its ask (§725, §726): the run's output kept in
// test-runs-objects under the run's key, its outcome, words, counts and node written, and every other reader ended with it.
// A step that asked no run ends nothing more.
func (h *Hearer) endRunOf(p seat.Pipeline, s seat.StepResult, ev StepEnded) error {
run := sharedRun(s)
if run == "" || h.TestRuns == nil || s.Ask != ev.ID {
return nil
}
end := seat.TestRun{Outcome: ev.Outcome, Said: ev.Said, Node: ev.Node}
if ev.Tests != nil {
end.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration}
}
if ev.Output != "" && h.RunOutputs != nil {
if err := h.RunOutputs.Put(run, []byte(ev.Output)); err != nil {
return fmt.Errorf("the output of the test run %s cannot be kept: %w", run, err)
}
end.Output = run
}
return h.EndRun(run, end, &seat.TestRunReader{Pipeline: p.Pipeline, Artifact: s.Artifact})
}
// said says what a written pipeline became: an approval asked is a wait held on the operator (ADR 0283), never a stall.
func (h *Hearer) said(p seat.Pipeline, asked, ended bool) error {
if asked {
@@ -344,11 +403,19 @@ const (
budgetIntegration = 50
)
// testCounts is the module's test counts, the sum of its test steps' own counts; nil when no step counted any.
// testCounts is the module's test counts, the sum of its test steps' own counts, each test run counted once however many
// steps read it (§726, A-466i); nil when no step counted any.
func testCounts(steps []seat.StepResult) *seat.TestCounts {
var t seat.TestCounts
counted := false
runs := map[string]bool{}
for _, s := range steps {
if run := sharedRun(s); run != "" && s.Seat != wants.Seat {
if runs[run] {
continue
}
runs[run] = true
}
if s.Tests != nil {
t.Unit, t.Integration, counted = t.Unit+s.Tests.Unit, t.Integration+s.Tests.Integration, true
}
@@ -113,7 +113,8 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop
if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting {
continue
}
if s.Ask != "" && BoundSeat(s.Seat) {
// A step of a test run leaves it: its ask is dropped only with its last reader (§726, leaveRun).
if s.Ask != "" && BoundSeat(s.Seat) && sharedRun(*s) == "" {
drops = append(drops, *s)
}
s.Outcome, s.Why, s.Ended = seat.StepResultOutcomeSuperseded, why, ptrTime(now)
@@ -136,6 +137,9 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop
if len(changed) > 0 {
h.appendUnlogged(key)
}
for _, i := range changed {
h.leaveRun(key, p.Steps[i], why)
}
for _, s := range drops {
if drop != nil {
if err := drop(s.Seat, s.Node, s.Ask, why); err != nil {
@@ -151,6 +151,21 @@ func (s *Starter) share(key string, asks []Asked) []Asked {
}
if len(joined) > 0 {
s.readOthers(key, joined)
// A run that ended within the candidate is read at once (§724): its readers end with its result.
for run, r := range joined {
if r.Ended == nil || s.EndReaders == nil {
continue
}
var readers []seat.TestRunReader
for _, a := range asks {
if a.Run == run {
readers = a.Readers
}
}
if err := s.EndReaders(run, readers, r); err != nil {
s.logf("the steps of %s reading the ended test run %s are not ended: %v", key, run, err)
}
}
}
return out
}
@@ -8,6 +8,7 @@ import (
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
)
// clockMem is the bound keys of test-runs in memory, each with the time to live it was last written with.
@@ -219,3 +220,75 @@ func TestATestRunHasOneClockACapAndAnAge(t *testing.T) {
type agedDeleter struct{ memStore }
func (d agedDeleter) Delete(key string) error { delete(d.memStore, key); return nil }
// How a test run ends and is said (design 47 G §726; A-466i, A-468d, A-469m, A-469z): the run's ask's end ends every step
// reading it with its outcome, counts and shared, its output kept in test-runs-objects; a pipeline counts the run's tests
// once; a step superseded leaves the run to its other readers, and the last to leave drops its ask and ends it could not
// run, no reader left; a composition step's end keeps the runs mesh-wants names.
func TestATestRunsEndMovesEveryReaderAndIsCountedOnce(t *testing.T) {
f := newRunFixture()
a := f.start(t, 4, 9)
b := f.start(t, 6, 9) // the same candidate: it reads a's runs
unit := stepNamed(a, "app:program", "unit")
ended := time.Now()
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
Pipeline: a.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed",
Said: "TestX failed", Output: "--- FAIL: TestX", Tests: &nodebuildgo.TestCounts{Unit: 5}, Ended: &ended}); err != nil {
t.Fatal(err)
}
for _, p := range []seat.Pipeline{f.pipeline(a), f.pipeline(b)} {
for _, art := range []string{"app:program", "app:tools"} {
s := stepNamed(p, art, "unit")
if s.Outcome != seat.StepResultOutcomeFailed || s.Why != "TestX failed" || s.Tests == nil || s.Tests.Unit != 5 ||
len(s.Shared) != 1 || s.Shared[0].Run != unit.Shared[0].Run || s.Shared[0].Ask != unit.Ask {
t.Errorf("%s's %s unit step = %+v, want failed with the run's result and shared", p.Pipeline, art, s)
}
}
if c := testCounts(p.Steps); c == nil || c.Unit != 5 {
t.Errorf("%s counts %+v, want the run's 5 unit tests once", p.Pipeline, c)
}
}
if r, _, _ := f.runs.Get(unit.Shared[0].Run); r.Outcome != "failed" || r.Output != unit.Shared[0].Run || string(f.objects[r.Output]) != "--- FAIL: TestX" {
t.Errorf("the ended run = %+v, objects %v; want its output kept under its key", r, f.objects)
}
saidReaders := 0
for _, e := range f.said {
if e.Result.Step == "unit" && e.Result.Outcome == "failed" {
saidReaders++
}
}
if saidReaders != 4 {
t.Errorf("said %d unit step-ended, want one per reading step", saidReaders)
}
// a, superseded, leaves the lint run to b's readers; b, superseded after, leaves none: the ask is dropped and the run ends.
lint := stepNamed(a, "app:program", "lint")
if _, err := f.h.Supersede(seat.Pipeline{Pipeline: "main.5.app", Branch: "main", Source: 5, Module: "app", Commit: "c2", Fingerprint: "src2:bb"}, nil, nil); err != nil {
t.Fatal(err)
}
if r, _, _ := f.runs.Get(lint.Shared[0].Run); r.Ended != nil || len(r.Readers) != 2 || len(f.drops) != 0 {
t.Fatalf("after a left the lint run = %+v, drops %v; want it running for b's two readers", r, f.drops)
}
if _, err := f.h.Supersede(seat.Pipeline{Pipeline: "main.7.app", Branch: "main", Source: 7, Module: "app", Commit: "c3", Fingerprint: "src2:cc"}, nil, nil); err != nil {
t.Fatal(err)
}
if r, _, _ := f.runs.Get(lint.Shared[0].Run); r.Outcome != seat.StepResultOutcomeCouldNotRun || r.Said != NoReaderLeft || len(f.drops) != 2 || f.drops[0] != "node-build-go "+lint.Ask {
t.Errorf("after the last reader left the lint run = %+v, drops %v; want it ended, no reader left, its ask and the integration run's dropped", r, f.drops)
}
// A composition step's end keeps the runs mesh-wants names, as they came (A-468d).
c := newRunFixture()
p := c.start(t, 4, 9)
comp := stepNamed(p, "", "composition")
bs := stepSeatOf(t, "mesh-wants")
raw := []byte(`{"id":"` + comp.Ask + `","pipeline":"main.4.app","step":"composition","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"passed",` +
`"shared":[{"machine":"ace","run":"main.9.ace","ask":"r1"}]}`)
ev, err := bs.Decode(raw)
if err != nil {
t.Fatal(err)
}
if _, err := c.h.OnStepEndedShared(bs, bs.Prefix+"n1", ev, bs.Shared(raw)); err != nil {
t.Fatal(err)
}
if s := stepNamed(c.pipeline(p), "", "composition"); len(s.Shared) != 1 || s.Shared[0] != (seat.Shared{Machine: "ace", Run: "main.9.ace", Ask: "r1"}) {
t.Errorf("the composition step's shared = %+v, want mesh-wants' as it came", s.Shared)
}
}
@@ -206,11 +206,15 @@ func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, er
}
now := h.now()
var drop []string
var leave []seat.StepResult
for i := range p.Steps {
s := &p.Steps[i]
if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting {
continue
}
if sharedRun(*s) != "" && s.Seat != wants.Seat {
leave = append(leave, *s) // it leaves its test run, dropped only with the last reader (§726)
}
if s.Ask != "" {
drop = append(drop, s.Ask)
}
@@ -223,6 +227,9 @@ func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, er
} else if err != nil {
return seat.CancelAnswer{}, fmt.Errorf("pipeline %s cannot be written: %w", in.Pipeline, err)
}
for _, s := range leave {
h.leaveRun(in.Pipeline, s, "the pipeline "+in.Pipeline+" is cancelled: "+in.Why)
}
if v.Drop != nil {
for _, ask := range drop {
if err := v.Drop(ask); err != nil {