runner: a test run's end moves every step reading it, counted once, dropped only with its last reader (R3)
The run's ask's end now ends each reading step with the run's outcome, counts and shared, its output kept in test-runs-objects; a pipeline counts a run's tests once; a superseded or cancelled step leaves the run, the last to leave dropping its ask and ending it; a composition step keeps the runs mesh-wants names. Builds: design 47 G §726, §725 (output object), §724 (an ended run read), §729; A-466i, A-468d, A-469m, A-469t, A-469z
This commit is contained in:
@@ -114,7 +114,11 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any))
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
_, err = h.OnStepEnded(bs, m.Subject(), ev)
|
||||
var shared []seat.Shared
|
||||
if bs.Shared != nil {
|
||||
shared = bs.Shared(m.Data())
|
||||
}
|
||||
_, err = h.OnStepEndedShared(bs, m.Subject(), ev, shared)
|
||||
switch {
|
||||
case errors.Is(err, pipeline.ErrRefused):
|
||||
logf("%v", err)
|
||||
|
||||
@@ -324,3 +324,57 @@ func (h *Hearer) endReader(reader seat.TestRunReader, run string, r seat.TestRun
|
||||
return h.said(p, asked, ended)
|
||||
}
|
||||
}
|
||||
|
||||
// NoReaderLeft is what a test run whose last reader left ends with (A-469z): its ask is dropped, and a later step of the
|
||||
// candidate starts a new run.
|
||||
const NoReaderLeft = "no reader is left"
|
||||
|
||||
// leaveRun takes the step of pipeline key that read or asked a test run out of the run's readers (§726): a step superseded
|
||||
// or cancelled. When no reader is left the run ends `could not run`, naming that (A-469z), and its ask is dropped (§119);
|
||||
// while one is left the ask stays, so a superseded pipeline never takes away another's run. A step of no test run leaves
|
||||
// nothing.
|
||||
func (h *Hearer) leaveRun(key string, s seat.StepResult, why string) {
|
||||
run := sharedRun(s)
|
||||
if run == "" || h.TestRuns == nil {
|
||||
return
|
||||
}
|
||||
reader := seat.TestRunReader{Pipeline: key, Artifact: s.Artifact}
|
||||
for {
|
||||
r, rev, err := h.TestRuns.Get(run)
|
||||
if errors.Is(err, ErrNoRun) {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
h.logf("the test run %s is not read, so the step %s of %s does not leave it: %v", run, s.Step, key, err)
|
||||
return
|
||||
}
|
||||
left := r.Readers[:0:0]
|
||||
for _, o := range r.Readers {
|
||||
if o != reader {
|
||||
left = append(left, o)
|
||||
}
|
||||
}
|
||||
if len(left) == len(r.Readers) {
|
||||
return
|
||||
}
|
||||
r.Readers = left
|
||||
if _, err := h.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
h.logf("the step %s of %s does not leave the test run %s: %v", s.Step, key, run, err)
|
||||
return
|
||||
}
|
||||
if len(left) > 0 || r.Ended != nil {
|
||||
return
|
||||
}
|
||||
if h.DropRun != nil {
|
||||
if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil {
|
||||
h.logf("the drop of ask %s of the test run %s, which no reader is left of, is not answered: %v", r.Ask, run, err)
|
||||
}
|
||||
}
|
||||
if err := h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: NoReaderLeft}, nil); err != nil {
|
||||
h.logf("the test run %s, which no reader is left of, is not ended: %v", run, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,6 +26,9 @@ type StepSeat struct {
|
||||
Prefix string
|
||||
// Decode reads the seat's step-ended body; nil is the build seats' one form.
|
||||
Decode func([]byte) (StepEnded, error)
|
||||
// Shared reads the runs a step-ended names in `shared`, carried into the step's result as they came (A-468d); nil for
|
||||
// a seat whose ends name none.
|
||||
Shared func([]byte) []seat.Shared
|
||||
}
|
||||
|
||||
// StepSeats are the three build seats, from their contracts.
|
||||
@@ -36,7 +39,25 @@ var StepSeats = []StepSeat{
|
||||
// The seat mesh-builds answers the build step of a candidate, one entry named by its fingerprint (A-97, A-99).
|
||||
{Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded},
|
||||
// The seat mesh-wants answers the composition step (§22, §24).
|
||||
{Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded},
|
||||
{Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded, Shared: wantsShared},
|
||||
}
|
||||
|
||||
// wantsShared is the composition runs a mesh-wants step-ended names, one per machine (design 47 G §723, §729; A-468d,
|
||||
// A-469t): the same form, under the same name.
|
||||
func wantsShared(raw []byte) []seat.Shared {
|
||||
var w wants.StepEnded
|
||||
if json.Unmarshal(raw, &w) != nil || len(w.Shared) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]seat.Shared, 0, len(w.Shared))
|
||||
for _, x := range w.Shared {
|
||||
e := seat.Shared{Machine: x.Machine, Run: x.Run, Ask: x.Ask}
|
||||
if x.AskedBy != nil {
|
||||
e.AskedBy = &seat.SharedAskedBy{Pipeline: x.AskedBy.Pipeline, Artifact: x.AskedBy.Artifact}
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// decodeWantsEnded reads the composition step's end of the seat mesh-wants as the form the hearer holds each end to; a
|
||||
@@ -152,6 +173,13 @@ func (h *Hearer) now() time.Time {
|
||||
// another seat, commit, fingerprint, module or step (ADR 0666 §21; the bus grants step-ended.<node> to that node's
|
||||
// holder alone, so the subject's node is the account's). A step already ended takes nothing (the bus gives an event again).
|
||||
func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pipeline, error) {
|
||||
return h.OnStepEndedShared(bs, subject, ev, nil)
|
||||
}
|
||||
|
||||
// OnStepEndedShared is OnStepEnded of an end naming the runs it read in shared (A-468d): kept on the step's result. The end
|
||||
// of a test run's ask also ends the run, its output kept in test-runs-objects, and every other step reading it (§725, §726,
|
||||
// A-469m).
|
||||
func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared) (seat.Pipeline, error) {
|
||||
node := strings.TrimPrefix(subject, bs.Prefix)
|
||||
if node == subject || node == "" || strings.Contains(node, ".") {
|
||||
return seat.Pipeline{}, fmt.Errorf("%w: %s is not a subject of %s's step-ended", ErrRefused, subject, bs.Seat)
|
||||
@@ -175,6 +203,10 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
|
||||
ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Commit, p.Fingerprint)
|
||||
case p.Steps[i].Outcome != seat.StepResultOutcomeRunning:
|
||||
if run := sharedRun(p.Steps[i]); run != "" && h.TestRuns != nil {
|
||||
if r, _, err := h.TestRuns.Get(run); err == nil && r.Ended == nil && r.Ask == ev.ID {
|
||||
// The asking step's end was written and its run's was not (a write failed): the run ends now, heard again.
|
||||
return p, h.endRunOf(p, p.Steps[i], ev)
|
||||
}
|
||||
// The run ended before its ask's end came (its bound, a drop): the end is ignored and said (§720, §725).
|
||||
h.logf("the step-ended of ask %s is ignored: its test run %s ended before it (%s)", ev.ID, run, p.Steps[i].Why)
|
||||
}
|
||||
@@ -185,6 +217,9 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
|
||||
now := h.now()
|
||||
s := &p.Steps[i]
|
||||
s.Outcome, s.Node, s.Why, s.Rerun, s.By, s.Ended = ev.Outcome, node, ev.Said, ev.Rerun, ev.By, &now
|
||||
if len(shared) > 0 {
|
||||
s.Shared = shared
|
||||
}
|
||||
if ev.Tests != nil {
|
||||
s.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration}
|
||||
}
|
||||
@@ -237,10 +272,34 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
|
||||
h.appendUnlogged(ev.Pipeline)
|
||||
h.Ager.Age()
|
||||
h.Fills.Check()
|
||||
return p, h.said(p, asked, ended)
|
||||
if err := h.said(p, asked, ended); err != nil {
|
||||
return p, err
|
||||
}
|
||||
return p, h.endRunOf(p, *s, ev)
|
||||
}
|
||||
}
|
||||
|
||||
// endRunOf ends the test run step s of p asked, with the end ev of its ask (§725, §726): the run's output kept in
|
||||
// test-runs-objects under the run's key, its outcome, words, counts and node written, and every other reader ended with it.
|
||||
// A step that asked no run ends nothing more.
|
||||
func (h *Hearer) endRunOf(p seat.Pipeline, s seat.StepResult, ev StepEnded) error {
|
||||
run := sharedRun(s)
|
||||
if run == "" || h.TestRuns == nil || s.Ask != ev.ID {
|
||||
return nil
|
||||
}
|
||||
end := seat.TestRun{Outcome: ev.Outcome, Said: ev.Said, Node: ev.Node}
|
||||
if ev.Tests != nil {
|
||||
end.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration}
|
||||
}
|
||||
if ev.Output != "" && h.RunOutputs != nil {
|
||||
if err := h.RunOutputs.Put(run, []byte(ev.Output)); err != nil {
|
||||
return fmt.Errorf("the output of the test run %s cannot be kept: %w", run, err)
|
||||
}
|
||||
end.Output = run
|
||||
}
|
||||
return h.EndRun(run, end, &seat.TestRunReader{Pipeline: p.Pipeline, Artifact: s.Artifact})
|
||||
}
|
||||
|
||||
// said says what a written pipeline became: an approval asked is a wait held on the operator (ADR 0283), never a stall.
|
||||
func (h *Hearer) said(p seat.Pipeline, asked, ended bool) error {
|
||||
if asked {
|
||||
@@ -344,11 +403,19 @@ const (
|
||||
budgetIntegration = 50
|
||||
)
|
||||
|
||||
// testCounts is the module's test counts, the sum of its test steps' own counts; nil when no step counted any.
|
||||
// testCounts is the module's test counts, the sum of its test steps' own counts, each test run counted once however many
|
||||
// steps read it (§726, A-466i); nil when no step counted any.
|
||||
func testCounts(steps []seat.StepResult) *seat.TestCounts {
|
||||
var t seat.TestCounts
|
||||
counted := false
|
||||
runs := map[string]bool{}
|
||||
for _, s := range steps {
|
||||
if run := sharedRun(s); run != "" && s.Seat != wants.Seat {
|
||||
if runs[run] {
|
||||
continue
|
||||
}
|
||||
runs[run] = true
|
||||
}
|
||||
if s.Tests != nil {
|
||||
t.Unit, t.Integration, counted = t.Unit+s.Tests.Unit, t.Integration+s.Tests.Integration, true
|
||||
}
|
||||
|
||||
@@ -113,7 +113,8 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop
|
||||
if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting {
|
||||
continue
|
||||
}
|
||||
if s.Ask != "" && BoundSeat(s.Seat) {
|
||||
// A step of a test run leaves it: its ask is dropped only with its last reader (§726, leaveRun).
|
||||
if s.Ask != "" && BoundSeat(s.Seat) && sharedRun(*s) == "" {
|
||||
drops = append(drops, *s)
|
||||
}
|
||||
s.Outcome, s.Why, s.Ended = seat.StepResultOutcomeSuperseded, why, ptrTime(now)
|
||||
@@ -136,6 +137,9 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop
|
||||
if len(changed) > 0 {
|
||||
h.appendUnlogged(key)
|
||||
}
|
||||
for _, i := range changed {
|
||||
h.leaveRun(key, p.Steps[i], why)
|
||||
}
|
||||
for _, s := range drops {
|
||||
if drop != nil {
|
||||
if err := drop(s.Seat, s.Node, s.Ask, why); err != nil {
|
||||
|
||||
@@ -151,6 +151,21 @@ func (s *Starter) share(key string, asks []Asked) []Asked {
|
||||
}
|
||||
if len(joined) > 0 {
|
||||
s.readOthers(key, joined)
|
||||
// A run that ended within the candidate is read at once (§724): its readers end with its result.
|
||||
for run, r := range joined {
|
||||
if r.Ended == nil || s.EndReaders == nil {
|
||||
continue
|
||||
}
|
||||
var readers []seat.TestRunReader
|
||||
for _, a := range asks {
|
||||
if a.Run == run {
|
||||
readers = a.Readers
|
||||
}
|
||||
}
|
||||
if err := s.EndReaders(run, readers, r); err != nil {
|
||||
s.logf("the steps of %s reading the ended test run %s are not ended: %v", key, run, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
)
|
||||
|
||||
// clockMem is the bound keys of test-runs in memory, each with the time to live it was last written with.
|
||||
@@ -219,3 +220,75 @@ func TestATestRunHasOneClockACapAndAnAge(t *testing.T) {
|
||||
type agedDeleter struct{ memStore }
|
||||
|
||||
func (d agedDeleter) Delete(key string) error { delete(d.memStore, key); return nil }
|
||||
|
||||
// How a test run ends and is said (design 47 G §726; A-466i, A-468d, A-469m, A-469z): the run's ask's end ends every step
|
||||
// reading it with its outcome, counts and shared, its output kept in test-runs-objects; a pipeline counts the run's tests
|
||||
// once; a step superseded leaves the run to its other readers, and the last to leave drops its ask and ends it could not
|
||||
// run, no reader left; a composition step's end keeps the runs mesh-wants names.
|
||||
func TestATestRunsEndMovesEveryReaderAndIsCountedOnce(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
a := f.start(t, 4, 9)
|
||||
b := f.start(t, 6, 9) // the same candidate: it reads a's runs
|
||||
unit := stepNamed(a, "app:program", "unit")
|
||||
ended := time.Now()
|
||||
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
|
||||
Pipeline: a.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed",
|
||||
Said: "TestX failed", Output: "--- FAIL: TestX", Tests: &nodebuildgo.TestCounts{Unit: 5}, Ended: &ended}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, p := range []seat.Pipeline{f.pipeline(a), f.pipeline(b)} {
|
||||
for _, art := range []string{"app:program", "app:tools"} {
|
||||
s := stepNamed(p, art, "unit")
|
||||
if s.Outcome != seat.StepResultOutcomeFailed || s.Why != "TestX failed" || s.Tests == nil || s.Tests.Unit != 5 ||
|
||||
len(s.Shared) != 1 || s.Shared[0].Run != unit.Shared[0].Run || s.Shared[0].Ask != unit.Ask {
|
||||
t.Errorf("%s's %s unit step = %+v, want failed with the run's result and shared", p.Pipeline, art, s)
|
||||
}
|
||||
}
|
||||
if c := testCounts(p.Steps); c == nil || c.Unit != 5 {
|
||||
t.Errorf("%s counts %+v, want the run's 5 unit tests once", p.Pipeline, c)
|
||||
}
|
||||
}
|
||||
if r, _, _ := f.runs.Get(unit.Shared[0].Run); r.Outcome != "failed" || r.Output != unit.Shared[0].Run || string(f.objects[r.Output]) != "--- FAIL: TestX" {
|
||||
t.Errorf("the ended run = %+v, objects %v; want its output kept under its key", r, f.objects)
|
||||
}
|
||||
saidReaders := 0
|
||||
for _, e := range f.said {
|
||||
if e.Result.Step == "unit" && e.Result.Outcome == "failed" {
|
||||
saidReaders++
|
||||
}
|
||||
}
|
||||
if saidReaders != 4 {
|
||||
t.Errorf("said %d unit step-ended, want one per reading step", saidReaders)
|
||||
}
|
||||
// a, superseded, leaves the lint run to b's readers; b, superseded after, leaves none: the ask is dropped and the run ends.
|
||||
lint := stepNamed(a, "app:program", "lint")
|
||||
if _, err := f.h.Supersede(seat.Pipeline{Pipeline: "main.5.app", Branch: "main", Source: 5, Module: "app", Commit: "c2", Fingerprint: "src2:bb"}, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r, _, _ := f.runs.Get(lint.Shared[0].Run); r.Ended != nil || len(r.Readers) != 2 || len(f.drops) != 0 {
|
||||
t.Fatalf("after a left the lint run = %+v, drops %v; want it running for b's two readers", r, f.drops)
|
||||
}
|
||||
if _, err := f.h.Supersede(seat.Pipeline{Pipeline: "main.7.app", Branch: "main", Source: 7, Module: "app", Commit: "c3", Fingerprint: "src2:cc"}, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r, _, _ := f.runs.Get(lint.Shared[0].Run); r.Outcome != seat.StepResultOutcomeCouldNotRun || r.Said != NoReaderLeft || len(f.drops) != 2 || f.drops[0] != "node-build-go "+lint.Ask {
|
||||
t.Errorf("after the last reader left the lint run = %+v, drops %v; want it ended, no reader left, its ask and the integration run's dropped", r, f.drops)
|
||||
}
|
||||
// A composition step's end keeps the runs mesh-wants names, as they came (A-468d).
|
||||
c := newRunFixture()
|
||||
p := c.start(t, 4, 9)
|
||||
comp := stepNamed(p, "", "composition")
|
||||
bs := stepSeatOf(t, "mesh-wants")
|
||||
raw := []byte(`{"id":"` + comp.Ask + `","pipeline":"main.4.app","step":"composition","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"passed",` +
|
||||
`"shared":[{"machine":"ace","run":"main.9.ace","ask":"r1"}]}`)
|
||||
ev, err := bs.Decode(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := c.h.OnStepEndedShared(bs, bs.Prefix+"n1", ev, bs.Shared(raw)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s := stepNamed(c.pipeline(p), "", "composition"); len(s.Shared) != 1 || s.Shared[0] != (seat.Shared{Machine: "ace", Run: "main.9.ace", Ask: "r1"}) {
|
||||
t.Errorf("the composition step's shared = %+v, want mesh-wants' as it came", s.Shared)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -206,11 +206,15 @@ func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, er
|
||||
}
|
||||
now := h.now()
|
||||
var drop []string
|
||||
var leave []seat.StepResult
|
||||
for i := range p.Steps {
|
||||
s := &p.Steps[i]
|
||||
if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting {
|
||||
continue
|
||||
}
|
||||
if sharedRun(*s) != "" && s.Seat != wants.Seat {
|
||||
leave = append(leave, *s) // it leaves its test run, dropped only with the last reader (§726)
|
||||
}
|
||||
if s.Ask != "" {
|
||||
drop = append(drop, s.Ask)
|
||||
}
|
||||
@@ -223,6 +227,9 @@ func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, er
|
||||
} else if err != nil {
|
||||
return seat.CancelAnswer{}, fmt.Errorf("pipeline %s cannot be written: %w", in.Pipeline, err)
|
||||
}
|
||||
for _, s := range leave {
|
||||
h.leaveRun(in.Pipeline, s, "the pipeline "+in.Pipeline+" is cancelled: "+in.Why)
|
||||
}
|
||||
if v.Drop != nil {
|
||||
for _, ask := range drop {
|
||||
if err := v.Drop(ask); err != nil {
|
||||
|
||||
Reference in New Issue
Block a user