runner: a test run's steps are said waiting for a build node until the holder takes the run's ask (A-469b)

A test step queued on a build seat was said running, so the watchman's build-capacity waits never saw
it; every step of a test run now waits for a build node from the run's ask, and at the holder's
ask-taken each is said started again, running.

Builds: design 47 G §725, §732, §86; A-469b
This commit is contained in:
2026-11-17 11:48:28 +01:00
parent 7923ab63b1
commit ebe49b9bdd
11 changed files with 80 additions and 17 deletions
@@ -38,7 +38,7 @@ func approvalDue(p seat.Pipeline) int {
}
continue
}
if s.Outcome == seat.StepResultOutcomeWaiting || s.Outcome == seat.StepResultOutcomeRunning {
if s.Outcome == seat.StepResultOutcomeWaiting || open(s) {
return -1
}
if s.Status == seat.StepResultStatusRequired && s.Outcome != seat.StepResultOutcomePassed && s.Outcome != seat.StepResultOutcomeCarried {
@@ -139,12 +139,13 @@ func place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog
// Another artifact's step of this pipeline asks the run: this one reads it.
first := &asks[j]
first.Readers = append(first.Readers, reader)
s.Ask = ""
s.Ask, s.Outcome, s.On = "", seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode
s.Shared = []seat.Shared{{Run: run, Ask: first.ID, AskedBy: &seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: first.Readers[0].Artifact}}}
steps = append(steps, s)
continue
}
s.Shared = []seat.Shared{{Run: run, Ask: s.Ask}}
// A test run's steps wait for a build node from its ask until the holder's ask-taken (A-469b).
s.Shared, s.Outcome, s.On = []seat.Shared{{Run: run, Ask: s.Ask}}, seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode
ask.Run, ask.Level, ask.Digest, ask.Readers = run, step, digest, []seat.TestRunReader{reader}
askOfRun[run] = len(asks)
}
@@ -407,7 +407,7 @@ func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) {
if asker.Ask == "" || len(asker.Shared) != 1 || asker.Shared[0].Ask != asker.Ask || asker.Shared[0].AskedBy != nil {
t.Errorf("the asking %s step = %+v, want its ask named in shared with no asked-by", level, asker)
}
if reader.Ask != "" || reader.Outcome != seat.StepResultOutcomeRunning || len(reader.Shared) != 1 || reader.Shared[0] != (seat.Shared{
if reader.Ask != "" || reader.Outcome != seat.StepResultOutcomeWaitingForABuildNode || reader.On != seat.WaitOnBuildNode || len(reader.Shared) != 1 || reader.Shared[0] != (seat.Shared{
Run: asker.Shared[0].Run, Ask: asker.Ask, AskedBy: reader.Shared[0].AskedBy}) || reader.Shared[0].AskedBy == nil ||
*reader.Shared[0].AskedBy != (seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: "app:program"}) {
t.Errorf("the reading %s step = %+v, want no ask and the program's run named", level, reader)
@@ -95,7 +95,7 @@ func (r Reads) Stands() (seat.StandsAnswer, error) {
if s.Ended != nil || s.Started == nil {
continue
}
if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting {
if !open(s) && s.Outcome != seat.StepResultOutcomeWaiting {
continue
}
out = append(out, seat.Wait{Pipeline: p.Pipeline, Module: p.Module, Branch: p.Branch, Commit: p.Commit,
@@ -116,6 +116,7 @@ func (h *Hearer) RunTaken(ask, node string) error {
return fmt.Errorf("the test run %s is not written: %w", run, err)
}
}
h.runStarted(run, r)
break
}
if err := h.Clock.Bound(BoundPrefix+run, h.runSettings().StepBound); err != nil {
@@ -296,7 +297,7 @@ func (h *Hearer) endReader(reader RunReader, run string, r Run) error {
if s.Artifact != reader.Artifact || sharedRun(s) != run {
continue
}
if s.Outcome == seat.StepResultOutcomeRunning {
if open(s) {
i, moved = j, false
} else if reader.Rerun && s.Shared[0].Ask != r.Ask && s.Accepted == nil {
i, moved = j, true
@@ -392,3 +393,44 @@ func (h *Hearer) leaveRun(key string, s seat.StepResult, why string) {
return
}
}
// open says whether a step is asked and not ended: running, or a test step waiting for a build node to take its run's ask
// (A-469b).
func open(s seat.StepResult) bool {
return s.Outcome == seat.StepResultOutcomeRunning || s.Outcome == seat.StepResultOutcomeWaitingForABuildNode
}
// runStarted moves each step reading run from waiting for a build node to running, the holder's ask-taken heard (A-469b),
// and says its step-started again, so the watchman's build-capacity waits let it go (§86, §732).
func (h *Hearer) runStarted(run string, r Run) {
for _, reader := range r.Readers {
for {
p, rev, err := h.Store.Get(reader.Pipeline)
if err != nil {
break
}
var moved []int
for j, s := range p.Steps {
if s.Artifact == reader.Artifact && sharedRun(s) == run && s.Outcome == seat.StepResultOutcomeWaitingForABuildNode {
p.Steps[j].Outcome, p.Steps[j].On = seat.StepResultOutcomeRunning, ""
moved = append(moved, j)
}
}
if len(moved) == 0 {
break
}
if _, err := h.Store.Update(reader.Pipeline, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
h.logf("the steps of %s reading the taken test run %s are not written running: %v", reader.Pipeline, run, err)
break
}
for _, j := range moved {
if err := h.say(seat.EventStepStarted, p, &p.Steps[j]); err != nil {
h.logf("step-started of %s's %s is not said: %v", reader.Pipeline, p.Steps[j].Step, err)
}
}
break
}
}
}
@@ -105,7 +105,7 @@ func (s *Starter) holderless(steps []seat.StepResult, asks []Asked, now time.Tim
none := map[string]bool{}
for i := range steps {
st := &steps[i]
if st.Outcome != seat.StepResultOutcomeRunning || !BoundSeat(st.Seat) {
if !open(*st) || !BoundSeat(st.Seat) {
continue
}
if holders, read := s.Holders(st.Seat); !read || len(holders) > 0 {
@@ -162,7 +162,7 @@ func (s *Starter) sayStarted(key string, asks []Asked) {
}
at := s.now()
for _, st := range p.Steps {
if st.Outcome != seat.StepResultOutcomeRunning || !(ids[st.Ask] || (st.Ask == "" && runs[sharedRun(st)])) {
if !open(st) || !(ids[st.Ask] || (st.Ask == "" && runs[sharedRun(st)])) {
continue
}
subject, ev := stepEvent(seat.EventStepStarted, p, st, at)
@@ -278,7 +278,7 @@ func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) {
// runs and keeps its own ask, ending by the path it was asked under (§730, A-468k).
running, own := map[string]bool{}, map[string]bool{}
for _, st := range p.Steps {
if st.Outcome == seat.StepResultOutcomeRunning && st.Ask != "" {
if open(st) && st.Ask != "" {
running[st.Ask], own[st.Ask] = true, len(st.Shared) == 0
}
}
@@ -201,7 +201,7 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh
case ev.Commit != p.Commit || ev.Fingerprint != p.Fingerprint || ev.Module != p.Module || ev.Step != p.Steps[i].Step:
return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s at %s (%s)", ErrRefused, ev.ID, ev.Step,
ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Commit, p.Fingerprint)
case p.Steps[i].Outcome != seat.StepResultOutcomeRunning:
case !open(p.Steps[i]):
if run := sharedRun(p.Steps[i]); run != "" && h.TestRuns != nil {
if r, _, err := h.TestRuns.Get(run); err == nil && r.Ended == nil && r.Ask == ev.ID {
// The asking step's end was written and its run's was not (a write failed): the run ends now, heard again.
@@ -310,7 +310,7 @@ func (h *Hearer) sameRunReaders(p *seat.Pipeline, i int, ev StepEnded, now time.
continue
}
switch m := marks[s.Artifact]; {
case s.Outcome == seat.StepResultOutcomeRunning && s.Ask == "":
case open(*s) && s.Ask == "":
case m.Rerun && s.Shared[0].Ask != ev.ID && s.Accepted == nil:
s.Rerun, s.By = true, m.By
s.Shared = []seat.Shared{{Run: run, Ask: ev.ID, AskedBy: &seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: asker.Artifact}}}
@@ -550,7 +550,7 @@ func endPipeline(p *seat.Pipeline, now time.Time) bool {
// An optional step still waiting (a code review nobody has given) holds nothing: only a required one holds the
// pipeline's end (A-104).
continue
case s.Outcome == seat.StepResultOutcomeWaiting, s.Outcome == seat.StepResultOutcomeRunning:
case s.Outcome == seat.StepResultOutcomeWaiting, open(s):
return false
}
if s.Status == seat.StepResultStatusRequired && s.Outcome != seat.StepResultOutcomePassed && s.Outcome != seat.StepResultOutcomeCarried {
@@ -30,7 +30,7 @@ func (h *Hearer) Taken(ask, node string) error {
break
}
i := stepOfAsk(p, ask)
if i < 0 || p.Steps[i].Outcome != seat.StepResultOutcomeRunning || p.Steps[i].Node == node {
if i < 0 || !open(p.Steps[i]) || p.Steps[i].Node == node {
break
}
p.Steps[i].Node = node
@@ -110,7 +110,7 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop
var changed []int
for i := range p.Steps {
s := &p.Steps[i]
if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting {
if !open(*s) && s.Outcome != seat.StepResultOutcomeWaiting {
continue
}
// A step of a test run leaves it: its ask is dropped only with its last reader (§726, leaveRun).
@@ -321,9 +321,12 @@ func (s *Starter) readOthers(key string, joined map[string]Run) {
for i := range p.Steps {
st := &p.Steps[i]
r, ok := joined[sharedRun(*st)]
if !ok || st.Outcome != seat.StepResultOutcomeRunning {
if !ok || !open(*st) {
continue
}
if r.Node != "" {
st.Outcome, st.On = seat.StepResultOutcomeRunning, "" // its run's ask is taken already (A-469b)
}
st.Ask = ""
st.Shared = []seat.Shared{{Run: sharedRun(*st), Ask: r.Ask, AskedBy: by[sharedRun(*st)]}}
}
@@ -114,9 +114,22 @@ func TestATestRunHasOneClockACapAndAnAge(t *testing.T) {
if f.clock.bound[UntakenPrefix+run] != 24*time.Hour || len(f.clock.bound) != 3 {
t.Fatalf("bounds at the start = %v, want the three runs' untaken bounds of 24h", f.clock.bound)
}
// A-469b: the run's steps wait for a build node until the holder's ask-taken, then run, each said started again.
if unit.Outcome != seat.StepResultOutcomeWaitingForABuildNode || stepNamed(p, "app:tools", "unit").Outcome != seat.StepResultOutcomeWaitingForABuildNode {
t.Fatalf("the unit steps before ask-taken = %s, %s", unit.Outcome, stepNamed(p, "app:tools", "unit").Outcome)
}
n := len(f.said)
if err := f.h.RunTaken(unit.Ask, "ace"); err != nil {
t.Fatal(err)
}
for _, art := range []string{"app:program", "app:tools"} {
if s := stepNamed(f.pipeline(p), art, "unit"); s.Outcome != seat.StepResultOutcomeRunning {
t.Errorf("the %s unit step after ask-taken is %s, want running", art, s.Outcome)
}
}
if len(f.said)-n != 2 || f.said[n].Result.Outcome != seat.StepResultOutcomeRunning {
t.Errorf("ask-taken said %d step events, want the two unit steps started running", len(f.said)-n)
}
if r, _, _ := f.runs.Get(run); r.Node != "ace" || f.clock.bound[BoundPrefix+run] != 30*time.Minute {
t.Fatalf("after ask-taken the run = %+v, bounds %v; want its node and its clock of 30m", r, f.clock.bound)
}
@@ -206,7 +206,7 @@ func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, er
var leave []seat.StepResult
for i := range p.Steps {
s := &p.Steps[i]
if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting {
if !open(*s) && s.Outcome != seat.StepResultOutcomeWaiting {
continue
}
if sharedRun(*s) != "" && s.Seat != wants.Seat {
@@ -306,7 +306,7 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult) (seat.RerunAnswer, error) {
h := v.Hearer
run := sharedRun(st)
if st.Outcome == seat.StepResultOutcomeRunning {
if open(st) {
return seat.RerunAnswer{}, fmt.Errorf("the %s step of %s is running: it is not asked again", in.Step, in.Pipeline)
}
r, rev, err := h.TestRuns.Get(run)
@@ -357,6 +357,7 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult)
switch {
case ask != nil:
s.Ask, s.Shared = ask.ID, []seat.Shared{{Run: run, Ask: ask.ID}}
s.Outcome, s.On = seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode // until ask-taken (A-469b)
case read:
s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: asker}}
s.Outcome, s.Why, s.Node, s.Ended = r.Outcome, r.Said, r.Node, &now
@@ -366,6 +367,9 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult)
}
default:
s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: asker}}
if r.Node == "" {
s.Outcome, s.On = seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode
}
}
if p.Outcome != seat.PipelineOutcomeRunning {
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeRunning, "", nil