Design 47 H: a pipeline with nothing open ends at once; the end's reads are made once (H-A-288, H-A-289) #315

Merged
mesh-tools merged 1 commits from fix/47-H1-runner-end-when-empty into main 2026-11-21 17:22:44 +00:00
9 changed files with 289 additions and 21 deletions
@@ -150,7 +150,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
logger.Printf("widens: each read bound %s, %d catalog revisions kept", readBound, revisionsKept)
widener := widenerOf(calls.Call, readBound, revisionsKept, logger.Printf)
hearer.Widens = widener.Widens
hearer.Widens = widener.Reads
hearer.LogLast = stepLog{ctx: ctx, js: js, module: cred.Module}.Last
hearer.Where = routerRecordOf(cred.Module, calls.Call, calls.Ask)
hearer.Failures = &pipeline.Failures{Findings: rv.Findings, Observer: cred.Module, Author: commitAuthor(calls.Call), Logf: logger.Printf}
@@ -274,6 +274,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err := placeLostRuns(starter.UnplacedRuns, calls.CallKept, placeHalf, logger.Printf); err != nil {
logger.Printf("the runs not placed at the start: %v", err)
}
// A running pipeline nothing of which is still open ends now, its turn freed, before the queue is placed (H-A-288 (c)).
if n, err := hearer.EndAtStart(); err != nil || n > 0 {
logger.Printf("the pipelines with nothing open at the start: %d ended (%v)", n, err)
}
// The queue is placed at the start from its own state (§748).
if n, err := starter.StartQueue(rv.Catalog.View); err != nil || n > 0 {
logger.Printf("the queue at the start: %d waiting pipelines placed (%v)", n, err)
@@ -206,6 +206,7 @@ func (h *Hearer) OnDecided(event oc.Warrant) (seat.Pipeline, error) {
return seat.Pipeline{}, err
}
for _, key := range found {
wr := h.widensReads() // made before the compare-and-set, kept for its retries (H-A-289)
for {
p, rev, err := h.Store.Get(key)
if err != nil {
@@ -219,7 +220,7 @@ func (h *Hearer) OnDecided(event oc.Warrant) (seat.Pipeline, error) {
if err != nil || !changed {
return p, err
}
ended := h.endPipeline(&p, now)
ended := h.endPipeline(&p, now, wr)
i := stepOfAsk(p, w.Ask)
if i >= 0 {
p.Unlogged = append(p.Unlogged, endEntry(key, p.Steps[i], now))
@@ -308,6 +308,7 @@ func markRerun(a *Asked, caller string) {
// other mover's validate reading the ask with it, a composition its validate held ended not asked, the pipeline ended when
// nothing else is open, said and logged as any step's end.
func (h *Hearer) endUnanswered(key, ask, why string) error {
wr := h.widensReads() // made before the compare-and-set, kept for its retries (H-A-289)
for {
p, rev, err := h.Store.Get(key)
if err != nil {
@@ -331,7 +332,7 @@ func (h *Hearer) endUnanswered(key, ask, why string) error {
if err != nil {
return err
}
ended := h.endPipeline(&p, now)
ended := h.endPipeline(&p, now, wr)
if _, err := h.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
@@ -0,0 +1,129 @@
package pipeline
import (
"fmt"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshgrants"
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// refusedOnce is a store whose first write of a pipeline is refused as changed, another write having come between (the
// pipeline's key moving while its end is judged, as main.117's did).
type refusedOnce struct {
memStore
refused map[string]bool
}
func (r *refusedOnce) Update(key string, p seat.Pipeline, rev uint64) (uint64, error) {
if !r.refused[key] {
r.refused[key] = true
e := r.memStore[key]
e.rev++
r.memStore[key] = e
return 0, ErrChanged
}
return r.memStore.Update(key, p, rev)
}
// H-A-288, H-A-289 (met with main.117 and main.76): a pipeline ends, and the next waiting is placed, the moment nothing of it is
// still open: when its last mover leaves, when its last open step ends by the silent bound, and at the runner's start pass;
// its outcome is said in the journal. The reads its end needs are made once, before the compare-and-set, shared by every mover
// and every retry of the write: a write refused once is written at the retry without asking grants-for again.
func TestAPipelineWithNothingOpenEndsAtOnce(t *testing.T) {
t0 := time.Date(2026, 11, 21, 17, 45, 19, 0, time.UTC)
movers := []seat.Mover{{Module: "a", Commit: "c1", Fingerprint: "src2:a1"}, {Module: "b", Commit: "c1", Fingerprint: "src2:b1"},
{Module: "c", Commit: "c1", Fingerprint: "src2:c1"}}
step := func(module, outcome, ask string) seat.StepResult {
return seat.StepResult{Module: module, Step: seat.StepResultStepBuild, Status: seat.StepResultStatusRequired, Outcome: outcome, Ask: ask, Seat: builds.Seat}
}
rows := []struct {
name string
steps []seat.StepResult
end func(t *testing.T, h *Hearer)
outcome string
}{
{"its last mover leaves for a newer pipeline (H-A-223)",
[]seat.StepResult{step("a", seat.StepResultOutcomeRunning, "x1"), step("b", seat.StepResultOutcomePassed, "x2"), step("c", seat.StepResultOutcomePassed, "x3")},
func(t *testing.T, h *Hearer) {
h.SupersedeRunning(seat.Pipeline{Pipeline: "main.120", Branch: "main", Revision: 120, Movers: []seat.Mover{
{Module: "a", Commit: "c2", Fingerprint: "src2:a2"}, {Module: "b", Commit: "c2", Fingerprint: "src2:b2"}, {Module: "c", Commit: "c2", Fingerprint: "src2:c2"}}})
}, seat.PipelineOutcomeSuperseded},
{"its last open step ends could not run on the silent bound (H-A-243)",
[]seat.StepResult{step("a", seat.StepResultOutcomeRunning, "mwql5pio3390aa"), step("b", seat.StepResultOutcomePassed, "x2"), step("c", seat.StepResultOutcomePassed, "x3")},
func(t *testing.T, h *Hearer) {
if err := h.endUnanswered("main.117", "mwql5pio3390aa", SilentWhy("mwql5pio3390aa")); err != nil {
t.Fatal(err)
}
}, seat.PipelineOutcomeFailed},
{"the runner's start pass finds it with nothing open",
[]seat.StepResult{step("a", seat.StepResultOutcomePassed, "x1"), step("b", seat.StepResultOutcomePassed, "x2"), step("c", seat.StepResultOutcomePassed, "x3")},
func(t *testing.T, h *Hearer) {
if n, err := h.EndAtStart(); n != 1 || err != nil {
t.Fatalf("the start pass ended %d (%v), want 1", n, err)
}
}, seat.PipelineOutcomePassed},
}
for _, r := range rows {
t.Run(r.name, func(t *testing.T) {
store := &refusedOnce{memStore: memStore{}, refused: map[string]bool{}}
store.memStore["main.117"] = struct {
p seat.Pipeline
rev uint64
}{seat.Pipeline{V: 1, Pipeline: "main.117", Branch: "main", Source: 117, Revision: 117, Outcome: seat.PipelineOutcomeRunning,
Movers: slices.Clone(movers), Steps: slices.Clone(r.steps)}, 1}
// Another running pipeline with a step open, and one waiting its turn: neither ends.
store.memStore["main.118"] = struct {
p seat.Pipeline
rev uint64
}{seat.Pipeline{V: 1, Pipeline: "main.118", Branch: "other", Source: 118, Revision: 118, Outcome: seat.PipelineOutcomeRunning,
Movers: movers[:1], Steps: []seat.StepResult{step("a", seat.StepResultOutcomeRunning, "y1")}}, 1}
store.memStore["main.119"] = struct {
p seat.Pipeline
rev uint64
}{seat.Pipeline{V: 1, Pipeline: "main.119", Branch: "other", Source: 119, Revision: 119, Outcome: seat.PipelineOutcomeWaiting, Movers: movers[:1]}, 1}
store.refused["main.118"], store.refused["main.119"] = true, true
grants, revisions := 0, 0
w := &Widener{
Revision: func(string, int64) (Half, error) { revisions++; return Half{}, nil },
Holder: func(s string) (string, error) { return s, nil },
GrantsFor: func(string, int64) (meshgrants.GrantsFor, error) {
grants++
return meshgrants.GrantsFor{}, fmt.Errorf("cut after 5s")
},
}
var turned, journal []string
h := &Hearer{Store: store, Say: func(string, any) error { return nil }, Now: func() time.Time { return t0 }, Widens: w.Reads,
Turn: func(key string) { turned = append(turned, key) },
Logf: func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }}
r.end(t, h)
p, _, _ := store.Get("main.117")
if p.Outcome != r.outcome {
t.Fatalf("main.117 = %s (%s), want %s", p.Outcome, p.Why, r.outcome)
}
if !slices.Equal(turned, []string{"main.117"}) {
t.Errorf("the turns freed = %v, want main.117's alone, so the next waiting is placed", turned)
}
if !slices.ContainsFunc(journal, func(l string) bool { return strings.HasPrefix(l, "main.117 ended "+r.outcome) }) {
t.Errorf("the journal does not say main.117's end %s: %q", r.outcome, journal)
}
if grants != 1 || revisions != 2 {
t.Errorf("the end asked grants-for %d times and a revision %d times over 3 movers and a refused write, want 1 and 2", grants, revisions)
}
for _, m := range p.Movers {
if len(m.Widens) != 1 || m.Widens[0].Name != "mesh-grants" {
t.Errorf("%s's widens = %+v, want the cut grants-for said", m.Module, m.Widens)
}
}
for _, k := range []string{"main.118", "main.119"} {
if q, _, _ := store.Get(k); q.Outcome == seat.PipelineOutcomeSuperseded || q.Outcome == seat.PipelineOutcomePassed || q.Outcome == seat.PipelineOutcomeFailed {
t.Errorf("%s ended %s with a step open or waiting its turn", k, q.Outcome)
}
}
})
}
}
@@ -154,8 +154,9 @@ type Hearer struct {
Ager *Ager
// Fills says the stores' fill at each step's end (§366); nil says none.
Fills *Fills
// Widens computes a mover's widens, kept in its value at the pipeline's end (§71, A-207; H-A-51); nil keeps none.
Widens func(p seat.Pipeline, module string) []seat.Widening
// Widens makes, for one end, the reader of each mover's widens, kept in its value at the pipeline's end (§71, A-207;
// H-A-51): made before the end's compare-and-set, its reads shared by every mover and every retry (H-A-289); nil keeps none.
Widens func() func(p seat.Pipeline, module string) []seat.Widening
// TestRuns is the state test-runs and Clock its bound keys (§725); Runs the settings of its runs; DropRun drops a
// run's ask when its run ends unfinished (§119). A nil TestRuns keeps no test run.
TestRuns TestRuns
@@ -245,6 +246,7 @@ func (h *Hearer) OnStepEndedPinned(bs StepSeat, subject string, ev StepEnded, sh
if ev.Asker != "" && ev.Asker != h.Name {
return seat.Pipeline{}, nil
}
wr := h.widensReads() // made before the compare-and-set, kept for its retries (H-A-289)
for {
p, rev, err := h.Store.Get(ev.Pipeline)
if err != nil {
@@ -337,7 +339,7 @@ func (h *Hearer) OnStepEndedPinned(bs StepSeat, subject string, ev StepEnded, sh
// A mover's build passed is what a kind-3 edge waits for (§746, H-A-83 (2)); one that did not pass ends its
// dependents' steps not asked.
release := s.Step == seat.StepResultStepBuild && (!stepPassed(*s) || built(&p, s.Module))
ended := h.endPipeline(&p, now)
ended := h.endPipeline(&p, now, wr)
if _, err := h.Store.Update(ev.Pipeline, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
@@ -477,6 +479,12 @@ func (h *Hearer) said(p seat.Pipeline, asked []int, ended bool) error {
h.Turn(p.Pipeline)
}
if ended {
// The journal says which end it was (H-A-288).
if p.Why != "" {
h.logf("%s ended %s: %s", p.Pipeline, p.Outcome, p.Why)
} else {
h.logf("%s ended %s", p.Pipeline, p.Outcome)
}
h.Failures.Ended(p)
if h.RegistrationWaits != nil {
h.RegistrationWaits()
@@ -493,6 +501,51 @@ func (h *Hearer) said(p seat.Pipeline, asked []int, ended bool) error {
return nil
}
// EndAtStart judges, at the runner's start pass, the end of every running pipeline nothing of which is still open (H-A-288
// (c)): one whose last mover left or whose last open step ended while no runner wrote its end ends as at a step's end,
// superseded, failed or passed by the end judgement, said, and the next waiting placed. It answers how many it ended.
func (h *Hearer) EndAtStart() (int, error) {
pkeys, err := allKeys(h.Store)
if err != nil {
return 0, fmt.Errorf("the pipelines are not listed: %w", err)
}
n := 0
var first error
for _, key := range pkeys {
ended, err := h.endIfNothingOpen(key)
if err != nil && first == nil {
first = err
}
if ended {
n++
}
}
return n, first
}
// endIfNothingOpen ends the running pipeline key when nothing of it is still open (H-A-288), its reads made before the
// compare-and-set (H-A-289); true when it ended.
func (h *Hearer) endIfNothingOpen(key string) (bool, error) {
wr := h.widensReads()
for {
p, rev, err := h.Store.Get(key)
if err != nil || p.Outcome != seat.PipelineOutcomeRunning {
return false, nil
}
now := h.now()
if !h.endPipeline(&p, now, wr) {
return false, nil
}
if _, err := h.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return false, fmt.Errorf("pipeline %s cannot be written: %w", key, err)
}
h.logf("%s had nothing open at the runner's start: its end is written now (H-A-288)", key)
return true, h.said(p, nil, true)
}
}
// endEntry is the step log's entry of a step's end, with the keys of the kept results it wrote or carried.
func endEntry(key string, s seat.StepResult, at time.Time, kept ...string) seat.LogEntry {
return seat.LogEntry{Pipeline: key, Module: s.Module, Step: s.Step, Artifact: s.Artifact, What: seat.LogEntryWhatEnded, Outcome: s.Outcome,
@@ -729,19 +782,28 @@ func endPipeline(p *seat.Pipeline, now time.Time, unplaced func(seat.Pipeline, s
return true
}
// endPipeline is endPipeline with each mover's widens computed into its value at the end (§71, A-207; H-A-51).
func (h *Hearer) endPipeline(p *seat.Pipeline, now time.Time) bool {
// endPipeline is endPipeline with each mover's widens computed into its value at the end (§71, A-207; H-A-51), by wr, the
// reads the caller made before its compare-and-set (H-A-289); nil computes none.
func (h *Hearer) endPipeline(p *seat.Pipeline, now time.Time, wr func(seat.Pipeline, string) []seat.Widening) bool {
if !endPipeline(p, now, h.Unplaced) {
return false
}
if h.Widens != nil {
if wr != nil {
for i := range p.Movers {
p.Movers[i].Widens = h.Widens(*p, p.Movers[i].Module)
p.Movers[i].Widens = wr(*p, p.Movers[i].Module)
}
}
return true
}
// widensReads is the reader of one end's widens, made before its compare-and-set and kept for every retry (H-A-289).
func (h *Hearer) widensReads() func(seat.Pipeline, string) []seat.Widening {
if h.Widens == nil {
return nil
}
return h.Widens()
}
// EndedAtPlacing says a pipeline its starter ended at placing, a seat with no holder (A-470g, A-470h): the step-ended of each
// step of steps, its step log entries appended, the head's finding pipeline-failed (§33), then its pipeline-ended.
func (h *Hearer) EndedAtPlacing(p seat.Pipeline, steps []int) error {
@@ -91,6 +91,7 @@ func (h *Hearer) SupersedeRunning(newer seat.Pipeline) []string {
}
func (h *Hearer) supersedeRunningOne(key string, newer seat.Pipeline) bool {
wr := h.widensReads() // made before the compare-and-set, kept for its retries (H-A-289)
for {
p, rev, err := h.Store.Get(key)
if err != nil || p.Outcome != seat.PipelineOutcomeRunning || p.Branch != newer.Branch || p.Run != "" || newer.Run != "" ||
@@ -123,7 +124,7 @@ func (h *Hearer) supersedeRunningOne(key string, newer seat.Pipeline) bool {
ended = append(ended, i)
}
p.Superseded = append(p.Superseded, moved...)
done := h.endPipeline(&p, now)
done := h.endPipeline(&p, now, wr)
if _, err := h.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
@@ -52,6 +52,7 @@ func (v Verbs) record(verb, what, why string) error {
func (v Verbs) change(key, event string, record func() error, mutate func(p *seat.Pipeline, now time.Time) (int, error)) (seat.Pipeline, int, error) {
h := v.Hearer
recorded := false
wr := h.widensReads() // made before the compare-and-set, kept for its retries (H-A-289)
for {
p, rev, err := h.Store.Get(key)
if err != nil {
@@ -73,7 +74,7 @@ func (v Verbs) change(key, event string, record func() error, mutate func(p *sea
if err != nil {
return p, i, err
}
ended := h.endPipeline(&p, now)
ended := h.endPipeline(&p, now, wr)
if i >= 0 && event == seat.EventStepEnded {
p.Unlogged = append(p.Unlogged, endEntry(key, p.Steps[i], now))
}
@@ -31,20 +31,87 @@ func (w *Widener) Widens(p seat.Pipeline, module string) []seat.Widening {
if w == nil || w.Revision == nil {
return nil
}
return w.Reads()(p, module)
}
// Reads is Widens over reads made once and kept (H-A-289): the two revisions, each serving seat's holder and the grants part's
// grants-for are each read at their first need and shared by every mover of an end and every retry of its write, so a retry
// rereads only the pipeline's key. One end takes one Reads, made before its compare-and-set.
func (w *Widener) Reads() func(p seat.Pipeline, module string) []seat.Widening {
if w == nil || w.Revision == nil {
return func(seat.Pipeline, string) []seat.Widening { return nil }
}
r := &endReads{w: w, revisions: map[string]revisionRead{}, holders: map[string]holderRead{}, grants: map[string]grantsRead{}}
return r.widens
}
type revisionRead struct {
half Half
err error
}
type holderRead struct {
holder string
err error
}
type grantsRead struct {
g meshgrants.GrantsFor
err error
}
// endReads are the reads of one end, each made at its first need and kept.
type endReads struct {
w *Widener
revisions map[string]revisionRead
holders map[string]holderRead
grants map[string]grantsRead
}
func (r *endReads) revision(branch string, rev int64) (Half, error) {
k := fmt.Sprintf("%s.%d", branch, rev)
got, ok := r.revisions[k]
if !ok {
got.half, got.err = r.w.Revision(branch, rev)
r.revisions[k] = got
}
return got.half, got.err
}
func (r *endReads) holder(s string) (string, error) {
got, ok := r.holders[s]
if !ok {
got.holder, got.err = r.w.Holder(s)
r.holders[s] = got
}
return got.holder, got.err
}
func (r *endReads) grantsFor(branch string, source int64) (meshgrants.GrantsFor, error) {
k := fmt.Sprintf("%s.%d", branch, source)
got, ok := r.grants[k]
if !ok {
got.g, got.err = r.w.GrantsFor(branch, source)
r.grants[k] = got
}
return got.g, got.err
}
func (r *endReads) widens(p seat.Pipeline, module string) []seat.Widening {
could := func(name, said string) []seat.Widening {
return []seat.Widening{{What: seat.WideningWhatCouldNotRun, Name: name, Said: said}}
}
cand, err := w.Revision(p.Branch, p.Revision)
cand, err := r.revision(p.Branch, p.Revision)
if err != nil {
return could("mesh-catalog", fmt.Sprintf("the candidate's revision %d is not read: %v", p.Revision, err))
}
newest, err := w.Revision(p.Branch, 0)
newest, err := r.revision(p.Branch, 0)
if err != nil {
return could("mesh-catalog", fmt.Sprintf("the branch's newest revision is not read: %v", err))
}
var out []seat.Widening
for _, s := range servingSeats(p) {
holder, err := w.Holder(s)
holder, err := r.holder(s)
if err != nil {
out = append(out, seat.Widening{What: seat.WideningWhatCouldNotRun, Name: s, Said: fmt.Sprintf("the holder of %s is not read: %v", s, err)})
continue
@@ -57,8 +124,8 @@ func (w *Widener) Widens(p seat.Pipeline, module string) []seat.Widening {
}
out = append(out, ruleWidens(module, newest.Entries[module], cand.Entries[module])...)
out = append(out, credentialWidens(module, newest.Entries[module], cand.Entries[module])...)
if w.GrantsFor != nil {
g, err := w.GrantsFor(p.Branch, p.Source)
if r.w.GrantsFor != nil {
g, err := r.grantsFor(p.Branch, p.Source)
switch {
case err != nil:
out = append(out, seat.Widening{What: seat.WideningWhatCouldNotRun, Name: "mesh-grants", Said: err.Error()})
@@ -88,17 +88,19 @@ func TestWidensIsTheCandidateAgainstTheNewest(t *testing.T) {
// A pipeline's end keeps each mover's widens in its value, whether or not anything widens (design 47 G §71, A-207; H H-A-51).
func TestAnEndKeepsEachMoversWidens(t *testing.T) {
h := &Hearer{Widens: func(p seat.Pipeline, module string) []seat.Widening {
return []seat.Widening{{What: seat.WideningWhatGrant, Name: module + " s v"}}
h := &Hearer{Widens: func() func(seat.Pipeline, string) []seat.Widening {
return func(p seat.Pipeline, module string) []seat.Widening {
return []seat.Widening{{What: seat.WideningWhatGrant, Name: module + " s v"}}
}
}}
p := seat.Pipeline{Outcome: seat.PipelineOutcomeRunning, Movers: []seat.Mover{{Module: "app"}, {Module: "lib"}},
Steps: []seat.StepResult{{Module: "app", Step: "build", Outcome: "passed"}, {Module: "lib", Step: "build", Outcome: "passed"}}}
if !h.endPipeline(&p, time.Unix(1, 0)) || len(p.Movers[0].Widens) != 1 || p.Movers[0].Widens[0].Name != "app s v" || p.Movers[1].Widens[0].Name != "lib s v" {
if !h.endPipeline(&p, time.Unix(1, 0), h.widensReads()) || len(p.Movers[0].Widens) != 1 || p.Movers[0].Widens[0].Name != "app s v" || p.Movers[1].Widens[0].Name != "lib s v" {
t.Errorf("the ended movers = %+v", p.Movers)
}
h.Widens = nil
q := seat.Pipeline{Outcome: seat.PipelineOutcomeRunning, Movers: []seat.Mover{{Module: "app"}}, Steps: []seat.StepResult{{Module: "app", Step: "build", Outcome: "passed"}}}
if !h.endPipeline(&q, time.Unix(1, 0)) || len(q.Movers[0].Widens) != 0 {
if !h.endPipeline(&q, time.Unix(1, 0), h.widensReads()) || len(q.Movers[0].Widens) != 0 {
t.Errorf("a runner with no widener kept %v", q.Movers[0].Widens)
}
}