runner: a test step is asked once per candidate, level and share digest, the others read it (R1)

Two artifacts of one directory ran the same tests twice and failed them twice; the first step of
a test-runs key now asks the run, written create-only, and every other step of the candidate reads it.

Builds: design 47 G §724, §725 (the key), §730; A-468k, A-469c, A-469m, A-469q
This commit is contained in:
2026-11-17 11:22:55 +01:00
parent 8fa1b8aee7
commit 5388bbce43
6 changed files with 409 additions and 14 deletions
+2
View File
@@ -9,6 +9,8 @@ require (
)
require (
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
@@ -77,7 +77,18 @@ func startOf(p seat.Pipeline, now time.Time) time.Time {
// could not run naming the missing seat; the module's steps wait. It returns the steps and the asks to put on the seats'
// work queues, one per running step.
func Place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog.CatalogEntry, now time.Time, asker string) ([]seat.StepResult, []Asked) {
return place(p, reg, entries, now, asker, true)
}
// place is Place; share false places each build step its own ask, as a step asked before the test runs is (§730, A-468k).
// With share, the build steps of one candidate, level and share digest are asked once (§724): the first artifact's step, in
// reference order, asks the run and names it in Shared; every other step of the key reads it, keeping no ask of its own and
// naming the run's ask and the step that asked it (A-469m, A-469q).
func place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog.CatalogEntry, now time.Time, asker string, share bool) ([]seat.StepResult, []Asked) {
entry := entries[p.Module]
candidate, named := CandidateOf(p)
share = share && named
askOfRun := map[string]int{} // a test-runs key to the index of its ask in asks
arts := append([]catalog.Artifact(nil), entry.Artifacts...)
sort.Slice(arts, func(i, j int) bool { return arts[i].Ref < arts[j].Ref })
var steps []seat.StepResult
@@ -118,8 +129,26 @@ func Place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog
}
s.Seat, s.Outcome, s.Started = bs, seat.StepResultOutcomeRunning, &now
s.Ask = AskID(startOf(p, now), p.Pipeline, a.Ref, step)
body := stepAsk(p, entry, a, step, s.Ask, asker, now)
ask := Asked{Seat: bs, ID: s.Ask, Ask: body}
if share && BoundSeat(bs) {
digest := ShareDigest(p.Fingerprint, step, status, body.Step.Fetch)
run := TestRunKey(candidate, step, digest)
reader := seat.TestRunReader{Pipeline: p.Pipeline, Artifact: a.Ref}
if j, asked := askOfRun[run]; asked {
// Another artifact's step of this pipeline asks the run: this one reads it.
first := &asks[j]
first.Readers = append(first.Readers, reader)
s.Ask = ""
s.Shared = []seat.Shared{{Run: run, Ask: first.ID, AskedBy: &seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: first.Readers[0].Artifact}}}
steps = append(steps, s)
continue
}
s.Shared = []seat.Shared{{Run: run, Ask: s.Ask}}
ask.Run, ask.Level, ask.Digest, ask.Readers = run, step, digest, []seat.TestRunReader{reader}
askOfRun[run] = len(asks)
}
steps = append(steps, s)
ask := Asked{Seat: bs, ID: s.Ask, Ask: stepAsk(p, entry, a, step, s.Ask, asker, now)}
if a.Package != nil {
ask.Package = a.Package.Name
}
@@ -156,6 +185,10 @@ type Asked struct {
// Package is the name of the artifact's package, from the catalog entry, for a step on a build seat; its kept results
// are the ask's (§25, §416).
Package string
// Run is the test run the ask asks, its key in test-runs, with its level, its share digest and the steps of the pipeline
// reading it, the asking step first (§724, §725); empty for an ask of no test run.
Run, Level, Digest string
Readers []seat.TestRunReader
}
func stepAsk(p seat.Pipeline, e catalog.CatalogEntry, a catalog.Artifact, step, id, asker string, now time.Time) nodebuildgo.StepAsk {
@@ -328,3 +328,121 @@ func TestAnArtifactWithoutALanguageIsPlacedByItsKind(t *testing.T) {
t.Errorf("asked %d, want the build, the image's three steps and the composition", len(asks))
}
}
// testRunsMem is the state test-runs in memory, a write by compare-and-set on the revision read.
type testRunsMem map[string]struct {
r seat.TestRun
rev uint64
}
func (m testRunsMem) Create(key string, r seat.TestRun) error {
if _, ok := m[key]; ok {
return ErrExists
}
m[key] = struct {
r seat.TestRun
rev uint64
}{r, 1}
return nil
}
func (m testRunsMem) Get(key string) (seat.TestRun, uint64, error) {
e, ok := m[key]
if !ok {
return seat.TestRun{}, 0, ErrNoRun
}
e.r.Readers = append([]seat.TestRunReader(nil), e.r.Readers...)
return e.r, e.rev, nil
}
func (m testRunsMem) Update(key string, r seat.TestRun, rev uint64) (uint64, error) {
if m[key].rev != rev {
return 0, ErrChanged
}
m[key] = struct {
r seat.TestRun
rev uint64
}{r, rev + 1}
return rev + 1, nil
}
// One test run per candidate, level and tested packages (design 47 G §724, A-469c): a module whose program and tools bundle
// test one directory asks one lint, one unit and one integration step, the other artifact's steps reading them; a second
// pipeline of the same candidate reads the runs; a run ended could not run is asked again; a second revision with the same
// digests asks its own runs.
func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) {
reg := catalog.Registration{Module: "app", Branches: []string{"main"}}
entries := map[string]catalog.CatalogEntry{
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
}
store, runs, asks := memStore{}, testRunsMem{}, &stepAsks{}
s := &Starter{Pipelines: memCreated{store}, Store: store, TestRuns: runs, Say: func(string, any) error { return nil }, Steps: asks,
Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }}
start := func(source, revision int64) seat.Pipeline {
t.Helper()
if ok, err := s.Start(Begin{Branch: "main", Source: source, Module: "app", Commit: "c1", Fp: "src2:aa", Revision: revision, Entries: entries}); !ok || err != nil {
t.Fatalf("start %d = %v, %v", source, ok, err)
}
p, _, _ := store.Get(Key("main", source, "app"))
return p
}
testAsks := func(from int) []string {
var out []string
for _, a := range asks.asks[from:] {
if b, ok := a.(nodebuildgo.StepAsk); ok {
out = append(out, b.Step.Artifact+" "+b.Step.Step)
}
}
return out
}
p := start(4, 9)
if got := strings.Join(testAsks(0), "|"); got != "app:program lint|app:program unit|app:program integration" {
t.Fatalf("the test steps asked = %q, want the program's three alone", got)
}
for _, level := range buildSteps {
asker, reader := stepNamed(p, "app:program", level), stepNamed(p, "app:tools", level)
if asker.Ask == "" || len(asker.Shared) != 1 || asker.Shared[0].Ask != asker.Ask || asker.Shared[0].AskedBy != nil {
t.Errorf("the asking %s step = %+v, want its ask named in shared with no asked-by", level, asker)
}
if reader.Ask != "" || reader.Outcome != seat.StepResultOutcomeRunning || len(reader.Shared) != 1 || reader.Shared[0] != (seat.Shared{
Run: asker.Shared[0].Run, Ask: asker.Ask, AskedBy: reader.Shared[0].AskedBy}) || reader.Shared[0].AskedBy == nil ||
*reader.Shared[0].AskedBy != (seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: "app:program"}) {
t.Errorf("the reading %s step = %+v, want no ask and the program's run named", level, reader)
}
r, _, err := runs.Get(asker.Shared[0].Run)
if err != nil || r.Ask != asker.Ask || r.Level != level || len(r.Readers) != 2 || !strings.HasPrefix(asker.Shared[0].Run, "main.9."+level+".") {
t.Errorf("the run of %s = %+v, %v: want the program's ask and both artifacts reading it", level, r, err)
}
}
// A second pipeline of the same candidate (another source revision of the same catalog revision) reads the runs.
n := len(asks.asks)
q := start(5, 9)
if got := testAsks(n); len(got) != 0 {
t.Errorf("the second pipeline of the candidate asked %q, want it to read the runs", got)
}
unit := stepNamed(q, "app:program", "unit")
if unit.Ask != "" || unit.Shared[0].Ask != stepNamed(p, "app:program", "unit").Ask || *unit.Shared[0].AskedBy != (seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: "app:program"}) {
t.Errorf("the second pipeline's unit step = %+v, want it reading the first's run", unit)
}
if r, _, _ := runs.Get(unit.Shared[0].Run); len(r.Readers) != 4 {
t.Errorf("the unit run's readers = %+v, want both pipelines' two artifacts", r.Readers)
}
// A run ended could not run is never read: the next step of its key asks it again.
key := stepNamed(p, "app:program", "lint").Shared[0].Run
r, rev, _ := runs.Get(key)
ended := time.Unix(9, 0)
r.Ended, r.Outcome = &ended, seat.StepResultOutcomeCouldNotRun
_, _ = runs.Update(key, r, rev)
n = len(asks.asks)
start(6, 9)
if got := strings.Join(testAsks(n), "|"); got != "app:program lint" {
t.Errorf("after a lint run that could not run, asked %q, want the lint step alone", got)
}
// A second revision with the same digests asks its own runs, so a failed test runs again.
n = len(asks.asks)
start(7, 10)
if got := len(testAsks(n)); got != 3 {
t.Errorf("a second revision asked %d test steps, want its own three", got)
}
}
@@ -247,8 +247,7 @@ func (s *Starter) placeRunPipeline(key, id string, revision int64, half Half) (b
s.logf("pipeline-started of %s is not said: %v", key, err)
}
if s.Steps != nil && len(asks) > 0 {
s.carry(key, asks)
if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, p.Commit)); err != nil {
if err := s.askSteps(key, p.Commit, asks); err != nil {
return true, err
}
}
@@ -83,6 +83,20 @@ type Starter struct {
Store Updater
// Supersede ends the older running pipelines the pipeline just written supersedes (§31; Hearer.Supersede); nil ends none.
Supersede func(newer seat.Pipeline)
// TestRuns is the state test-runs, where a test step's run is shared by every step of its candidate, level and digest
// (§724, §725); nil shares none, each step's ask put.
TestRuns TestRuns
}
// askSteps puts a placed pipeline's asks (§22, §24): a test run another step asked is read, not asked again (share), then
// the kept results and toolchain are carried and each ask is put.
func (s *Starter) askSteps(key, commit string, asks []Asked) error {
asks = s.share(key, asks)
if len(asks) == 0 {
return nil
}
s.carry(key, asks)
return askAll(s.Steps, s.Arm, key, asks, s.recovered(key, commit))
}
func newPipeline(b Begin, key string, now time.Time) seat.Pipeline {
@@ -144,8 +158,7 @@ func (s *Starter) Start(b Begin) (bool, error) {
s.Supersede(p)
}
if s.Steps != nil && len(asks) > 0 {
s.carry(key, asks)
if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, b.Commit)); err != nil {
if err := s.askSteps(key, b.Commit, asks); err != nil {
return true, err
}
}
@@ -187,24 +200,32 @@ func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) {
if !ok {
continue
}
running := map[string]bool{}
// A step that names its test run is asked as the run's asker (share); one that names none was asked before the test
// runs and keeps its own ask, ending by the path it was asked under (§730, A-468k).
running, own := map[string]bool{}, map[string]bool{}
for _, st := range p.Steps {
if st.Outcome == seat.StepResultOutcomeRunning && st.Ask != "" {
running[st.Ask] = true
running[st.Ask], own[st.Ask] = true, len(st.Shared) == 0
}
}
_, placed := Place(p, reg, half.Entries, now, s.Name)
_, shared := Place(p, reg, half.Entries, now, s.Name)
_, unshared := place(p, reg, half.Entries, now, s.Name, false)
var due []Asked
for _, a := range placed {
if running[a.ID] {
for _, a := range shared {
if running[a.ID] && !own[a.ID] {
due = append(due, a)
}
}
for _, a := range unshared {
if running[a.ID] && own[a.ID] {
a.Run = ""
due = append(due, a)
}
}
if len(due) == 0 {
continue
}
s.carry(key, due)
if err := askAll(s.Steps, s.Arm, key, due, s.recovered(key, p.Commit)); err != nil && first == nil {
if err := s.askSteps(key, p.Commit, due); err != nil && first == nil {
first = err
}
asked += len(due)
@@ -300,8 +321,7 @@ func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipe
}
}
if s.Steps != nil && len(asks) > 0 {
s.carry(key, asks)
if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, p.Commit)); err != nil {
if err := s.askSteps(key, p.Commit, asks); err != nil {
note(err)
}
}
@@ -0,0 +1,223 @@
package pipeline
import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"sort"
"strings"
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// TestRuns is the runner's state test-runs (design 47 G §725): one key per test run of a candidate, level and tested
// packages, written create-only by the step that asks the run and changed by compare-and-set.
type TestRuns interface {
// Create writes key when it is absent; ErrExists when it is written already.
Create(key string, r seat.TestRun) error
// Get reads key and its revision; ErrNoRun when it is absent.
Get(key string) (seat.TestRun, uint64, error)
// Update writes key at the revision read; ErrChanged when another write came between.
Update(key string, r seat.TestRun, revision uint64) (uint64, error)
}
// ErrNoRun is a Get of a test-runs key that is not written.
var ErrNoRun = errors.New("no test run under the key")
// CandidateOf is a pipeline's candidate in A-226's form (§724): `run-<id>` for a run on request, else `<branch>.<revision>`
// of the catalog graph revision it was placed from, built by the node-engine seat's client as the assigner builds it. A
// pipeline naming neither has no candidate, and its steps share no run.
func CandidateOf(p seat.Pipeline) (string, bool) {
var key string
var err error
if p.Run != "" {
key, err = nec.RunKey(p.Run)
} else {
key, err = nec.RevisionKey(p.Branch, p.Revision)
}
return key, err == nil
}
// ShareDigest is a test step's share digest (design 47 G §724, A-469c): the sha256 of the module's build-input fingerprint,
// the level, the step's status in the policy and the step's fetch (its directory and its sorted paths), what the runner
// holds, so two artifacts with one fetch meet and two with different fetches do not.
func ShareDigest(fingerprint, level, status string, fetch nodebuildgo.Fetch) string {
paths := append([]string(nil), fetch.Paths...)
sort.Strings(paths)
h := sha256.New()
for _, part := range []string{fingerprint, level, status, fetch.Dir} {
h.Write([]byte(part + "\n"))
}
h.Write([]byte(strings.Join(paths, "\n")))
return hex.EncodeToString(h.Sum(nil))
}
// TestRunKey is a test run's key in test-runs (§725): `<candidate key>.<level>.<digest>`, the level and the digest each
// escaped as a key's name, the candidate already in its key form.
func TestRunKey(candidate, level, digest string) string {
return candidate + "." + keys.EscapeName(level) + "." + keys.EscapeName(digest)
}
// sharedRun is the one run a step's Shared names, empty when it names none.
func sharedRun(s seat.StepResult) string {
if len(s.Shared) == 0 {
return ""
}
return s.Shared[0].Run
}
// readsRun says whether step s of a pipeline reads run as one step that does not ask it: it names the run and keeps no ask
// of its own (A-469m).
func readsRun(s seat.StepResult, run string) bool {
return run != "" && sharedRun(s) == run && s.Ask == ""
}
// addReaders adds each reader not in r's readers yet, in order.
func addReaders(r *seat.TestRun, readers []seat.TestRunReader) {
for _, n := range readers {
have := false
for _, o := range r.Readers {
have = have || o == n
}
if !have {
r.Readers = append(r.Readers, n)
}
}
}
// askerOf is the step that asked the run under ask: the step of a pipeline the runner keeps whose own ask it is.
func askerOf(store Updater, ask string) (*seat.SharedAskedBy, error) {
if store == nil || ask == "" {
return nil, nil
}
pkeys, err := store.Keys("*.*.*")
if err != nil {
return nil, fmt.Errorf("the pipelines are not listed: %w", err)
}
for _, key := range pkeys {
p, _, err := store.Get(key)
if err != nil {
continue
}
if i := stepOfAsk(p, ask); i >= 0 {
return &seat.SharedAskedBy{Pipeline: key, Artifact: p.Steps[i].Artifact}, nil
}
}
return nil, nil
}
// share writes the test run of each ask that asks one (§724, §725): the step that writes the key create-only asks the run;
// one that finds the key written by another step reads it, its pipeline's steps of that run then keeping no ask of their
// own and naming the run's ask and its asker (A-469m), and its ask is not put. A run ended `could not run` is never read:
// it is replaced by compare-and-set and asked again. A key written by this very ask (a start asked again) asks it again,
// kept once by its id. It answers the asks still to put.
func (s *Starter) share(key string, asks []Asked) []Asked {
if s.TestRuns == nil {
return asks
}
var out []Asked
joined := map[string]seat.TestRun{}
for _, a := range asks {
if a.Run == "" {
out = append(out, a)
continue
}
run, ok, err := s.claimRun(a)
switch {
case err != nil:
// A run that cannot be written or read is asked as the step's own, said: slower, never lost.
s.logf("the test run %s of %s is not written, so the step is asked on its own: %v", a.Run, key, err)
out = append(out, a)
case ok:
out = append(out, a)
default:
joined[a.Run] = run
}
}
if len(joined) > 0 {
s.readOthers(key, joined)
}
return out
}
// claimRun writes a's test run create-only; ok says the ask is put (the key is a's own), else run is the run it reads.
func (s *Starter) claimRun(a Asked) (seat.TestRun, bool, error) {
now := s.now()
value := seat.TestRun{Level: a.Level, Digest: a.Digest, Seat: a.Seat, Ask: a.ID, Started: &now, Readers: a.Readers}
for {
err := s.TestRuns.Create(a.Run, value)
if err == nil {
return value, true, nil
}
if !errors.Is(err, ErrExists) {
return seat.TestRun{}, false, err
}
r, rev, err := s.TestRuns.Get(a.Run)
if errors.Is(err, ErrNoRun) {
continue // deleted meanwhile: written again
}
if err != nil {
return seat.TestRun{}, false, err
}
if r.Ask == a.ID {
return r, true, nil
}
if r.Ended != nil && r.Outcome == seat.StepResultOutcomeCouldNotRun {
if _, err := s.TestRuns.Update(a.Run, value, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return seat.TestRun{}, false, err
}
return value, true, nil
}
addReaders(&r, a.Readers)
if _, err := s.TestRuns.Update(a.Run, r, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return seat.TestRun{}, false, err
}
return r, false, nil
}
}
// readOthers writes pipeline key's steps that read another step's run: no ask of their own, the run's ask and its asker
// in Shared (A-469m, A-469q).
func (s *Starter) readOthers(key string, joined map[string]seat.TestRun) {
if s.Store == nil {
return
}
by := map[string]*seat.SharedAskedBy{}
for run, r := range joined {
asker, err := askerOf(s.Store, r.Ask)
if err != nil {
s.logf("the step that asked the test run %s is not read: %v", run, err)
}
by[run] = asker
}
for {
p, rev, err := s.Store.Get(key)
if err != nil {
s.logf("the pipeline %s is not read, so its steps reading other runs are not written: %v", key, err)
return
}
for i := range p.Steps {
st := &p.Steps[i]
r, ok := joined[sharedRun(*st)]
if !ok || st.Outcome != seat.StepResultOutcomeRunning {
continue
}
st.Ask = ""
st.Shared = []seat.Shared{{Run: sharedRun(*st), Ask: r.Ask, AskedBy: by[sharedRun(*st)]}}
}
if _, err := s.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
s.logf("the pipeline %s's steps reading other runs are not written: %v", key, err)
}
return
}
}