From 5388bbce43347e0aff59c56260f9a6dba34b4bbf Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 11:22:42 +0100 Subject: [PATCH] runner: a test step is asked once per candidate, level and share digest, the others read it (R1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two artifacts of one directory ran the same tests twice and failed them twice; the first step of a test-runs key now asks the run, written create-only, and every other step of the candidate reads it. Builds: design 47 G §724, §725 (the key), §730; A-468k, A-469c, A-469m, A-469q --- modules/mesh-delivery-pipeline/go.mod | 2 + .../internal/pipeline/place.go | 35 ++- .../internal/pipeline/place_test.go | 118 +++++++++ .../internal/pipeline/run.go | 3 +- .../internal/pipeline/start.go | 42 +++- .../internal/pipeline/testruns.go | 223 ++++++++++++++++++ 6 files changed, 409 insertions(+), 14 deletions(-) create mode 100644 modules/mesh-delivery-pipeline/internal/pipeline/testruns.go diff --git a/modules/mesh-delivery-pipeline/go.mod b/modules/mesh-delivery-pipeline/go.mod index 50349fa3..58a8de3f 100644 --- a/modules/mesh-delivery-pipeline/go.mod +++ b/modules/mesh-delivery-pipeline/go.mod @@ -9,6 +9,8 @@ require ( ) require ( + git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 // indirect git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index 653ca3a3..fd76d1bf 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -77,7 +77,18 @@ func startOf(p seat.Pipeline, now time.Time) time.Time { // could not run naming the missing seat; the module's steps wait. It returns the steps and the asks to put on the seats' // work queues, one per running step. func Place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog.CatalogEntry, now time.Time, asker string) ([]seat.StepResult, []Asked) { + return place(p, reg, entries, now, asker, true) +} + +// place is Place; share false places each build step its own ask, as a step asked before the test runs is (§730, A-468k). +// With share, the build steps of one candidate, level and share digest are asked once (§724): the first artifact's step, in +// reference order, asks the run and names it in Shared; every other step of the key reads it, keeping no ask of its own and +// naming the run's ask and the step that asked it (A-469m, A-469q). +func place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog.CatalogEntry, now time.Time, asker string, share bool) ([]seat.StepResult, []Asked) { entry := entries[p.Module] + candidate, named := CandidateOf(p) + share = share && named + askOfRun := map[string]int{} // a test-runs key to the index of its ask in asks arts := append([]catalog.Artifact(nil), entry.Artifacts...) sort.Slice(arts, func(i, j int) bool { return arts[i].Ref < arts[j].Ref }) var steps []seat.StepResult @@ -118,8 +129,26 @@ func Place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog } s.Seat, s.Outcome, s.Started = bs, seat.StepResultOutcomeRunning, &now s.Ask = AskID(startOf(p, now), p.Pipeline, a.Ref, step) + body := stepAsk(p, entry, a, step, s.Ask, asker, now) + ask := Asked{Seat: bs, ID: s.Ask, Ask: body} + if share && BoundSeat(bs) { + digest := ShareDigest(p.Fingerprint, step, status, body.Step.Fetch) + run := TestRunKey(candidate, step, digest) + reader := seat.TestRunReader{Pipeline: p.Pipeline, Artifact: a.Ref} + if j, asked := askOfRun[run]; asked { + // Another artifact's step of this pipeline asks the run: this one reads it. + first := &asks[j] + first.Readers = append(first.Readers, reader) + s.Ask = "" + s.Shared = []seat.Shared{{Run: run, Ask: first.ID, AskedBy: &seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: first.Readers[0].Artifact}}} + steps = append(steps, s) + continue + } + s.Shared = []seat.Shared{{Run: run, Ask: s.Ask}} + ask.Run, ask.Level, ask.Digest, ask.Readers = run, step, digest, []seat.TestRunReader{reader} + askOfRun[run] = len(asks) + } steps = append(steps, s) - ask := Asked{Seat: bs, ID: s.Ask, Ask: stepAsk(p, entry, a, step, s.Ask, asker, now)} if a.Package != nil { ask.Package = a.Package.Name } @@ -156,6 +185,10 @@ type Asked struct { // Package is the name of the artifact's package, from the catalog entry, for a step on a build seat; its kept results // are the ask's (§25, §416). Package string + // Run is the test run the ask asks, its key in test-runs, with its level, its share digest and the steps of the pipeline + // reading it, the asking step first (§724, §725); empty for an ask of no test run. + Run, Level, Digest string + Readers []seat.TestRunReader } func stepAsk(p seat.Pipeline, e catalog.CatalogEntry, a catalog.Artifact, step, id, asker string, now time.Time) nodebuildgo.StepAsk { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index 5336fef0..b8240359 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -328,3 +328,121 @@ func TestAnArtifactWithoutALanguageIsPlacedByItsKind(t *testing.T) { t.Errorf("asked %d, want the build, the image's three steps and the composition", len(asks)) } } + +// testRunsMem is the state test-runs in memory, a write by compare-and-set on the revision read. +type testRunsMem map[string]struct { + r seat.TestRun + rev uint64 +} + +func (m testRunsMem) Create(key string, r seat.TestRun) error { + if _, ok := m[key]; ok { + return ErrExists + } + m[key] = struct { + r seat.TestRun + rev uint64 + }{r, 1} + return nil +} + +func (m testRunsMem) Get(key string) (seat.TestRun, uint64, error) { + e, ok := m[key] + if !ok { + return seat.TestRun{}, 0, ErrNoRun + } + e.r.Readers = append([]seat.TestRunReader(nil), e.r.Readers...) + return e.r, e.rev, nil +} + +func (m testRunsMem) Update(key string, r seat.TestRun, rev uint64) (uint64, error) { + if m[key].rev != rev { + return 0, ErrChanged + } + m[key] = struct { + r seat.TestRun + rev uint64 + }{r, rev + 1} + return rev + 1, nil +} + +// One test run per candidate, level and tested packages (design 47 G §724, A-469c): a module whose program and tools bundle +// test one directory asks one lint, one unit and one integration step, the other artifact's steps reading them; a second +// pipeline of the same candidate reads the runs; a run ended could not run is asked again; a second revision with the same +// digests asks its own runs. +func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) { + reg := catalog.Registration{Module: "app", Branches: []string{"main"}} + entries := map[string]catalog.CatalogEntry{ + "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, + "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, + } + store, runs, asks := memStore{}, testRunsMem{}, &stepAsks{} + s := &Starter{Pipelines: memCreated{store}, Store: store, TestRuns: runs, Say: func(string, any) error { return nil }, Steps: asks, + Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }} + start := func(source, revision int64) seat.Pipeline { + t.Helper() + if ok, err := s.Start(Begin{Branch: "main", Source: source, Module: "app", Commit: "c1", Fp: "src2:aa", Revision: revision, Entries: entries}); !ok || err != nil { + t.Fatalf("start %d = %v, %v", source, ok, err) + } + p, _, _ := store.Get(Key("main", source, "app")) + return p + } + testAsks := func(from int) []string { + var out []string + for _, a := range asks.asks[from:] { + if b, ok := a.(nodebuildgo.StepAsk); ok { + out = append(out, b.Step.Artifact+" "+b.Step.Step) + } + } + return out + } + p := start(4, 9) + if got := strings.Join(testAsks(0), "|"); got != "app:program lint|app:program unit|app:program integration" { + t.Fatalf("the test steps asked = %q, want the program's three alone", got) + } + for _, level := range buildSteps { + asker, reader := stepNamed(p, "app:program", level), stepNamed(p, "app:tools", level) + if asker.Ask == "" || len(asker.Shared) != 1 || asker.Shared[0].Ask != asker.Ask || asker.Shared[0].AskedBy != nil { + t.Errorf("the asking %s step = %+v, want its ask named in shared with no asked-by", level, asker) + } + if reader.Ask != "" || reader.Outcome != seat.StepResultOutcomeRunning || len(reader.Shared) != 1 || reader.Shared[0] != (seat.Shared{ + Run: asker.Shared[0].Run, Ask: asker.Ask, AskedBy: reader.Shared[0].AskedBy}) || reader.Shared[0].AskedBy == nil || + *reader.Shared[0].AskedBy != (seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: "app:program"}) { + t.Errorf("the reading %s step = %+v, want no ask and the program's run named", level, reader) + } + r, _, err := runs.Get(asker.Shared[0].Run) + if err != nil || r.Ask != asker.Ask || r.Level != level || len(r.Readers) != 2 || !strings.HasPrefix(asker.Shared[0].Run, "main.9."+level+".") { + t.Errorf("the run of %s = %+v, %v: want the program's ask and both artifacts reading it", level, r, err) + } + } + // A second pipeline of the same candidate (another source revision of the same catalog revision) reads the runs. + n := len(asks.asks) + q := start(5, 9) + if got := testAsks(n); len(got) != 0 { + t.Errorf("the second pipeline of the candidate asked %q, want it to read the runs", got) + } + unit := stepNamed(q, "app:program", "unit") + if unit.Ask != "" || unit.Shared[0].Ask != stepNamed(p, "app:program", "unit").Ask || *unit.Shared[0].AskedBy != (seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: "app:program"}) { + t.Errorf("the second pipeline's unit step = %+v, want it reading the first's run", unit) + } + if r, _, _ := runs.Get(unit.Shared[0].Run); len(r.Readers) != 4 { + t.Errorf("the unit run's readers = %+v, want both pipelines' two artifacts", r.Readers) + } + // A run ended could not run is never read: the next step of its key asks it again. + key := stepNamed(p, "app:program", "lint").Shared[0].Run + r, rev, _ := runs.Get(key) + ended := time.Unix(9, 0) + r.Ended, r.Outcome = &ended, seat.StepResultOutcomeCouldNotRun + _, _ = runs.Update(key, r, rev) + n = len(asks.asks) + start(6, 9) + if got := strings.Join(testAsks(n), "|"); got != "app:program lint" { + t.Errorf("after a lint run that could not run, asked %q, want the lint step alone", got) + } + // A second revision with the same digests asks its own runs, so a failed test runs again. + n = len(asks.asks) + start(7, 10) + if got := len(testAsks(n)); got != 3 { + t.Errorf("a second revision asked %d test steps, want its own three", got) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/run.go b/modules/mesh-delivery-pipeline/internal/pipeline/run.go index 16e85b51..61d35175 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/run.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/run.go @@ -247,8 +247,7 @@ func (s *Starter) placeRunPipeline(key, id string, revision int64, half Half) (b s.logf("pipeline-started of %s is not said: %v", key, err) } if s.Steps != nil && len(asks) > 0 { - s.carry(key, asks) - if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, p.Commit)); err != nil { + if err := s.askSteps(key, p.Commit, asks); err != nil { return true, err } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 9db155f9..a2e793bd 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -83,6 +83,20 @@ type Starter struct { Store Updater // Supersede ends the older running pipelines the pipeline just written supersedes (§31; Hearer.Supersede); nil ends none. Supersede func(newer seat.Pipeline) + // TestRuns is the state test-runs, where a test step's run is shared by every step of its candidate, level and digest + // (§724, §725); nil shares none, each step's ask put. + TestRuns TestRuns +} + +// askSteps puts a placed pipeline's asks (§22, §24): a test run another step asked is read, not asked again (share), then +// the kept results and toolchain are carried and each ask is put. +func (s *Starter) askSteps(key, commit string, asks []Asked) error { + asks = s.share(key, asks) + if len(asks) == 0 { + return nil + } + s.carry(key, asks) + return askAll(s.Steps, s.Arm, key, asks, s.recovered(key, commit)) } func newPipeline(b Begin, key string, now time.Time) seat.Pipeline { @@ -144,8 +158,7 @@ func (s *Starter) Start(b Begin) (bool, error) { s.Supersede(p) } if s.Steps != nil && len(asks) > 0 { - s.carry(key, asks) - if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, b.Commit)); err != nil { + if err := s.askSteps(key, b.Commit, asks); err != nil { return true, err } } @@ -187,24 +200,32 @@ func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) { if !ok { continue } - running := map[string]bool{} + // A step that names its test run is asked as the run's asker (share); one that names none was asked before the test + // runs and keeps its own ask, ending by the path it was asked under (§730, A-468k). + running, own := map[string]bool{}, map[string]bool{} for _, st := range p.Steps { if st.Outcome == seat.StepResultOutcomeRunning && st.Ask != "" { - running[st.Ask] = true + running[st.Ask], own[st.Ask] = true, len(st.Shared) == 0 } } - _, placed := Place(p, reg, half.Entries, now, s.Name) + _, shared := Place(p, reg, half.Entries, now, s.Name) + _, unshared := place(p, reg, half.Entries, now, s.Name, false) var due []Asked - for _, a := range placed { - if running[a.ID] { + for _, a := range shared { + if running[a.ID] && !own[a.ID] { + due = append(due, a) + } + } + for _, a := range unshared { + if running[a.ID] && own[a.ID] { + a.Run = "" due = append(due, a) } } if len(due) == 0 { continue } - s.carry(key, due) - if err := askAll(s.Steps, s.Arm, key, due, s.recovered(key, p.Commit)); err != nil && first == nil { + if err := s.askSteps(key, p.Commit, due); err != nil && first == nil { first = err } asked += len(due) @@ -300,8 +321,7 @@ func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipe } } if s.Steps != nil && len(asks) > 0 { - s.carry(key, asks) - if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, p.Commit)); err != nil { + if err := s.askSteps(key, p.Commit, asks); err != nil { note(err) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go new file mode 100644 index 00000000..77b2e435 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go @@ -0,0 +1,223 @@ +package pipeline + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "sort" + "strings" + + nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// TestRuns is the runner's state test-runs (design 47 G §725): one key per test run of a candidate, level and tested +// packages, written create-only by the step that asks the run and changed by compare-and-set. +type TestRuns interface { + // Create writes key when it is absent; ErrExists when it is written already. + Create(key string, r seat.TestRun) error + // Get reads key and its revision; ErrNoRun when it is absent. + Get(key string) (seat.TestRun, uint64, error) + // Update writes key at the revision read; ErrChanged when another write came between. + Update(key string, r seat.TestRun, revision uint64) (uint64, error) +} + +// ErrNoRun is a Get of a test-runs key that is not written. +var ErrNoRun = errors.New("no test run under the key") + +// CandidateOf is a pipeline's candidate in A-226's form (§724): `run-` for a run on request, else `.` +// of the catalog graph revision it was placed from, built by the node-engine seat's client as the assigner builds it. A +// pipeline naming neither has no candidate, and its steps share no run. +func CandidateOf(p seat.Pipeline) (string, bool) { + var key string + var err error + if p.Run != "" { + key, err = nec.RunKey(p.Run) + } else { + key, err = nec.RevisionKey(p.Branch, p.Revision) + } + return key, err == nil +} + +// ShareDigest is a test step's share digest (design 47 G §724, A-469c): the sha256 of the module's build-input fingerprint, +// the level, the step's status in the policy and the step's fetch (its directory and its sorted paths), what the runner +// holds, so two artifacts with one fetch meet and two with different fetches do not. +func ShareDigest(fingerprint, level, status string, fetch nodebuildgo.Fetch) string { + paths := append([]string(nil), fetch.Paths...) + sort.Strings(paths) + h := sha256.New() + for _, part := range []string{fingerprint, level, status, fetch.Dir} { + h.Write([]byte(part + "\n")) + } + h.Write([]byte(strings.Join(paths, "\n"))) + return hex.EncodeToString(h.Sum(nil)) +} + +// TestRunKey is a test run's key in test-runs (§725): `..`, the level and the digest each +// escaped as a key's name, the candidate already in its key form. +func TestRunKey(candidate, level, digest string) string { + return candidate + "." + keys.EscapeName(level) + "." + keys.EscapeName(digest) +} + +// sharedRun is the one run a step's Shared names, empty when it names none. +func sharedRun(s seat.StepResult) string { + if len(s.Shared) == 0 { + return "" + } + return s.Shared[0].Run +} + +// readsRun says whether step s of a pipeline reads run as one step that does not ask it: it names the run and keeps no ask +// of its own (A-469m). +func readsRun(s seat.StepResult, run string) bool { + return run != "" && sharedRun(s) == run && s.Ask == "" +} + +// addReaders adds each reader not in r's readers yet, in order. +func addReaders(r *seat.TestRun, readers []seat.TestRunReader) { + for _, n := range readers { + have := false + for _, o := range r.Readers { + have = have || o == n + } + if !have { + r.Readers = append(r.Readers, n) + } + } +} + +// askerOf is the step that asked the run under ask: the step of a pipeline the runner keeps whose own ask it is. +func askerOf(store Updater, ask string) (*seat.SharedAskedBy, error) { + if store == nil || ask == "" { + return nil, nil + } + pkeys, err := store.Keys("*.*.*") + if err != nil { + return nil, fmt.Errorf("the pipelines are not listed: %w", err) + } + for _, key := range pkeys { + p, _, err := store.Get(key) + if err != nil { + continue + } + if i := stepOfAsk(p, ask); i >= 0 { + return &seat.SharedAskedBy{Pipeline: key, Artifact: p.Steps[i].Artifact}, nil + } + } + return nil, nil +} + +// share writes the test run of each ask that asks one (§724, §725): the step that writes the key create-only asks the run; +// one that finds the key written by another step reads it, its pipeline's steps of that run then keeping no ask of their +// own and naming the run's ask and its asker (A-469m), and its ask is not put. A run ended `could not run` is never read: +// it is replaced by compare-and-set and asked again. A key written by this very ask (a start asked again) asks it again, +// kept once by its id. It answers the asks still to put. +func (s *Starter) share(key string, asks []Asked) []Asked { + if s.TestRuns == nil { + return asks + } + var out []Asked + joined := map[string]seat.TestRun{} + for _, a := range asks { + if a.Run == "" { + out = append(out, a) + continue + } + run, ok, err := s.claimRun(a) + switch { + case err != nil: + // A run that cannot be written or read is asked as the step's own, said: slower, never lost. + s.logf("the test run %s of %s is not written, so the step is asked on its own: %v", a.Run, key, err) + out = append(out, a) + case ok: + out = append(out, a) + default: + joined[a.Run] = run + } + } + if len(joined) > 0 { + s.readOthers(key, joined) + } + return out +} + +// claimRun writes a's test run create-only; ok says the ask is put (the key is a's own), else run is the run it reads. +func (s *Starter) claimRun(a Asked) (seat.TestRun, bool, error) { + now := s.now() + value := seat.TestRun{Level: a.Level, Digest: a.Digest, Seat: a.Seat, Ask: a.ID, Started: &now, Readers: a.Readers} + for { + err := s.TestRuns.Create(a.Run, value) + if err == nil { + return value, true, nil + } + if !errors.Is(err, ErrExists) { + return seat.TestRun{}, false, err + } + r, rev, err := s.TestRuns.Get(a.Run) + if errors.Is(err, ErrNoRun) { + continue // deleted meanwhile: written again + } + if err != nil { + return seat.TestRun{}, false, err + } + if r.Ask == a.ID { + return r, true, nil + } + if r.Ended != nil && r.Outcome == seat.StepResultOutcomeCouldNotRun { + if _, err := s.TestRuns.Update(a.Run, value, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return seat.TestRun{}, false, err + } + return value, true, nil + } + addReaders(&r, a.Readers) + if _, err := s.TestRuns.Update(a.Run, r, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return seat.TestRun{}, false, err + } + return r, false, nil + } +} + +// readOthers writes pipeline key's steps that read another step's run: no ask of their own, the run's ask and its asker +// in Shared (A-469m, A-469q). +func (s *Starter) readOthers(key string, joined map[string]seat.TestRun) { + if s.Store == nil { + return + } + by := map[string]*seat.SharedAskedBy{} + for run, r := range joined { + asker, err := askerOf(s.Store, r.Ask) + if err != nil { + s.logf("the step that asked the test run %s is not read: %v", run, err) + } + by[run] = asker + } + for { + p, rev, err := s.Store.Get(key) + if err != nil { + s.logf("the pipeline %s is not read, so its steps reading other runs are not written: %v", key, err) + return + } + for i := range p.Steps { + st := &p.Steps[i] + r, ok := joined[sharedRun(*st)] + if !ok || st.Outcome != seat.StepResultOutcomeRunning { + continue + } + st.Ask = "" + st.Shared = []seat.Shared{{Run: sharedRun(*st), Ask: r.Ask, AskedBy: by[sharedRun(*st)]}} + } + if _, err := s.Store.Update(key, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + s.logf("the pipeline %s's steps reading other runs are not written: %v", key, err) + } + return + } +}