From 5388bbce43347e0aff59c56260f9a6dba34b4bbf Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 11:22:42 +0100 Subject: [PATCH 01/14] runner: a test step is asked once per candidate, level and share digest, the others read it (R1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two artifacts of one directory ran the same tests twice and failed them twice; the first step of a test-runs key now asks the run, written create-only, and every other step of the candidate reads it. Builds: design 47 G §724, §725 (the key), §730; A-468k, A-469c, A-469m, A-469q --- modules/mesh-delivery-pipeline/go.mod | 2 + .../internal/pipeline/place.go | 35 ++- .../internal/pipeline/place_test.go | 118 +++++++++ .../internal/pipeline/run.go | 3 +- .../internal/pipeline/start.go | 42 +++- .../internal/pipeline/testruns.go | 223 ++++++++++++++++++ 6 files changed, 409 insertions(+), 14 deletions(-) create mode 100644 modules/mesh-delivery-pipeline/internal/pipeline/testruns.go diff --git a/modules/mesh-delivery-pipeline/go.mod b/modules/mesh-delivery-pipeline/go.mod index 50349fa3..58a8de3f 100644 --- a/modules/mesh-delivery-pipeline/go.mod +++ b/modules/mesh-delivery-pipeline/go.mod @@ -9,6 +9,8 @@ require ( ) require ( + git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 // indirect git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index 653ca3a3..fd76d1bf 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -77,7 +77,18 @@ func startOf(p seat.Pipeline, now time.Time) time.Time { // could not run naming the missing seat; the module's steps wait. It returns the steps and the asks to put on the seats' // work queues, one per running step. func Place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog.CatalogEntry, now time.Time, asker string) ([]seat.StepResult, []Asked) { + return place(p, reg, entries, now, asker, true) +} + +// place is Place; share false places each build step its own ask, as a step asked before the test runs is (§730, A-468k). +// With share, the build steps of one candidate, level and share digest are asked once (§724): the first artifact's step, in +// reference order, asks the run and names it in Shared; every other step of the key reads it, keeping no ask of its own and +// naming the run's ask and the step that asked it (A-469m, A-469q). +func place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog.CatalogEntry, now time.Time, asker string, share bool) ([]seat.StepResult, []Asked) { entry := entries[p.Module] + candidate, named := CandidateOf(p) + share = share && named + askOfRun := map[string]int{} // a test-runs key to the index of its ask in asks arts := append([]catalog.Artifact(nil), entry.Artifacts...) sort.Slice(arts, func(i, j int) bool { return arts[i].Ref < arts[j].Ref }) var steps []seat.StepResult @@ -118,8 +129,26 @@ func Place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog } s.Seat, s.Outcome, s.Started = bs, seat.StepResultOutcomeRunning, &now s.Ask = AskID(startOf(p, now), p.Pipeline, a.Ref, step) + body := stepAsk(p, entry, a, step, s.Ask, asker, now) + ask := Asked{Seat: bs, ID: s.Ask, Ask: body} + if share && BoundSeat(bs) { + digest := ShareDigest(p.Fingerprint, step, status, body.Step.Fetch) + run := TestRunKey(candidate, step, digest) + reader := seat.TestRunReader{Pipeline: p.Pipeline, Artifact: a.Ref} + if j, asked := askOfRun[run]; asked { + // Another artifact's step of this pipeline asks the run: this one reads it. + first := &asks[j] + first.Readers = append(first.Readers, reader) + s.Ask = "" + s.Shared = []seat.Shared{{Run: run, Ask: first.ID, AskedBy: &seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: first.Readers[0].Artifact}}} + steps = append(steps, s) + continue + } + s.Shared = []seat.Shared{{Run: run, Ask: s.Ask}} + ask.Run, ask.Level, ask.Digest, ask.Readers = run, step, digest, []seat.TestRunReader{reader} + askOfRun[run] = len(asks) + } steps = append(steps, s) - ask := Asked{Seat: bs, ID: s.Ask, Ask: stepAsk(p, entry, a, step, s.Ask, asker, now)} if a.Package != nil { ask.Package = a.Package.Name } @@ -156,6 +185,10 @@ type Asked struct { // Package is the name of the artifact's package, from the catalog entry, for a step on a build seat; its kept results // are the ask's (§25, §416). Package string + // Run is the test run the ask asks, its key in test-runs, with its level, its share digest and the steps of the pipeline + // reading it, the asking step first (§724, §725); empty for an ask of no test run. + Run, Level, Digest string + Readers []seat.TestRunReader } func stepAsk(p seat.Pipeline, e catalog.CatalogEntry, a catalog.Artifact, step, id, asker string, now time.Time) nodebuildgo.StepAsk { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index 5336fef0..b8240359 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -328,3 +328,121 @@ func TestAnArtifactWithoutALanguageIsPlacedByItsKind(t *testing.T) { t.Errorf("asked %d, want the build, the image's three steps and the composition", len(asks)) } } + +// testRunsMem is the state test-runs in memory, a write by compare-and-set on the revision read. +type testRunsMem map[string]struct { + r seat.TestRun + rev uint64 +} + +func (m testRunsMem) Create(key string, r seat.TestRun) error { + if _, ok := m[key]; ok { + return ErrExists + } + m[key] = struct { + r seat.TestRun + rev uint64 + }{r, 1} + return nil +} + +func (m testRunsMem) Get(key string) (seat.TestRun, uint64, error) { + e, ok := m[key] + if !ok { + return seat.TestRun{}, 0, ErrNoRun + } + e.r.Readers = append([]seat.TestRunReader(nil), e.r.Readers...) + return e.r, e.rev, nil +} + +func (m testRunsMem) Update(key string, r seat.TestRun, rev uint64) (uint64, error) { + if m[key].rev != rev { + return 0, ErrChanged + } + m[key] = struct { + r seat.TestRun + rev uint64 + }{r, rev + 1} + return rev + 1, nil +} + +// One test run per candidate, level and tested packages (design 47 G §724, A-469c): a module whose program and tools bundle +// test one directory asks one lint, one unit and one integration step, the other artifact's steps reading them; a second +// pipeline of the same candidate reads the runs; a run ended could not run is asked again; a second revision with the same +// digests asks its own runs. +func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) { + reg := catalog.Registration{Module: "app", Branches: []string{"main"}} + entries := map[string]catalog.CatalogEntry{ + "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, + "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, + } + store, runs, asks := memStore{}, testRunsMem{}, &stepAsks{} + s := &Starter{Pipelines: memCreated{store}, Store: store, TestRuns: runs, Say: func(string, any) error { return nil }, Steps: asks, + Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }} + start := func(source, revision int64) seat.Pipeline { + t.Helper() + if ok, err := s.Start(Begin{Branch: "main", Source: source, Module: "app", Commit: "c1", Fp: "src2:aa", Revision: revision, Entries: entries}); !ok || err != nil { + t.Fatalf("start %d = %v, %v", source, ok, err) + } + p, _, _ := store.Get(Key("main", source, "app")) + return p + } + testAsks := func(from int) []string { + var out []string + for _, a := range asks.asks[from:] { + if b, ok := a.(nodebuildgo.StepAsk); ok { + out = append(out, b.Step.Artifact+" "+b.Step.Step) + } + } + return out + } + p := start(4, 9) + if got := strings.Join(testAsks(0), "|"); got != "app:program lint|app:program unit|app:program integration" { + t.Fatalf("the test steps asked = %q, want the program's three alone", got) + } + for _, level := range buildSteps { + asker, reader := stepNamed(p, "app:program", level), stepNamed(p, "app:tools", level) + if asker.Ask == "" || len(asker.Shared) != 1 || asker.Shared[0].Ask != asker.Ask || asker.Shared[0].AskedBy != nil { + t.Errorf("the asking %s step = %+v, want its ask named in shared with no asked-by", level, asker) + } + if reader.Ask != "" || reader.Outcome != seat.StepResultOutcomeRunning || len(reader.Shared) != 1 || reader.Shared[0] != (seat.Shared{ + Run: asker.Shared[0].Run, Ask: asker.Ask, AskedBy: reader.Shared[0].AskedBy}) || reader.Shared[0].AskedBy == nil || + *reader.Shared[0].AskedBy != (seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: "app:program"}) { + t.Errorf("the reading %s step = %+v, want no ask and the program's run named", level, reader) + } + r, _, err := runs.Get(asker.Shared[0].Run) + if err != nil || r.Ask != asker.Ask || r.Level != level || len(r.Readers) != 2 || !strings.HasPrefix(asker.Shared[0].Run, "main.9."+level+".") { + t.Errorf("the run of %s = %+v, %v: want the program's ask and both artifacts reading it", level, r, err) + } + } + // A second pipeline of the same candidate (another source revision of the same catalog revision) reads the runs. + n := len(asks.asks) + q := start(5, 9) + if got := testAsks(n); len(got) != 0 { + t.Errorf("the second pipeline of the candidate asked %q, want it to read the runs", got) + } + unit := stepNamed(q, "app:program", "unit") + if unit.Ask != "" || unit.Shared[0].Ask != stepNamed(p, "app:program", "unit").Ask || *unit.Shared[0].AskedBy != (seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: "app:program"}) { + t.Errorf("the second pipeline's unit step = %+v, want it reading the first's run", unit) + } + if r, _, _ := runs.Get(unit.Shared[0].Run); len(r.Readers) != 4 { + t.Errorf("the unit run's readers = %+v, want both pipelines' two artifacts", r.Readers) + } + // A run ended could not run is never read: the next step of its key asks it again. + key := stepNamed(p, "app:program", "lint").Shared[0].Run + r, rev, _ := runs.Get(key) + ended := time.Unix(9, 0) + r.Ended, r.Outcome = &ended, seat.StepResultOutcomeCouldNotRun + _, _ = runs.Update(key, r, rev) + n = len(asks.asks) + start(6, 9) + if got := strings.Join(testAsks(n), "|"); got != "app:program lint" { + t.Errorf("after a lint run that could not run, asked %q, want the lint step alone", got) + } + // A second revision with the same digests asks its own runs, so a failed test runs again. + n = len(asks.asks) + start(7, 10) + if got := len(testAsks(n)); got != 3 { + t.Errorf("a second revision asked %d test steps, want its own three", got) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/run.go b/modules/mesh-delivery-pipeline/internal/pipeline/run.go index 16e85b51..61d35175 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/run.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/run.go @@ -247,8 +247,7 @@ func (s *Starter) placeRunPipeline(key, id string, revision int64, half Half) (b s.logf("pipeline-started of %s is not said: %v", key, err) } if s.Steps != nil && len(asks) > 0 { - s.carry(key, asks) - if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, p.Commit)); err != nil { + if err := s.askSteps(key, p.Commit, asks); err != nil { return true, err } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 9db155f9..a2e793bd 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -83,6 +83,20 @@ type Starter struct { Store Updater // Supersede ends the older running pipelines the pipeline just written supersedes (§31; Hearer.Supersede); nil ends none. Supersede func(newer seat.Pipeline) + // TestRuns is the state test-runs, where a test step's run is shared by every step of its candidate, level and digest + // (§724, §725); nil shares none, each step's ask put. + TestRuns TestRuns +} + +// askSteps puts a placed pipeline's asks (§22, §24): a test run another step asked is read, not asked again (share), then +// the kept results and toolchain are carried and each ask is put. +func (s *Starter) askSteps(key, commit string, asks []Asked) error { + asks = s.share(key, asks) + if len(asks) == 0 { + return nil + } + s.carry(key, asks) + return askAll(s.Steps, s.Arm, key, asks, s.recovered(key, commit)) } func newPipeline(b Begin, key string, now time.Time) seat.Pipeline { @@ -144,8 +158,7 @@ func (s *Starter) Start(b Begin) (bool, error) { s.Supersede(p) } if s.Steps != nil && len(asks) > 0 { - s.carry(key, asks) - if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, b.Commit)); err != nil { + if err := s.askSteps(key, b.Commit, asks); err != nil { return true, err } } @@ -187,24 +200,32 @@ func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) { if !ok { continue } - running := map[string]bool{} + // A step that names its test run is asked as the run's asker (share); one that names none was asked before the test + // runs and keeps its own ask, ending by the path it was asked under (§730, A-468k). + running, own := map[string]bool{}, map[string]bool{} for _, st := range p.Steps { if st.Outcome == seat.StepResultOutcomeRunning && st.Ask != "" { - running[st.Ask] = true + running[st.Ask], own[st.Ask] = true, len(st.Shared) == 0 } } - _, placed := Place(p, reg, half.Entries, now, s.Name) + _, shared := Place(p, reg, half.Entries, now, s.Name) + _, unshared := place(p, reg, half.Entries, now, s.Name, false) var due []Asked - for _, a := range placed { - if running[a.ID] { + for _, a := range shared { + if running[a.ID] && !own[a.ID] { + due = append(due, a) + } + } + for _, a := range unshared { + if running[a.ID] && own[a.ID] { + a.Run = "" due = append(due, a) } } if len(due) == 0 { continue } - s.carry(key, due) - if err := askAll(s.Steps, s.Arm, key, due, s.recovered(key, p.Commit)); err != nil && first == nil { + if err := s.askSteps(key, p.Commit, due); err != nil && first == nil { first = err } asked += len(due) @@ -300,8 +321,7 @@ func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipe } } if s.Steps != nil && len(asks) > 0 { - s.carry(key, asks) - if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, p.Commit)); err != nil { + if err := s.askSteps(key, p.Commit, asks); err != nil { note(err) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go new file mode 100644 index 00000000..77b2e435 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go @@ -0,0 +1,223 @@ +package pipeline + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "sort" + "strings" + + nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// TestRuns is the runner's state test-runs (design 47 G §725): one key per test run of a candidate, level and tested +// packages, written create-only by the step that asks the run and changed by compare-and-set. +type TestRuns interface { + // Create writes key when it is absent; ErrExists when it is written already. + Create(key string, r seat.TestRun) error + // Get reads key and its revision; ErrNoRun when it is absent. + Get(key string) (seat.TestRun, uint64, error) + // Update writes key at the revision read; ErrChanged when another write came between. + Update(key string, r seat.TestRun, revision uint64) (uint64, error) +} + +// ErrNoRun is a Get of a test-runs key that is not written. +var ErrNoRun = errors.New("no test run under the key") + +// CandidateOf is a pipeline's candidate in A-226's form (§724): `run-` for a run on request, else `.` +// of the catalog graph revision it was placed from, built by the node-engine seat's client as the assigner builds it. A +// pipeline naming neither has no candidate, and its steps share no run. +func CandidateOf(p seat.Pipeline) (string, bool) { + var key string + var err error + if p.Run != "" { + key, err = nec.RunKey(p.Run) + } else { + key, err = nec.RevisionKey(p.Branch, p.Revision) + } + return key, err == nil +} + +// ShareDigest is a test step's share digest (design 47 G §724, A-469c): the sha256 of the module's build-input fingerprint, +// the level, the step's status in the policy and the step's fetch (its directory and its sorted paths), what the runner +// holds, so two artifacts with one fetch meet and two with different fetches do not. +func ShareDigest(fingerprint, level, status string, fetch nodebuildgo.Fetch) string { + paths := append([]string(nil), fetch.Paths...) + sort.Strings(paths) + h := sha256.New() + for _, part := range []string{fingerprint, level, status, fetch.Dir} { + h.Write([]byte(part + "\n")) + } + h.Write([]byte(strings.Join(paths, "\n"))) + return hex.EncodeToString(h.Sum(nil)) +} + +// TestRunKey is a test run's key in test-runs (§725): `..`, the level and the digest each +// escaped as a key's name, the candidate already in its key form. +func TestRunKey(candidate, level, digest string) string { + return candidate + "." + keys.EscapeName(level) + "." + keys.EscapeName(digest) +} + +// sharedRun is the one run a step's Shared names, empty when it names none. +func sharedRun(s seat.StepResult) string { + if len(s.Shared) == 0 { + return "" + } + return s.Shared[0].Run +} + +// readsRun says whether step s of a pipeline reads run as one step that does not ask it: it names the run and keeps no ask +// of its own (A-469m). +func readsRun(s seat.StepResult, run string) bool { + return run != "" && sharedRun(s) == run && s.Ask == "" +} + +// addReaders adds each reader not in r's readers yet, in order. +func addReaders(r *seat.TestRun, readers []seat.TestRunReader) { + for _, n := range readers { + have := false + for _, o := range r.Readers { + have = have || o == n + } + if !have { + r.Readers = append(r.Readers, n) + } + } +} + +// askerOf is the step that asked the run under ask: the step of a pipeline the runner keeps whose own ask it is. +func askerOf(store Updater, ask string) (*seat.SharedAskedBy, error) { + if store == nil || ask == "" { + return nil, nil + } + pkeys, err := store.Keys("*.*.*") + if err != nil { + return nil, fmt.Errorf("the pipelines are not listed: %w", err) + } + for _, key := range pkeys { + p, _, err := store.Get(key) + if err != nil { + continue + } + if i := stepOfAsk(p, ask); i >= 0 { + return &seat.SharedAskedBy{Pipeline: key, Artifact: p.Steps[i].Artifact}, nil + } + } + return nil, nil +} + +// share writes the test run of each ask that asks one (§724, §725): the step that writes the key create-only asks the run; +// one that finds the key written by another step reads it, its pipeline's steps of that run then keeping no ask of their +// own and naming the run's ask and its asker (A-469m), and its ask is not put. A run ended `could not run` is never read: +// it is replaced by compare-and-set and asked again. A key written by this very ask (a start asked again) asks it again, +// kept once by its id. It answers the asks still to put. +func (s *Starter) share(key string, asks []Asked) []Asked { + if s.TestRuns == nil { + return asks + } + var out []Asked + joined := map[string]seat.TestRun{} + for _, a := range asks { + if a.Run == "" { + out = append(out, a) + continue + } + run, ok, err := s.claimRun(a) + switch { + case err != nil: + // A run that cannot be written or read is asked as the step's own, said: slower, never lost. + s.logf("the test run %s of %s is not written, so the step is asked on its own: %v", a.Run, key, err) + out = append(out, a) + case ok: + out = append(out, a) + default: + joined[a.Run] = run + } + } + if len(joined) > 0 { + s.readOthers(key, joined) + } + return out +} + +// claimRun writes a's test run create-only; ok says the ask is put (the key is a's own), else run is the run it reads. +func (s *Starter) claimRun(a Asked) (seat.TestRun, bool, error) { + now := s.now() + value := seat.TestRun{Level: a.Level, Digest: a.Digest, Seat: a.Seat, Ask: a.ID, Started: &now, Readers: a.Readers} + for { + err := s.TestRuns.Create(a.Run, value) + if err == nil { + return value, true, nil + } + if !errors.Is(err, ErrExists) { + return seat.TestRun{}, false, err + } + r, rev, err := s.TestRuns.Get(a.Run) + if errors.Is(err, ErrNoRun) { + continue // deleted meanwhile: written again + } + if err != nil { + return seat.TestRun{}, false, err + } + if r.Ask == a.ID { + return r, true, nil + } + if r.Ended != nil && r.Outcome == seat.StepResultOutcomeCouldNotRun { + if _, err := s.TestRuns.Update(a.Run, value, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return seat.TestRun{}, false, err + } + return value, true, nil + } + addReaders(&r, a.Readers) + if _, err := s.TestRuns.Update(a.Run, r, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return seat.TestRun{}, false, err + } + return r, false, nil + } +} + +// readOthers writes pipeline key's steps that read another step's run: no ask of their own, the run's ask and its asker +// in Shared (A-469m, A-469q). +func (s *Starter) readOthers(key string, joined map[string]seat.TestRun) { + if s.Store == nil { + return + } + by := map[string]*seat.SharedAskedBy{} + for run, r := range joined { + asker, err := askerOf(s.Store, r.Ask) + if err != nil { + s.logf("the step that asked the test run %s is not read: %v", run, err) + } + by[run] = asker + } + for { + p, rev, err := s.Store.Get(key) + if err != nil { + s.logf("the pipeline %s is not read, so its steps reading other runs are not written: %v", key, err) + return + } + for i := range p.Steps { + st := &p.Steps[i] + r, ok := joined[sharedRun(*st)] + if !ok || st.Outcome != seat.StepResultOutcomeRunning { + continue + } + st.Ask = "" + st.Shared = []seat.Shared{{Run: sharedRun(*st), Ask: r.Ask, AskedBy: by[sharedRun(*st)]}} + } + if _, err := s.Store.Update(key, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + s.logf("the pipeline %s's steps reading other runs are not written: %v", key, err) + } + return + } +} From cd08f1e1c2f94d1d5fd04402a0c1c8a0b54625c0 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 11:31:26 +0100 Subject: [PATCH 02/14] runner: a test run is kept in the state test-runs with one clock, a cap and an age (R2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A test run is now one key of test-runs: its untaken bound written at its start, its clock at ask-taken and each step-progress, an expiry ending the run and every reader could not run and dropping its ask, a run past its bound while the runner was away ended at its start, the cap judged by the process, and an ended run aged with its output object in the step log's order. Builds: design 47 G §725, §720 (When the clock ends, A run never taken); A-466n, A-466p, A-466s, A-468b, A-468c, A-469d, A-469e, A-469w, A-469x, A-470, A-470a --- .../cmd/mesh-delivery-pipeline/main.go | 24 +- .../run_integration_test.go | 7 +- .../cmd/mesh-delivery-pipeline/testruns.go | 329 ++++++++++++++++++ .../mesh-delivery-pipeline/testruns_test.go | 58 +++ .../internal/pipeline/aged.go | 61 +++- .../internal/pipeline/place_test.go | 2 + .../internal/pipeline/runends.go | 326 +++++++++++++++++ .../internal/pipeline/start.go | 10 +- .../internal/pipeline/steps.go | 24 ++ .../internal/pipeline/testruns.go | 71 +++- .../internal/pipeline/testruns_test.go | 221 ++++++++++++ .../mesh-delivery-pipeline/manifest_test.go | 48 ++- modules/mesh-delivery-pipeline/module.json | 75 +++- 13 files changed, 1244 insertions(+), 12 deletions(-) create mode 100644 modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go create mode 100644 modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_test.go create mode 100644 modules/mesh-delivery-pipeline/internal/pipeline/runends.go create mode 100644 modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 7d973c3d..17c35f90 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -234,13 +234,31 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro logger.Printf("the pipeline %s superseded %d older pipelines", newer.Pipeline, n) } } + // The test runs (§724, §725): one run per candidate, level and digest, kept in the state test-runs with its clock. + runDrop := pipeline.BoundDrop(dropAsk(calls, cred.Module), armDropBound(ctx, bounds), dropAnswered(ctx, deleteBound(ctx, bounds), rv.Findings)) + runsKV, _, err := openTestRuns(ctx, js, cred, getenv, starter, hearer, runDrop, logger.Printf) + if err != nil { + return err + } + if err := watchRunBounds(ctx, runsKV, hearer.RunExpired, logger.Printf); err != nil { + return err + } + if err := hearStepProgress(heard, hearer.RunProgress, logger.Printf); err != nil { + return err + } if err := hearStepEnded(heard, hearer, logger.Printf); err != nil { return err } if err := hearToolchain(ctx, heard, toolchains, logger.Printf); err != nil { return err } - if err := hearAskTaken(ctx, heard, bounds, hearer.Taken, logger.Printf); err != nil { + taken := func(ask, node string) error { + if err := hearer.Taken(ask, node); err != nil { + return err + } + return hearer.RunTaken(ask, node) + } + if err := hearAskTaken(ctx, heard, bounds, taken, logger.Printf); err != nil { return err } expired := boundExpired(hearer, dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf) @@ -261,6 +279,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } // An entry whose step ended and was not appended is appended at the start, in order (§606, §549). hearer.AppendUnlogged() + // A test run whose bound passed while the runner was away ends now, as its expiry would have ended it (§725). + if n, err := hearer.TakeUp(); err != nil || n > 0 { + logger.Printf("the test runs taken up at the start: %d ended past their bound (%v)", n, err) + } go whileTheLogIsFull(ctx, fills, hearer, time.Minute) // The seat list, read over the calls connection and heard on the hub (seatlist.Part), which the seat's write verbs check // their callers against (A-186): registered before the hub starts, read again once its events are live. diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go index 25489f6a..32ee6038 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go @@ -65,8 +65,11 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes t.Fatal(err) } } - if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_bounds", History: 1, LimitMarkerTTL: time.Minute}); err != nil { - t.Fatal(err) + // bounds and test-runs keep keys with their own time to live (§725), as the bus makes them from the manifest's key-ttl. + for _, state := range []string{"bounds", TestRunsState} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1, LimitMarkerTTL: time.Minute}); err != nil { + t.Fatal(err) + } } // The step log, the runner's old logs section (§470): the stream the bus makes for it. if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go new file mode 100644 index 00000000..ca1091ea --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go @@ -0,0 +1,329 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "strconv" + "strings" + "sync" + "time" + + "github.com/nats-io/nats.go/jetstream" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + nodebuildimage "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage" + nodebuildnode "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-pipeline/internal/pipeline" +) + +// TestRunsState is the runner's state of its test runs (design 47 G §725), declared in its manifest's old state section. +const TestRunsState = "test-runs" + +// testRunsKV is the state test-runs on the bus: its runs by compare-and-set, and its bound keys, each written with its own +// time to live (A-77, A-470). +type testRunsKV struct { + ctx context.Context + kv jetstream.KeyValue +} + +func (t testRunsKV) Create(key string, r seat.TestRun) error { + raw, err := json.Marshal(r) + if err != nil { + return err + } + if _, err := t.kv.Create(t.ctx, key, raw); errors.Is(err, jetstream.ErrKeyExists) { + return pipeline.ErrExists + } else if err != nil { + return err + } + return nil +} + +func (t testRunsKV) Get(key string) (seat.TestRun, uint64, error) { + e, err := t.kv.Get(t.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return seat.TestRun{}, 0, pipeline.ErrNoRun + } + if err != nil { + return seat.TestRun{}, 0, err + } + var r seat.TestRun + if err := json.Unmarshal(e.Value(), &r); err != nil { + return seat.TestRun{}, 0, fmt.Errorf("the test run %s is no test run: %w", key, err) + } + return r, e.Revision(), nil +} + +func (t testRunsKV) Update(key string, r seat.TestRun, revision uint64) (uint64, error) { + raw, err := json.Marshal(r) + if err != nil { + return 0, err + } + rev, err := t.kv.Update(t.ctx, key, raw, revision) + if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) { + return 0, pipeline.ErrChanged + } + return rev, err +} + +func (t testRunsKV) Delete(key string) error { + if err := t.kv.Delete(t.ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + return err + } + return nil +} + +// Bound writes key again with its time to live: deleted, then created with ttl, so its expiry is heard only ttl after +// the last write (a delete is no expiry). +func (t testRunsKV) Bound(key string, ttl time.Duration) error { + if err := t.Delete(key); err != nil { + return err + } + _, err := t.kv.Create(t.ctx, key, []byte(`{"bound":"`+key+`"}`), jetstream.KeyTTL(ttl)) + if errors.Is(err, jetstream.ErrKeyExists) { + return nil + } + return err +} + +func (t testRunsKV) Unbound(key string) error { return t.Delete(key) } + +func (t testRunsKV) Has(key string) (bool, error) { + _, err := t.kv.Get(t.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return false, nil + } + return err == nil, err +} + +func (t testRunsKV) Keys() ([]string, error) { + all, err := t.kv.Keys(t.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return all, err +} + +// runObjects is the state test-runs' own object store, `test-runs-objects`, made with the state (A-469e), opened when +// first used. +type runObjects struct { + ctx context.Context + js jetstream.JetStream + module string + mu sync.Mutex + store jetstream.ObjectStore +} + +func (o *runObjects) open() (jetstream.ObjectStore, error) { + o.mu.Lock() + defer o.mu.Unlock() + if o.store != nil { + return o.store, nil + } + s, err := o.js.ObjectStore(o.ctx, catalogue.BusName(o.module, catalogue.StateObjectsName(TestRunsState))) + if err != nil { + return nil, fmt.Errorf("the object store of the state %s is not open: %w", TestRunsState, err) + } + o.store = s + return s, nil +} + +func (o *runObjects) Put(name string, data []byte) error { + s, err := o.open() + if err != nil { + return err + } + _, err = s.PutBytes(o.ctx, name, data) + return err +} + +func (o *runObjects) Get(name string) ([]byte, error) { + s, err := o.open() + if err != nil { + return nil, err + } + data, err := s.GetBytes(o.ctx, name) + if errors.Is(err, jetstream.ErrObjectNotFound) { + return nil, pipeline.ErrNoRun + } + return data, err +} + +func (o *runObjects) Delete(name string) error { + s, err := o.open() + if err != nil { + return err + } + if err := s.Delete(o.ctx, name); err != nil && !errors.Is(err, jetstream.ErrObjectNotFound) { + return err + } + return nil +} + +// runsFill reads test-runs' fill, its key-value stream's bytes (A-469d): judged by the process against its cap setting. +func runsFill(ctx context.Context, js jetstream.JetStream, module string, now func() time.Time) func() (seat.Fill, error) { + return func() (seat.Fill, error) { + st, err := js.Stream(ctx, "KV_"+ownBucket(module, TestRunsState)) + if err != nil { + return seat.Fill{}, err + } + info, err := st.Info(ctx) + if err != nil { + return seat.Fill{}, err + } + at := now().UTC() + return seat.Fill{Store: TestRunsState, Bytes: int64(info.State.Bytes), Cap: info.Config.MaxBytes, At: &at}, nil + } +} + +// The files the manifest writes each setting of the test runs into, `${config:}`, named by the environment (A-469x). +const ( + envRunsCap = "MESH_RUNNER_TEST_RUNS_CAP_FILE" + envRunsKept = "MESH_RUNNER_TEST_RUNS_KEPT_FILE" + envStepBound = "MESH_RUNNER_TEST_STEP_BOUND_FILE" + envUntakenBound = "MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE" +) + +// maxRunsKept is test-runs-kept's maximum (A-470a): the pipeline naming a run's key is kept at least that long. +const maxRunsKept = 168 * time.Hour + +// runSettings reads the settings of the test runs at the start (§725; A-469x): test-runs-cap-mib a whole number of MiB, +// test-runs-kept, test-step-bound and run-untaken-bound Go durations. A setting whose file is missing or empty is its +// default; a content that is neither is an error naming the file. +func runSettings(getenv func(string) string) (pipeline.RunSettings, error) { + s := pipeline.DefaultRunSettings + if v, err := settingText(getenv(envRunsCap)); err != nil { + return s, err + } else if v != "" { + n, err := strconv.Atoi(v) + if err != nil || n < 1 { + return s, fmt.Errorf("%s holds %q, which is not a positive whole number", getenv(envRunsCap), v) + } + s.CapBytes = int64(n) << 20 + } + for _, d := range []struct { + env string + to *time.Duration + }{{envRunsKept, &s.Kept}, {envStepBound, &s.StepBound}, {envUntakenBound, &s.Untaken}} { + v, err := settingText(getenv(d.env)) + if err != nil { + return s, err + } + if v == "" { + continue + } + t, err := time.ParseDuration(v) + if err != nil || t <= 0 { + return s, fmt.Errorf("%s holds %q, which is not a positive duration", getenv(d.env), v) + } + *d.to = t + } + if s.Kept > maxRunsKept { + return s, fmt.Errorf("%s holds %s, past test-runs-kept's maximum %s (A-470a)", getenv(envRunsKept), s.Kept, maxRunsKept) + } + return s, nil +} + +// settingText is one setting file's content, trimmed: empty for no path, no file or an empty one. +func settingText(path string) (string, error) { + if path == "" { + return "", nil + } + raw, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return "", nil + } + if err != nil { + return "", err + } + return strings.TrimSpace(string(raw)), nil +} + +// progressSubjects hear every build seat's step-progress (§725), from their contracts. +var progressSubjects = []string{nodebuildgo.SubjectStepProgressAll, nodebuildnode.SubjectStepProgressAll, nodebuildimage.SubjectStepProgressAll} + +// hearStepProgress begins again the clock of the test run whose ask a build seat's holder says progress of (§725, A-466p). +func hearStepProgress(hub handlers, progress func(ask string) error, logf func(string, ...any)) error { + for _, subject := range progressSubjects { + err := hub.Handle(subject, func(m jetstream.Msg) { + var ev nodebuildgo.StepProgress + if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" { + logf("%s is no step-progress: %v", m.Subject(), err) + _ = m.Term() + return + } + if err := progress(ev.Ask); err != nil { + logf("the clock of the test run of ask %s is not begun again, heard again: %v", ev.Ask, err) + _ = m.Nak() + return + } + _ = m.Ack() + }) + if err != nil { + return err + } + } + return nil +} + +// watchRunBounds hears the expiry of each test run's bound key, the bus's marker of the key's end (§725): no timer in the +// process. +func watchRunBounds(ctx context.Context, kv jetstream.KeyValue, expired func(key string) error, logf func(string, ...any)) error { + w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly()) + if err != nil { + return err + } + go func() { + for { + select { + case <-ctx.Done(): + _ = w.Stop() + return + case e, ok := <-w.Updates(): + if !ok { + return + } + if e == nil || e.Operation() != jetstream.KeyValuePurge || !pipeline.IsRunBound(e.Key()) { + continue + } + if err := expired(e.Key()); err != nil { + logf("the test run of the bound %s that ended is not ended: %v", e.Key(), err) + } + } + } + }() + return nil +} + +// openTestRuns opens the state test-runs and wires it into the starter, the hearer and the ager (§725). +func openTestRuns(ctx context.Context, js jetstream.JetStream, cred process.Account, getenv func(string) string, + starter *pipeline.Starter, hearer *pipeline.Hearer, drop pipeline.Drop, logf func(string, ...any)) (jetstream.KeyValue, *runObjects, error) { + settings, err := runSettings(getenv) + if err != nil { + return nil, nil, err + } + kv, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, TestRunsState), "share a test run") + if err != nil { + return nil, nil, err + } + runs := testRunsKV{ctx: ctx, kv: kv} + objects := &runObjects{ctx: ctx, js: js, module: cred.Module} + starter.TestRuns, starter.Clock, starter.Runs = runs, runs, settings + starter.RunsFill = runsFill(ctx, js, cred.Module, time.Now) + starter.EndReaders = hearer.EndReaders + hearer.TestRuns, hearer.Clock, hearer.Runs, hearer.DropRun = runs, runs, settings, drop + hearer.RunOutputs = objects + if hearer.Ager != nil { + hearer.Ager.Runs, hearer.Ager.RunOutputs, hearer.Ager.RunsKept = runs, objects.Delete, settings.Kept + } + logf("test runs: cap %d bytes, kept %s, step bound %s, untaken bound %s", settings.CapBytes, settings.Kept, settings.StepBound, settings.Untaken) + return kv, objects, nil +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_test.go new file mode 100644 index 00000000..c8f14d81 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_test.go @@ -0,0 +1,58 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +// The test runs' settings reach the process through the files the manifest writes (design 47 G §725; A-469x, A-470a): the +// cap a whole number of MiB, the times Go durations, a missing file its default, a content that is neither, or a kept time +// past 168h, an error naming the file. +func TestTheTestRunsSettingsAreReadFromTheirFiles(t *testing.T) { + dir := t.TempDir() + write := func(name, content string) string { + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return p + } + rows := []struct { + name string + files map[string]string + ok bool + check func(t *testing.T, cap int64, kept, step, untaken time.Duration) + }{ + {"defaults", nil, true, func(t *testing.T, cap int64, kept, step, untaken time.Duration) { + if cap != 16<<20 || kept != 168*time.Hour || step != 30*time.Minute || untaken != 24*time.Hour { + t.Errorf("defaults = %d %s %s %s", cap, kept, step, untaken) + } + }}, + {"set", map[string]string{envRunsCap: "8\n", envRunsKept: "48h", envStepBound: "2s", envUntakenBound: "1m"}, true, + func(t *testing.T, cap int64, kept, step, untaken time.Duration) { + if cap != 8<<20 || kept != 48*time.Hour || step != 2*time.Second || untaken != time.Minute { + t.Errorf("settings = %d %s %s %s", cap, kept, step, untaken) + } + }}, + {"a cap that is no whole number", map[string]string{envRunsCap: "8.5"}, false, nil}, + {"a time that is no duration", map[string]string{envStepBound: "30"}, false, nil}, + {"kept past its maximum", map[string]string{envRunsKept: "169h"}, false, nil}, + } + for _, r := range rows { + t.Run(r.name, func(t *testing.T) { + env := map[string]string{} + for k, v := range r.files { + env[k] = write(r.name+k, v) + } + s, err := runSettings(func(k string) string { return env[k] }) + if (err == nil) != r.ok { + t.Fatalf("runSettings = %+v, %v", s, err) + } + if r.check != nil { + r.check(t, s.CapBytes, s.Kept, s.StepBound, s.Untaken) + } + }) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/aged.go b/modules/mesh-delivery-pipeline/internal/pipeline/aged.go index 487cc8c5..948ac76a 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/aged.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/aged.go @@ -53,7 +53,12 @@ type Ager struct { Get(key string) (seat.KeptResult, bool, error) Delete(key string) error } - Now func() time.Time + // Runs is the state test-runs, RunOutputs deletes an object of test-runs-objects and RunsKept is the setting + // test-runs-kept: an ended run is deleted with its output object once RunsKept has passed since its end (§725, A-470a). + Runs TestRuns + RunOutputs func(name string) error + RunsKept time.Duration + Now func() time.Time // Logf is the runner's journal: each delete writes a line naming the key or object and its retention. Logf func(string, ...any) } @@ -69,7 +74,16 @@ func (a *Ager) Age() { if a == nil || a.Log == nil { return } - for _, r := range []Retention{Retention7, Retention30, Retention90} { + retentions := []Retention{Retention7, Retention30, Retention90} + if a.Runs != nil { + // The test runs age first, in the same pass, so the pipeline that names a run's key is still kept (A-470a). + kept := a.RunsKept + if kept <= 0 { + kept = DefaultRunSettings.Kept + } + retentions = append([]Retention{{RetentionTestRuns, kept}}, retentions...) + } + for _, r := range retentions { if err := a.ageAt(r); err != nil { a.logf("aged: the data past %s are not aged: %v", r.Name, err) } @@ -103,8 +117,51 @@ func (a *Ager) ageAt(r Retention) error { return a.Aged.Put(r.Name, moved) } +// RetentionTestRuns is the name, in the state aged, of the test runs' retention, the setting test-runs-kept (§725). +const RetentionTestRuns = "test-runs" + +// ageRun deletes the test run the ended step of e read, and its output object, once its kept time has passed since the +// step's end, in the step log's order (§725, A-470a): the run's key is read from e's pipeline, its step's Shared run. +func (a *Ager) ageRun(r Retention, e seat.LogEntry) { + if e.What != seat.LogEntryWhatEnded { + return + } + p, _, err := a.Pipelines.Get(e.Pipeline) + if err != nil { + return + } + run := "" + for _, s := range p.Steps { + if s.Step == e.Step && s.Artifact == e.Artifact { + run = sharedRun(s) + } + } + if run == "" { + return + } + t, _, err := a.Runs.Get(run) + // The entry is past the retention (ageAt), and a run ends no later than the reader's end it was read at: a run not + // ended yet is aged at a later reader's end. + if err != nil || t.Ended == nil { + return + } + if t.Output != "" && a.RunOutputs != nil { + if err := a.RunOutputs(t.Output); err != nil { + a.logf("aged: the output %s of the test run %s is not deleted, so the run is kept: %v", t.Output, run, err) + return + } + } + if err := a.Runs.Delete(run); err != nil { + a.logf("aged: the test run %s is not deleted: %v", run, err) + return + } + a.logf("aged: deleted the test run %s and its output (%s after its end)", run, r.Age) +} + func (a *Ager) ageEntry(r Retention, e seat.LogEntry, now time.Time) { switch r.Name { + case RetentionTestRuns: + a.ageRun(r, e) case Retention7.Name: if e.Output != "" && a.Outputs != nil { if err := a.Outputs(e.Output); err != nil { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index b8240359..500075d4 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -355,6 +355,8 @@ func (m testRunsMem) Get(key string) (seat.TestRun, uint64, error) { return e.r, e.rev, nil } +func (m testRunsMem) Delete(key string) error { delete(m, key); return nil } + func (m testRunsMem) Update(key string, r seat.TestRun, rev uint64) (uint64, error) { if m[key].rev != rev { return 0, ErrChanged diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go new file mode 100644 index 00000000..7bfd4a94 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go @@ -0,0 +1,326 @@ +package pipeline + +import ( + "errors" + "fmt" + "strings" + "time" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" +) + +// RunSettings are the runner's settings of its test runs (design 47 G §725; A-469d, A-469w, A-469x): the cap of test-runs +// the process judges a write by, how long an ended run is kept, the bound of a taken run's owed progress and the bound of +// an ask never taken. +type RunSettings struct { + CapBytes int64 + Kept time.Duration + StepBound time.Duration + Untaken time.Duration +} + +// The settings' defaults (§725; A-466n, A-466s): 16 MiB, 7 days, 30 minutes, 24 hours. +var DefaultRunSettings = RunSettings{CapBytes: 16 << 20, Kept: 7 * 24 * time.Hour, StepBound: 30 * time.Minute, Untaken: 24 * time.Hour} + +// The two bound keys of a test run in test-runs, each with its own time to live (§725; A-468b, A-470): `bound.`, +// written at the build seat's ask-taken of the run's ask and again at each step-progress of it, and `untaken.`, +// written at the run's start. +const ( + BoundPrefix = "bound." + UntakenPrefix = "untaken." +) + +// RunObjects is the state test-runs' own object store, test-runs-objects (A-469e): a run's output, kept while its key lives. +type RunObjects interface { + Put(name string, data []byte) error + // Get reads an object; ErrNoRun when it is gone. + Get(name string) ([]byte, error) + Delete(name string) error +} + +// RunClock is the bound keys of the test runs, keys of test-runs with their own time to live. +type RunClock interface { + // Bound writes key again with ttl as its time to live. + Bound(key string, ttl time.Duration) error + // Unbound deletes key; a key gone is no fault. + Unbound(key string) error + // Has says whether key is written. + Has(key string) (bool, error) + // Keys are test-runs' keys. + Keys() ([]string, error) +} + +// ParseRunBound reads a key of test-runs as a run's bound: the run's key and whether it is the untaken bound. +func ParseRunBound(key string) (run string, untaken, ok bool) { + if run, ok = strings.CutPrefix(key, BoundPrefix); ok { + return run, false, run != "" + } + if run, ok = strings.CutPrefix(key, UntakenPrefix); ok { + return run, true, run != "" + } + return "", false, false +} + +// IsRunBound says whether a key of test-runs is a bound key rather than a run. +func IsRunBound(key string) bool { + _, _, ok := ParseRunBound(key) + return ok +} + +// runOfAsk is the test run asked under ask: the run its asking step names in Shared, read from the state pipelines. +func (h *Hearer) runOfAsk(ask string) (string, bool) { + if h.TestRuns == nil || ask == "" { + return "", false + } + pkeys, err := h.Store.Keys("*.*.*") + if err != nil { + return "", false + } + for _, key := range pkeys { + p, _, err := h.Store.Get(key) + if err != nil { + continue + } + if i := stepOfAsk(p, ask); i >= 0 { + run := sharedRun(p.Steps[i]) + return run, run != "" + } + } + return "", false +} + +// RunTaken takes the ask-taken of a test run's ask (§725, A-470): the node that took it is kept on the run, its untaken +// bound deleted and its clock begun, `bound.` written with test-step-bound as its time to live. An ask of no test +// run takes nothing. +func (h *Hearer) RunTaken(ask, node string) error { + run, ok := h.runOfAsk(ask) + if !ok { + return nil + } + for { + r, rev, err := h.TestRuns.Get(run) + if errors.Is(err, ErrNoRun) { + return nil + } + if err != nil { + return fmt.Errorf("the test run %s is not read: %w", run, err) + } + if r.Ask != ask || r.Ended != nil { + return nil + } + if r.Node != node { + r.Node = node + if _, err := h.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return fmt.Errorf("the test run %s is not written: %w", run, err) + } + } + break + } + if err := h.Clock.Bound(BoundPrefix+run, h.runSettings().StepBound); err != nil { + return fmt.Errorf("the clock of the test run %s is not begun: %w", run, err) + } + return h.Clock.Unbound(UntakenPrefix + run) +} + +// RunProgress takes a build seat's step-progress of a test run's ask (§725, A-466p): its clock begins again, so a run that +// keeps saying its progress is never ended by it. An ask of no running, taken test run takes nothing. +func (h *Hearer) RunProgress(ask string) error { + run, ok := h.runOfAsk(ask) + if !ok { + return nil + } + r, _, err := h.TestRuns.Get(run) + if errors.Is(err, ErrNoRun) || (err == nil && (r.Ask != ask || r.Ended != nil)) { + return nil + } + if err != nil { + return fmt.Errorf("the test run %s is not read: %w", run, err) + } + return h.Clock.Bound(BoundPrefix+run, h.runSettings().StepBound) +} + +// RunExpired takes the expiry of a test run's bound key (§725, §720 *When the clock ends*, A-466s): the run ends `could not +// run`, naming its ask and the bound, its ask dropped and every reader ended with it. An untaken bound of a run whose ask +// was taken, and a bound of a run ended already, end nothing. +func (h *Hearer) RunExpired(key string) error { + run, untaken, ok := ParseRunBound(key) + if !ok || h.TestRuns == nil { + return nil + } + r, _, err := h.TestRuns.Get(run) + if errors.Is(err, ErrNoRun) { + return nil + } + if err != nil { + return fmt.Errorf("the test run %s is not read: %w", run, err) + } + if r.Ended != nil || (untaken && r.Node != "") { + return nil + } + s := h.runSettings() + why := fmt.Sprintf("the ask %s of %s was taken by %s and said no progress within %s (test-step-bound)", r.Ask, r.Seat, r.Node, s.StepBound) + if untaken { + why = fmt.Sprintf("the ask %s of %s was not taken within %s of the run's start (run-untaken-bound)", r.Ask, r.Seat, s.Untaken) + } + if h.DropRun != nil { + if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil { + h.logf("the drop of ask %s of the test run %s is not answered: %v", r.Ask, run, err) + } + } + return h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil) +} + +// TakeUp takes up again, at the runner's start, each test run with no end (§725): a taken run whose clock is gone, and an +// untaken run whose untaken bound is gone, passed its bound while the runner was away and ends as its expiry would. It +// says how many it ended. +func (h *Hearer) TakeUp() (int, error) { + if h.TestRuns == nil || h.Clock == nil { + return 0, nil + } + all, err := h.Clock.Keys() + if err != nil { + return 0, fmt.Errorf("the test runs are not listed: %w", err) + } + ended := 0 + var first error + for _, run := range all { + if IsRunBound(run) { + continue + } + r, _, err := h.TestRuns.Get(run) + if err != nil || r.Ended != nil { + continue + } + bound := UntakenPrefix + run + if r.Node != "" { + bound = BoundPrefix + run + } + has, err := h.Clock.Has(bound) + if err != nil || has { + if err != nil && first == nil { + first = err + } + continue + } + if err := h.RunExpired(bound); err != nil && first == nil { + first = err + } + ended++ + } + return ended, first +} + +func (h *Hearer) runSettings() RunSettings { + s := h.Runs + d := DefaultRunSettings + if s.CapBytes <= 0 { + s.CapBytes = d.CapBytes + } + if s.Kept <= 0 { + s.Kept = d.Kept + } + if s.StepBound <= 0 { + s.StepBound = d.StepBound + } + if s.Untaken <= 0 { + s.Untaken = d.Untaken + } + return s +} + +// EndRun ends the test run under run with end's outcome, words, counts, node and output, written once by compare-and-set, +// and ends each of its readers with it but skip, the step whose own end ended it (§725, §726). A run ended already is +// not ended again. +func (h *Hearer) EndRun(run string, end seat.TestRun, skip *seat.TestRunReader) error { + var r seat.TestRun + for { + got, rev, err := h.TestRuns.Get(run) + if errors.Is(err, ErrNoRun) { + return nil + } + if err != nil { + return fmt.Errorf("the test run %s is not read: %w", run, err) + } + if got.Ended != nil { + return nil + } + now := h.now() + got.Ended, got.Outcome, got.Said, got.Tests = &now, end.Outcome, end.Said, end.Tests + if end.Node != "" { + got.Node = end.Node + } + if end.Output != "" { + got.Output = end.Output + } + if _, err := h.TestRuns.Update(run, got, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return fmt.Errorf("the test run %s is not written: %w", run, err) + } + r = got + break + } + if h.Clock != nil { + for _, k := range []string{BoundPrefix + run, UntakenPrefix + run} { + if err := h.Clock.Unbound(k); err != nil { + h.logf("the bound %s of the ended test run is not deleted: %v", k, err) + } + } + } + var first error + for _, reader := range r.Readers { + if skip != nil && reader == *skip { + continue + } + if err := h.endReader(reader, run, r); err != nil && first == nil { + first = err + } + } + return first +} + +// endReader ends reader's step of run with the run's result (§726): its outcome, words, counts and node, the step naming +// the run in Shared as it did. A step ended already takes nothing. +func (h *Hearer) endReader(reader seat.TestRunReader, run string, r seat.TestRun) error { + for { + p, rev, err := h.Store.Get(reader.Pipeline) + if err != nil { + return nil // a pipeline gone has no reader left to end + } + i := -1 + for j, s := range p.Steps { + if s.Artifact == reader.Artifact && sharedRun(s) == run && s.Outcome == seat.StepResultOutcomeRunning { + i = j + } + } + if i < 0 { + return nil + } + now := h.now() + s := &p.Steps[i] + s.Outcome, s.Why, s.Node, s.Ended = r.Outcome, r.Said, r.Node, &now + if r.Tests != nil { + t := *r.Tests + s.Tests = &t + } + p.Unlogged = append(p.Unlogged, endEntry(reader.Pipeline, *s, now)) + asked, err := askApproval(reader.Pipeline, &p, now, h.Asker) + if err != nil { + return err + } + ended := endPipeline(&p, now) + if _, err := h.Store.Update(reader.Pipeline, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return fmt.Errorf("pipeline %s cannot be written: %w", reader.Pipeline, err) + } + h.Failures.StepEnded(h.Store, p, *s) + if err := h.say(seat.EventStepEnded, p, s); err != nil { + return fmt.Errorf("pipeline %s's %s is written but not said: %w", reader.Pipeline, seat.EventStepEnded, err) + } + h.appendUnlogged(reader.Pipeline) + return h.said(p, asked, ended) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index a2e793bd..e8ee14d2 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -84,8 +84,14 @@ type Starter struct { // Supersede ends the older running pipelines the pipeline just written supersedes (§31; Hearer.Supersede); nil ends none. Supersede func(newer seat.Pipeline) // TestRuns is the state test-runs, where a test step's run is shared by every step of its candidate, level and digest - // (§724, §725); nil shares none, each step's ask put. - TestRuns TestRuns + // (§724, §725); nil shares none, each step's ask put. Clock writes a run's untaken bound at its start, Runs holds the + // settings, RunsFill reads test-runs' fill, judged against Runs' cap before a run is written (A-469d), and EndReaders ends + // the steps of a run read ended or never asked. + TestRuns TestRuns + Clock RunClock + Runs RunSettings + RunsFill func() (seat.Fill, error) + EndReaders func(run string, readers []seat.TestRunReader, r seat.TestRun) error } // askSteps puts a placed pipeline's asks (§22, §24): a test run another step asked is read, not asked again (share), then diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index a1c7a3c2..654bea6b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -118,6 +118,26 @@ type Hearer struct { Fills *Fills // Widens computes the widens a pipeline-ended carries (§71, A-207); nil carries none. Widens func(seat.Pipeline) []seat.Widening + // TestRuns is the state test-runs and Clock its bound keys (§725); Runs the settings of its runs; DropRun drops a + // run's ask when its run ends unfinished (§119). A nil TestRuns keeps no test run. + TestRuns TestRuns + Clock RunClock + Runs RunSettings + DropRun Drop + // RunOutputs keeps a test run's output in test-runs-objects (§725, A-466v). + RunOutputs RunObjects +} + +// EndReaders ends each of readers' steps of run with r's result (§725, §726): a step that read an ended run, or the steps +// of a run that was never asked. +func (h *Hearer) EndReaders(run string, readers []seat.TestRunReader, r seat.TestRun) error { + var first error + for _, reader := range readers { + if err := h.endReader(reader, run, r); err != nil && first == nil { + first = err + } + } + return first } func (h *Hearer) now() time.Time { @@ -154,6 +174,10 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s at %s (%s)", ErrRefused, ev.ID, ev.Step, ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Commit, p.Fingerprint) case p.Steps[i].Outcome != seat.StepResultOutcomeRunning: + if run := sharedRun(p.Steps[i]); run != "" && h.TestRuns != nil { + // The run ended before its ask's end came (its bound, a drop): the end is ignored and said (§720, §725). + h.logf("the step-ended of ask %s is ignored: its test run %s ended before it (%s)", ev.ID, run, p.Steps[i].Why) + } return p, nil case !endOutcome(ev.Outcome): return p, fmt.Errorf("%w: ask %s ended %q, not an outcome of step-ended", ErrRefused, ev.ID, ev.Outcome) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go index 77b2e435..1b1b787b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go @@ -23,6 +23,8 @@ type TestRuns interface { Get(key string) (seat.TestRun, uint64, error) // Update writes key at the revision read; ErrChanged when another write came between. Update(key string, r seat.TestRun, revision uint64) (uint64, error) + // Delete deletes key; a key gone is no fault. + Delete(key string) error } // ErrNoRun is a Get of a test-runs key that is not written. @@ -127,7 +129,16 @@ func (s *Starter) share(key string, asks []Asked) []Asked { continue } run, ok, err := s.claimRun(a) + var full *fullErr switch { + case errors.As(err, &full): + // At the cap no run is asked: the steps end could not run, naming the fill (§725, A-469d). + s.logf("the test run %s of %s is not asked: %v", a.Run, key, err) + if s.EndReaders != nil { + if err := s.EndReaders(a.Run, a.Readers, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: err.Error()}); err != nil { + s.logf("the steps of the test run %s, not asked, are not ended: %v", a.Run, err) + } + } case err != nil: // A run that cannot be written or read is asked as the step's own, said: slower, never lost. s.logf("the test run %s of %s is not written, so the step is asked on its own: %v", a.Run, key, err) @@ -144,15 +155,67 @@ func (s *Starter) share(key string, asks []Asked) []Asked { return out } +// fullErr is a run not written because test-runs is at its cap (A-469d), naming the fill. +type fullErr struct{ fill seat.Fill } + +func (e *fullErr) Error() string { + return fmt.Sprintf("the state %s holds %d of its %d bytes, so no test run is asked (test-runs-cap-mib)", e.fill.Store, e.fill.Bytes, e.fill.Cap) +} + +// atCap is the error of test-runs at its cap setting, nil below it or when its fill is not read (said). +func (s *Starter) atCap() error { + if s.RunsFill == nil { + return nil + } + f, err := s.RunsFill() + if err != nil { + s.logf("the fill of test-runs is not read, so its cap is not judged: %v", err) + return nil + } + cap := s.Runs.CapBytes + if cap <= 0 { + cap = DefaultRunSettings.CapBytes + } + if f.Bytes >= cap { + f.Cap = cap + return &fullErr{fill: f} + } + return nil +} + +// started writes a new run and its untaken bound (§725, A-470). +func (s *Starter) started(a Asked, write func() error) error { + if err := s.atCap(); err != nil { + return err + } + if err := write(); err != nil { + return err + } + if s.Clock != nil { + untaken := s.Runs.Untaken + if untaken <= 0 { + untaken = DefaultRunSettings.Untaken + } + if err := s.Clock.Bound(UntakenPrefix+a.Run, untaken); err != nil { + s.logf("the untaken bound of the test run %s is not written: %v", a.Run, err) + } + } + return nil +} + // claimRun writes a's test run create-only; ok says the ask is put (the key is a's own), else run is the run it reads. func (s *Starter) claimRun(a Asked) (seat.TestRun, bool, error) { now := s.now() value := seat.TestRun{Level: a.Level, Digest: a.Digest, Seat: a.Seat, Ask: a.ID, Started: &now, Readers: a.Readers} for { - err := s.TestRuns.Create(a.Run, value) + err := s.started(a, func() error { return s.TestRuns.Create(a.Run, value) }) if err == nil { return value, true, nil } + var full *fullErr + if errors.As(err, &full) { + return seat.TestRun{}, false, err + } if !errors.Is(err, ErrExists) { return seat.TestRun{}, false, err } @@ -167,7 +230,11 @@ func (s *Starter) claimRun(a Asked) (seat.TestRun, bool, error) { return r, true, nil } if r.Ended != nil && r.Outcome == seat.StepResultOutcomeCouldNotRun { - if _, err := s.TestRuns.Update(a.Run, value, rev); errors.Is(err, ErrChanged) { + err := s.started(a, func() error { + _, err := s.TestRuns.Update(a.Run, value, rev) + return err + }) + if errors.Is(err, ErrChanged) { continue } else if err != nil { return seat.TestRun{}, false, err diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go new file mode 100644 index 00000000..aee903a9 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -0,0 +1,221 @@ +package pipeline + +import ( + "fmt" + "strings" + "testing" + "time" + + catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" +) + +// clockMem is the bound keys of test-runs in memory, each with the time to live it was last written with. +type clockMem struct { + runs testRunsMem + bound map[string]time.Duration +} + +func (c *clockMem) Bound(key string, ttl time.Duration) error { c.bound[key] = ttl; return nil } +func (c *clockMem) Unbound(key string) error { delete(c.bound, key); return nil } +func (c *clockMem) Has(key string) (bool, error) { _, ok := c.bound[key]; return ok, nil } +func (c *clockMem) Keys() ([]string, error) { + var out []string + for k := range c.runs { + out = append(out, k) + } + for k := range c.bound { + out = append(out, k) + } + return out, nil +} + +type objectsMem map[string][]byte + +func (o objectsMem) Put(name string, data []byte) error { o[name] = data; return nil } +func (o objectsMem) Get(name string) ([]byte, error) { + if d, ok := o[name]; ok { + return d, nil + } + return nil, ErrNoRun +} +func (o objectsMem) Delete(name string) error { delete(o, name); return nil } + +// runFixture is a starter and a hearer over one pipelines store and one test-runs, the module's program and tools testing +// one directory. +type runFixture struct { + store memStore + runs testRunsMem + clock *clockMem + objects objectsMem + asks *stepAsks + drops []string + said []seat.StepEvent + logged []string + fill int64 + s *Starter + h *Hearer +} + +func newRunFixture() *runFixture { + f := &runFixture{store: memStore{}, runs: testRunsMem{}, objects: objectsMem{}, asks: &stepAsks{}} + f.clock = &clockMem{runs: f.runs, bound: map[string]time.Duration{}} + reg := catalog.Registration{Module: "app", Branches: []string{"main"}} + say := func(_ string, body any) error { + if e, ok := body.(seat.StepEvent); ok { + f.said = append(f.said, e) + } + return nil + } + f.h = &Hearer{Store: f.store, Say: say, TestRuns: f.runs, Clock: f.clock, RunOutputs: f.objects, + Logf: func(format string, a ...any) { f.logged = append(f.logged, fmt.Sprintf(format, a...)) }, + DropRun: func(seatName, node, ask, why string) error { + f.drops = append(f.drops, seatName+" "+node+" "+ask) + return nil + }} + f.s = &Starter{Pipelines: memCreated{f.store}, Store: f.store, TestRuns: f.runs, Clock: f.clock, Say: say, Steps: f.asks, + Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }, EndReaders: f.h.EndReaders, + RunsFill: func() (seat.Fill, error) { return seat.Fill{Store: "test-runs", Bytes: f.fill}, nil }, Logf: f.h.Logf} + return f +} + +func (f *runFixture) start(t *testing.T, source, revision int64) seat.Pipeline { + t.Helper() + entries := map[string]catalog.CatalogEntry{ + "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, + "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, + } + if ok, err := f.s.Start(Begin{Branch: "main", Source: source, Module: "app", Commit: "c1", Fp: "src2:aa", Revision: revision, Entries: entries}); !ok || err != nil { + t.Fatalf("start = %v, %v", ok, err) + } + p, _, _ := f.store.Get(Key("main", source, "app")) + return p +} + +func (f *runFixture) pipeline(p seat.Pipeline) seat.Pipeline { + q, _, _ := f.store.Get(p.Pipeline) + return q +} + +// The test runs' clocks and cap (design 47 G §725; A-466n, A-466p, A-466s, A-468b, A-469d, A-470, A-470a): a run's untaken +// bound is written at its start, its clock at ask-taken and again at each step-progress; a clock's expiry ends the run and +// every reader could not run and drops its ask, a later step-ended ignored and journaled; an untaken bound of a taken run +// ends nothing; a run whose bound passed while the runner was away ends at its start; at the cap no run is asked; an ended +// run is aged with its output object in the step log's order. +func TestATestRunHasOneClockACapAndAnAge(t *testing.T) { + t.Run("clock", func(t *testing.T) { + f := newRunFixture() + f.h.Runs = RunSettings{StepBound: 30 * time.Minute, Untaken: 24 * time.Hour} + f.s.Runs = f.h.Runs + p := f.start(t, 4, 9) + unit := stepNamed(p, "app:program", "unit") + run := unit.Shared[0].Run + if f.clock.bound[UntakenPrefix+run] != 24*time.Hour || len(f.clock.bound) != 3 { + t.Fatalf("bounds at the start = %v, want the three runs' untaken bounds of 24h", f.clock.bound) + } + if err := f.h.RunTaken(unit.Ask, "ace"); err != nil { + t.Fatal(err) + } + if r, _, _ := f.runs.Get(run); r.Node != "ace" || f.clock.bound[BoundPrefix+run] != 30*time.Minute { + t.Fatalf("after ask-taken the run = %+v, bounds %v; want its node and its clock of 30m", r, f.clock.bound) + } + if _, ok := f.clock.bound[UntakenPrefix+run]; ok { + t.Error("the untaken bound of a taken run is kept") + } + // An untaken bound of a taken run ends nothing; progress begins the clock again. + if err := f.h.RunExpired(UntakenPrefix + run); err != nil { + t.Fatal(err) + } + delete(f.clock.bound, BoundPrefix+run) + if err := f.h.RunProgress(unit.Ask); err != nil || f.clock.bound[BoundPrefix+run] != 30*time.Minute { + t.Fatalf("progress = %v, bounds %v: want the clock begun again", err, f.clock.bound) + } + if r, _, _ := f.runs.Get(run); r.Ended != nil { + t.Fatal("a run is ended by its untaken bound after it was taken") + } + if err := f.h.RunExpired(BoundPrefix + run); err != nil { + t.Fatal(err) + } + r, _, _ := f.runs.Get(run) + if r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, unit.Ask) || !strings.Contains(r.Said, "30m") { + t.Errorf("the run whose clock ended = %+v, want could not run naming its ask and the bound", r) + } + if len(f.drops) != 1 || f.drops[0] != "node-build-go ace "+unit.Ask { + t.Errorf("drops = %v, want the run's ask dropped of ace", f.drops) + } + q := f.pipeline(p) + for _, art := range []string{"app:program", "app:tools"} { + if s := stepNamed(q, art, "unit"); s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != r.Said { + t.Errorf("the %s unit step = %+v, want could not run with the run", art, s) + } + } + ended := time.Now() + if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask, + Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "passed", Ended: &ended}); err != nil { + t.Fatal(err) + } + if s := stepNamed(f.pipeline(p), "app:program", "unit"); s.Outcome != seat.StepResultOutcomeCouldNotRun { + t.Errorf("a step-ended after the run ended moved the step to %s", s.Outcome) + } + if !strings.Contains(strings.Join(f.logged, "\n"), "the step-ended of ask "+unit.Ask+" is ignored: its test run "+run) { + t.Errorf("the ignored step-ended is not journaled: %q", f.logged) + } + // An untaken run whose bound passed while the runner was away ends at the start. + lint := stepNamed(p, "app:program", "lint").Shared[0].Run + delete(f.clock.bound, UntakenPrefix+lint) + if n, err := f.h.TakeUp(); n != 1 || err != nil { + t.Fatalf("take-up ended %d, %v; want the lint run", n, err) + } + if r, _, _ := f.runs.Get(lint); r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, "run-untaken-bound") { + t.Errorf("the lint run taken up = %+v", r) + } + }) + t.Run("cap", func(t *testing.T) { + f := newRunFixture() + f.s.Runs.CapBytes, f.fill = 1<<20, 1<<20 + p := f.start(t, 4, 9) + for _, a := range f.asks.ids { + if a == stepNamed(p, "app:program", "unit").Ask { + t.Error("a unit run is asked at the cap") + } + } + q := f.pipeline(p) + for _, art := range []string{"app:program", "app:tools"} { + if s := stepNamed(q, art, "unit"); s.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(s.Why, "1048576 of its 1048576 bytes") { + t.Errorf("the %s unit step at the cap = %+v, want could not run naming the fill", art, s) + } + } + }) + t.Run("age", func(t *testing.T) { + f := newRunFixture() + p := f.start(t, 4, 9) + unit := stepNamed(p, "app:program", "unit") + run := unit.Shared[0].Run + _ = f.h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomePassed, Output: "out-1"}, nil) + f.objects["out-1"] = []byte("ok") + now := agedNow + a := &Ager{Runs: f.runs, RunOutputs: f.objects.Delete, RunsKept: 7 * 24 * time.Hour, Pipelines: agedDeleter{f.store}, + Aged: agedState{}, Now: func() time.Time { return now }, + Log: func(from uint64, limit int) ([]LoggedEntry, error) { + if from > 1 { + return nil, nil + } + return []LoggedEntry{{Sequence: 1, Entry: seat.LogEntry{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:tools", + What: seat.LogEntryWhatEnded, At: agedNow.Add(-6 * 24 * time.Hour)}}}, nil + }} + a.Age() + if _, _, err := f.runs.Get(run); err != nil { + t.Fatal("a run is aged before test-runs-kept passed") + } + now = now.Add(2 * 24 * time.Hour) + a.Age() + if _, _, err := f.runs.Get(run); err != ErrNoRun || len(f.objects) != 0 { + t.Errorf("the run past test-runs-kept is %v, objects %v; want both deleted", err, f.objects) + } + }) +} + +// agedDeleter is a memStore with the delete the ager makes. +type agedDeleter struct{ memStore } + +func (d agedDeleter) Delete(key string) error { delete(d.memStore, key); return nil } diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index 80d88e8a..cc545315 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -55,9 +55,12 @@ func TestManifest(t *testing.T) { {"claims the seat serving every verb of its definition", len(m.Claims) == 1 && m.Claims[0].Name == seat.Seat && equalSets(m.Claims[0].Serves, seat.Verbs())}, // §27, §85, §330, §355, §77: the old state sections. - {"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "recoveries", "bounds", "toolchains", "findings", "first-views", "consumers"})}, + {"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "recoveries", "bounds", "test-runs", "toolchains", "findings", "first-views", "consumers"})}, // §535, §647, A-77, A-104: an ask no holder takes is bound by a key with its own time to live. {"bounds is key-ttl", keyTTL["bounds"]}, + // §725, A-469w, A-470: test-runs holds its runs' bound keys, each with its own time to live; 64 MiB its ceiling. + {"test-runs is key-ttl of 64 MiB", keyTTL["test-runs"] && stateMiB["test-runs"] == 64}, + {"the test runs' settings", runSettingsOf(raw)}, // RN25 (§469, §85): an old state section declares its cap, pipelines 256 MiB and kept 128 MiB. {"pipelines 256 MiB", stateMiB[seat.StatePipelines] == 256}, {"kept 128 MiB", stateMiB[seat.StateKept] == 128}, @@ -78,7 +81,8 @@ func TestManifest(t *testing.T) { {"consumes", equalSets(m.Consumes, []string{ "mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-builds.step-ended", "operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended", - "node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", "mesh-wants.step-ended", "mesh-catalog.registration-changed", + "node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", + "node-build-go.step-progress", "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked", "mesh-delivery.recovery-merged", // A-186: the seat list its write verbs' callers are checked against, heard as it changes. "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed"})}, @@ -175,3 +179,43 @@ func equalSets(a, b []string) bool { slices.Sort(y) return slices.Equal(x, y) } + +// runSettingsOf says whether the manifest holds the test runs' four settings (design 47 G §725; A-469w, A-469x): the cap a +// whole number of at most 64, the three times Go durations, each written into the file its environment names. +func runSettingsOf(raw []byte) bool { + var m struct { + Config map[string]map[string]any `json:"config"` + Resources []struct { + Type string `json:"type"` + Path string `json:"path"` + Content string `json:"content"` + Env map[string]string `json:"env"` + } `json:"resources"` + } + if json.Unmarshal(raw, &m) != nil { + return false + } + want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string"} + if len(m.Config) != len(want) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) { + return false + } + env := map[string]bool{} + for _, r := range m.Resources { + for _, v := range r.Env { + env[v] = true + } + } + for key, kind := range want { + if m.Config[key]["type"] != kind { + return false + } + found := false + for _, r := range m.Resources { + found = found || (r.Type == "file" && r.Content == "${config:"+key+"}" && env[r.Path]) + } + if !found { + return false + } + } + return true +} diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index 9198f2fa..eec137ad 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -41,6 +41,9 @@ "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", + "node-build-go.step-progress", + "node-build-node.step-progress", + "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", @@ -79,6 +82,34 @@ "uses": [ "operator-channel" ], + "config": { + "test-runs-cap-mib": { + "type": "number", + "description": "MiB of the state test-runs at which the runner asks no new test run and ends the step could not run, naming the fill (design 47 G section 725, A-469d); the manifest's max-mib is the ceiling.", + "minimum": 1, + "maximum": 64, + "trusted": false, + "default": 16 + }, + "test-runs-kept": { + "type": "string", + "description": "how long an ended test run and its output object are kept after its end, a Go duration such as 168h (design 47 G section 725, A-469x, A-470a); at most 168h, so the pipeline naming its key is still kept.", + "trusted": false, + "default": "168h" + }, + "test-step-bound": { + "type": "string", + "description": "how long a taken test run's ask may say no progress before the run ends could not run, a Go duration such as 30m, counted from the build seat's ask-taken and again from each step-progress (design 47 G section 725, A-466n, A-469x).", + "trusted": false, + "default": "30m" + }, + "run-untaken-bound": { + "type": "string", + "description": "how long a test run's ask may wait untaken after the run's start before the run ends could not run, a Go duration such as 24h (design 47 G section 725, A-466s, A-469x).", + "trusted": false, + "default": "24h" + } + }, "state": [ { "name": "pipelines", @@ -107,6 +138,12 @@ "history": 1, "key-ttl": true }, + { + "name": "test-runs", + "history": 1, + "max-mib": 64, + "key-ttl": true + }, { "name": "findings", "history": 1 @@ -162,6 +199,38 @@ "place": ".", "unknown": "program" }, + { + "id": "test-runs-cap", + "type": "file", + "path": "${dir:state}/test-runs-cap-mib", + "owner": "mesh-delivery-pipeline", + "mode": "0600", + "content": "${config:test-runs-cap-mib}" + }, + { + "id": "test-runs-kept", + "type": "file", + "path": "${dir:state}/test-runs-kept", + "owner": "mesh-delivery-pipeline", + "mode": "0600", + "content": "${config:test-runs-kept}" + }, + { + "id": "test-step-bound", + "type": "file", + "path": "${dir:state}/test-step-bound", + "owner": "mesh-delivery-pipeline", + "mode": "0600", + "content": "${config:test-step-bound}" + }, + { + "id": "run-untaken-bound", + "type": "file", + "path": "${dir:state}/run-untaken-bound", + "owner": "mesh-delivery-pipeline", + "mode": "0600", + "content": "${config:run-untaken-bound}" + }, { "id": "pipeline", "type": "process", @@ -171,7 +240,11 @@ "./mesh-delivery-pipeline" ], "env": { - "MESH_BROKER_FILE": "${dir:state}/broker" + "MESH_BROKER_FILE": "${dir:state}/broker", + "MESH_RUNNER_TEST_RUNS_CAP_FILE": "${dir:state}/test-runs-cap-mib", + "MESH_RUNNER_TEST_RUNS_KEPT_FILE": "${dir:state}/test-runs-kept", + "MESH_RUNNER_TEST_STEP_BOUND_FILE": "${dir:state}/test-step-bound", + "MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE": "${dir:state}/run-untaken-bound" }, "user": "mesh-delivery-pipeline", "health": { From 97f68b04ea1f86e459482700d76b47f17a6fa39e Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 11:33:54 +0100 Subject: [PATCH 03/14] runner: a test run's end moves every step reading it, counted once, dropped only with its last reader (R3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The run's ask's end now ends each reading step with the run's outcome, counts and shared, its output kept in test-runs-objects; a pipeline counts a run's tests once; a superseded or cancelled step leaves the run, the last to leave dropping its ask and ending it; a composition step keeps the runs mesh-wants names. Builds: design 47 G §726, §725 (output object), §724 (an ended run read), §729; A-466i, A-468d, A-469m, A-469t, A-469z --- .../cmd/mesh-delivery-pipeline/steps.go | 6 +- .../internal/pipeline/runends.go | 54 ++++++++++++++ .../internal/pipeline/steps.go | 73 ++++++++++++++++++- .../internal/pipeline/supersede.go | 6 +- .../internal/pipeline/testruns.go | 15 ++++ .../internal/pipeline/testruns_test.go | 73 +++++++++++++++++++ .../internal/pipeline/verbs.go | 7 ++ 7 files changed, 229 insertions(+), 5 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go index ee7d7ded..4e01276d 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go @@ -114,7 +114,11 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any)) _ = m.Term() return } - _, err = h.OnStepEnded(bs, m.Subject(), ev) + var shared []seat.Shared + if bs.Shared != nil { + shared = bs.Shared(m.Data()) + } + _, err = h.OnStepEndedShared(bs, m.Subject(), ev, shared) switch { case errors.Is(err, pipeline.ErrRefused): logf("%v", err) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go index 7bfd4a94..ade028a9 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go @@ -324,3 +324,57 @@ func (h *Hearer) endReader(reader seat.TestRunReader, run string, r seat.TestRun return h.said(p, asked, ended) } } + +// NoReaderLeft is what a test run whose last reader left ends with (A-469z): its ask is dropped, and a later step of the +// candidate starts a new run. +const NoReaderLeft = "no reader is left" + +// leaveRun takes the step of pipeline key that read or asked a test run out of the run's readers (§726): a step superseded +// or cancelled. When no reader is left the run ends `could not run`, naming that (A-469z), and its ask is dropped (§119); +// while one is left the ask stays, so a superseded pipeline never takes away another's run. A step of no test run leaves +// nothing. +func (h *Hearer) leaveRun(key string, s seat.StepResult, why string) { + run := sharedRun(s) + if run == "" || h.TestRuns == nil { + return + } + reader := seat.TestRunReader{Pipeline: key, Artifact: s.Artifact} + for { + r, rev, err := h.TestRuns.Get(run) + if errors.Is(err, ErrNoRun) { + return + } + if err != nil { + h.logf("the test run %s is not read, so the step %s of %s does not leave it: %v", run, s.Step, key, err) + return + } + left := r.Readers[:0:0] + for _, o := range r.Readers { + if o != reader { + left = append(left, o) + } + } + if len(left) == len(r.Readers) { + return + } + r.Readers = left + if _, err := h.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + h.logf("the step %s of %s does not leave the test run %s: %v", s.Step, key, run, err) + return + } + if len(left) > 0 || r.Ended != nil { + return + } + if h.DropRun != nil { + if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil { + h.logf("the drop of ask %s of the test run %s, which no reader is left of, is not answered: %v", r.Ask, run, err) + } + } + if err := h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: NoReaderLeft}, nil); err != nil { + h.logf("the test run %s, which no reader is left of, is not ended: %v", run, err) + } + return + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 654bea6b..88bbacc7 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -26,6 +26,9 @@ type StepSeat struct { Prefix string // Decode reads the seat's step-ended body; nil is the build seats' one form. Decode func([]byte) (StepEnded, error) + // Shared reads the runs a step-ended names in `shared`, carried into the step's result as they came (A-468d); nil for + // a seat whose ends name none. + Shared func([]byte) []seat.Shared } // StepSeats are the three build seats, from their contracts. @@ -36,7 +39,25 @@ var StepSeats = []StepSeat{ // The seat mesh-builds answers the build step of a candidate, one entry named by its fingerprint (A-97, A-99). {Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded}, // The seat mesh-wants answers the composition step (§22, §24). - {Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded}, + {Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded, Shared: wantsShared}, +} + +// wantsShared is the composition runs a mesh-wants step-ended names, one per machine (design 47 G §723, §729; A-468d, +// A-469t): the same form, under the same name. +func wantsShared(raw []byte) []seat.Shared { + var w wants.StepEnded + if json.Unmarshal(raw, &w) != nil || len(w.Shared) == 0 { + return nil + } + out := make([]seat.Shared, 0, len(w.Shared)) + for _, x := range w.Shared { + e := seat.Shared{Machine: x.Machine, Run: x.Run, Ask: x.Ask} + if x.AskedBy != nil { + e.AskedBy = &seat.SharedAskedBy{Pipeline: x.AskedBy.Pipeline, Artifact: x.AskedBy.Artifact} + } + out = append(out, e) + } + return out } // decodeWantsEnded reads the composition step's end of the seat mesh-wants as the form the hearer holds each end to; a @@ -152,6 +173,13 @@ func (h *Hearer) now() time.Time { // another seat, commit, fingerprint, module or step (ADR 0666 §21; the bus grants step-ended. to that node's // holder alone, so the subject's node is the account's). A step already ended takes nothing (the bus gives an event again). func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pipeline, error) { + return h.OnStepEndedShared(bs, subject, ev, nil) +} + +// OnStepEndedShared is OnStepEnded of an end naming the runs it read in shared (A-468d): kept on the step's result. The end +// of a test run's ask also ends the run, its output kept in test-runs-objects, and every other step reading it (§725, §726, +// A-469m). +func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared) (seat.Pipeline, error) { node := strings.TrimPrefix(subject, bs.Prefix) if node == subject || node == "" || strings.Contains(node, ".") { return seat.Pipeline{}, fmt.Errorf("%w: %s is not a subject of %s's step-ended", ErrRefused, subject, bs.Seat) @@ -175,6 +203,10 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Commit, p.Fingerprint) case p.Steps[i].Outcome != seat.StepResultOutcomeRunning: if run := sharedRun(p.Steps[i]); run != "" && h.TestRuns != nil { + if r, _, err := h.TestRuns.Get(run); err == nil && r.Ended == nil && r.Ask == ev.ID { + // The asking step's end was written and its run's was not (a write failed): the run ends now, heard again. + return p, h.endRunOf(p, p.Steps[i], ev) + } // The run ended before its ask's end came (its bound, a drop): the end is ignored and said (§720, §725). h.logf("the step-ended of ask %s is ignored: its test run %s ended before it (%s)", ev.ID, run, p.Steps[i].Why) } @@ -185,6 +217,9 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi now := h.now() s := &p.Steps[i] s.Outcome, s.Node, s.Why, s.Rerun, s.By, s.Ended = ev.Outcome, node, ev.Said, ev.Rerun, ev.By, &now + if len(shared) > 0 { + s.Shared = shared + } if ev.Tests != nil { s.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration} } @@ -237,10 +272,34 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi h.appendUnlogged(ev.Pipeline) h.Ager.Age() h.Fills.Check() - return p, h.said(p, asked, ended) + if err := h.said(p, asked, ended); err != nil { + return p, err + } + return p, h.endRunOf(p, *s, ev) } } +// endRunOf ends the test run step s of p asked, with the end ev of its ask (§725, §726): the run's output kept in +// test-runs-objects under the run's key, its outcome, words, counts and node written, and every other reader ended with it. +// A step that asked no run ends nothing more. +func (h *Hearer) endRunOf(p seat.Pipeline, s seat.StepResult, ev StepEnded) error { + run := sharedRun(s) + if run == "" || h.TestRuns == nil || s.Ask != ev.ID { + return nil + } + end := seat.TestRun{Outcome: ev.Outcome, Said: ev.Said, Node: ev.Node} + if ev.Tests != nil { + end.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration} + } + if ev.Output != "" && h.RunOutputs != nil { + if err := h.RunOutputs.Put(run, []byte(ev.Output)); err != nil { + return fmt.Errorf("the output of the test run %s cannot be kept: %w", run, err) + } + end.Output = run + } + return h.EndRun(run, end, &seat.TestRunReader{Pipeline: p.Pipeline, Artifact: s.Artifact}) +} + // said says what a written pipeline became: an approval asked is a wait held on the operator (ADR 0283), never a stall. func (h *Hearer) said(p seat.Pipeline, asked, ended bool) error { if asked { @@ -344,11 +403,19 @@ const ( budgetIntegration = 50 ) -// testCounts is the module's test counts, the sum of its test steps' own counts; nil when no step counted any. +// testCounts is the module's test counts, the sum of its test steps' own counts, each test run counted once however many +// steps read it (§726, A-466i); nil when no step counted any. func testCounts(steps []seat.StepResult) *seat.TestCounts { var t seat.TestCounts counted := false + runs := map[string]bool{} for _, s := range steps { + if run := sharedRun(s); run != "" && s.Seat != wants.Seat { + if runs[run] { + continue + } + runs[run] = true + } if s.Tests != nil { t.Unit, t.Integration, counted = t.Unit+s.Tests.Unit, t.Integration+s.Tests.Integration, true } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go index a729b552..0a307702 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go @@ -113,7 +113,8 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting { continue } - if s.Ask != "" && BoundSeat(s.Seat) { + // A step of a test run leaves it: its ask is dropped only with its last reader (§726, leaveRun). + if s.Ask != "" && BoundSeat(s.Seat) && sharedRun(*s) == "" { drops = append(drops, *s) } s.Outcome, s.Why, s.Ended = seat.StepResultOutcomeSuperseded, why, ptrTime(now) @@ -136,6 +137,9 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop if len(changed) > 0 { h.appendUnlogged(key) } + for _, i := range changed { + h.leaveRun(key, p.Steps[i], why) + } for _, s := range drops { if drop != nil { if err := drop(s.Seat, s.Node, s.Ask, why); err != nil { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go index 1b1b787b..f382d652 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go @@ -151,6 +151,21 @@ func (s *Starter) share(key string, asks []Asked) []Asked { } if len(joined) > 0 { s.readOthers(key, joined) + // A run that ended within the candidate is read at once (§724): its readers end with its result. + for run, r := range joined { + if r.Ended == nil || s.EndReaders == nil { + continue + } + var readers []seat.TestRunReader + for _, a := range asks { + if a.Run == run { + readers = a.Readers + } + } + if err := s.EndReaders(run, readers, r); err != nil { + s.logf("the steps of %s reading the ended test run %s are not ended: %v", key, run, err) + } + } } return out } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go index aee903a9..ab79a96b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -8,6 +8,7 @@ import ( catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" ) // clockMem is the bound keys of test-runs in memory, each with the time to live it was last written with. @@ -219,3 +220,75 @@ func TestATestRunHasOneClockACapAndAnAge(t *testing.T) { type agedDeleter struct{ memStore } func (d agedDeleter) Delete(key string) error { delete(d.memStore, key); return nil } + +// How a test run ends and is said (design 47 G §726; A-466i, A-468d, A-469m, A-469z): the run's ask's end ends every step +// reading it with its outcome, counts and shared, its output kept in test-runs-objects; a pipeline counts the run's tests +// once; a step superseded leaves the run to its other readers, and the last to leave drops its ask and ends it could not +// run, no reader left; a composition step's end keeps the runs mesh-wants names. +func TestATestRunsEndMovesEveryReaderAndIsCountedOnce(t *testing.T) { + f := newRunFixture() + a := f.start(t, 4, 9) + b := f.start(t, 6, 9) // the same candidate: it reads a's runs + unit := stepNamed(a, "app:program", "unit") + ended := time.Now() + if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask, + Pipeline: a.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed", + Said: "TestX failed", Output: "--- FAIL: TestX", Tests: &nodebuildgo.TestCounts{Unit: 5}, Ended: &ended}); err != nil { + t.Fatal(err) + } + for _, p := range []seat.Pipeline{f.pipeline(a), f.pipeline(b)} { + for _, art := range []string{"app:program", "app:tools"} { + s := stepNamed(p, art, "unit") + if s.Outcome != seat.StepResultOutcomeFailed || s.Why != "TestX failed" || s.Tests == nil || s.Tests.Unit != 5 || + len(s.Shared) != 1 || s.Shared[0].Run != unit.Shared[0].Run || s.Shared[0].Ask != unit.Ask { + t.Errorf("%s's %s unit step = %+v, want failed with the run's result and shared", p.Pipeline, art, s) + } + } + if c := testCounts(p.Steps); c == nil || c.Unit != 5 { + t.Errorf("%s counts %+v, want the run's 5 unit tests once", p.Pipeline, c) + } + } + if r, _, _ := f.runs.Get(unit.Shared[0].Run); r.Outcome != "failed" || r.Output != unit.Shared[0].Run || string(f.objects[r.Output]) != "--- FAIL: TestX" { + t.Errorf("the ended run = %+v, objects %v; want its output kept under its key", r, f.objects) + } + saidReaders := 0 + for _, e := range f.said { + if e.Result.Step == "unit" && e.Result.Outcome == "failed" { + saidReaders++ + } + } + if saidReaders != 4 { + t.Errorf("said %d unit step-ended, want one per reading step", saidReaders) + } + // a, superseded, leaves the lint run to b's readers; b, superseded after, leaves none: the ask is dropped and the run ends. + lint := stepNamed(a, "app:program", "lint") + if _, err := f.h.Supersede(seat.Pipeline{Pipeline: "main.5.app", Branch: "main", Source: 5, Module: "app", Commit: "c2", Fingerprint: "src2:bb"}, nil, nil); err != nil { + t.Fatal(err) + } + if r, _, _ := f.runs.Get(lint.Shared[0].Run); r.Ended != nil || len(r.Readers) != 2 || len(f.drops) != 0 { + t.Fatalf("after a left the lint run = %+v, drops %v; want it running for b's two readers", r, f.drops) + } + if _, err := f.h.Supersede(seat.Pipeline{Pipeline: "main.7.app", Branch: "main", Source: 7, Module: "app", Commit: "c3", Fingerprint: "src2:cc"}, nil, nil); err != nil { + t.Fatal(err) + } + if r, _, _ := f.runs.Get(lint.Shared[0].Run); r.Outcome != seat.StepResultOutcomeCouldNotRun || r.Said != NoReaderLeft || len(f.drops) != 2 || f.drops[0] != "node-build-go "+lint.Ask { + t.Errorf("after the last reader left the lint run = %+v, drops %v; want it ended, no reader left, its ask and the integration run's dropped", r, f.drops) + } + // A composition step's end keeps the runs mesh-wants names, as they came (A-468d). + c := newRunFixture() + p := c.start(t, 4, 9) + comp := stepNamed(p, "", "composition") + bs := stepSeatOf(t, "mesh-wants") + raw := []byte(`{"id":"` + comp.Ask + `","pipeline":"main.4.app","step":"composition","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"passed",` + + `"shared":[{"machine":"ace","run":"main.9.ace","ask":"r1"}]}`) + ev, err := bs.Decode(raw) + if err != nil { + t.Fatal(err) + } + if _, err := c.h.OnStepEndedShared(bs, bs.Prefix+"n1", ev, bs.Shared(raw)); err != nil { + t.Fatal(err) + } + if s := stepNamed(c.pipeline(p), "", "composition"); len(s.Shared) != 1 || s.Shared[0] != (seat.Shared{Machine: "ace", Run: "main.9.ace", Ask: "r1"}) { + t.Errorf("the composition step's shared = %+v, want mesh-wants' as it came", s.Shared) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go index b12a19f7..91cbdf6e 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -206,11 +206,15 @@ func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, er } now := h.now() var drop []string + var leave []seat.StepResult for i := range p.Steps { s := &p.Steps[i] if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting { continue } + if sharedRun(*s) != "" && s.Seat != wants.Seat { + leave = append(leave, *s) // it leaves its test run, dropped only with the last reader (§726) + } if s.Ask != "" { drop = append(drop, s.Ask) } @@ -223,6 +227,9 @@ func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, er } else if err != nil { return seat.CancelAnswer{}, fmt.Errorf("pipeline %s cannot be written: %w", in.Pipeline, err) } + for _, s := range leave { + h.leaveRun(in.Pipeline, s, "the pipeline "+in.Pipeline+" is cancelled: "+in.Why) + } if v.Drop != nil { for _, ask := range drop { if err := v.Drop(ask); err != nil { From 72ebb4deb6645522d661bc0d29d52471505f9635 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 11:38:22 +0100 Subject: [PATCH 04/14] runner: no holder is read from the seat-holders view, and an ask waits for its holder with no five-minute bound (R4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A step queued for a present holder was ended failed after five minutes; the five-minute ask bound goes, and a seat ends its steps could not run only when the seat mesh-module-assigner's view seat-holders, followed from its first view on, names no holder for it. Builds: design 47 G §535, §647 (as A-466y and A-466z amend them), §717 (the follower) --- .../cmd/mesh-delivery-pipeline/drops.go | 17 ++--- .../cmd/mesh-delivery-pipeline/drops_test.go | 2 +- .../cmd/mesh-delivery-pipeline/main.go | 18 ++--- .../cmd/mesh-delivery-pipeline/steps.go | 28 ++------ .../cmd/mesh-delivery-pipeline/verbs.go | 18 +---- .../cmd/mesh-delivery-pipeline/views.go | 68 +++++++++++++++++- .../internal/pipeline/bounds.go | 72 ++----------------- .../internal/pipeline/pins_test.go | 2 +- .../internal/pipeline/place.go | 14 +--- .../internal/pipeline/place_test.go | 52 ++++++++++++++ .../internal/pipeline/registrations.go | 34 +++++++++ .../internal/pipeline/run.go | 1 + .../internal/pipeline/start.go | 41 ++++++++++- .../internal/pipeline/steps_test.go | 45 +----------- .../internal/pipeline/supersede.go | 15 ++-- .../internal/pipeline/supersede_test.go | 19 ++--- .../internal/pipeline/testruns_test.go | 4 +- .../internal/pipeline/verbs.go | 19 +---- .../internal/pipeline/verbs_test.go | 10 +-- .../mesh-delivery-pipeline/manifest_test.go | 4 +- modules/mesh-delivery-pipeline/module.json | 3 +- 21 files changed, 255 insertions(+), 231 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops.go index dc1fde33..1915061c 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops.go @@ -36,7 +36,7 @@ func deleteBound(ctx context.Context, bounds jetstream.KeyValue) func(key string // armDropBound arms the five-minute bound of a drop, a key of the state bounds with its own time to live (A-77, A-222). func armDropBound(ctx context.Context, bounds jetstream.KeyValue) func(key string) error { return func(key string) error { - _, err := bounds.Create(ctx, key, []byte(`{"drop":"`+key+`"}`), jetstream.KeyTTL(pipeline.AskBound)) + _, err := bounds.Create(ctx, key, []byte(`{"drop":"`+key+`"}`), jetstream.KeyTTL(pipeline.DropBound)) if errors.Is(err, jetstream.ErrKeyExists) { return nil } @@ -67,26 +67,19 @@ func dropUnanswered(ctx context.Context, w callFindings, module string, now func return w.Write(ctx, condseat.Finding{Scope: mcc.ScopeCall, ID: id, Kind: mcc.KindCallUnanswered, Severity: condseat.FindingSeverityWarning, Summary: fmt.Sprintf("the call %s of ask %s is unanswered", call, ask), Headline: "A call whose answer is kept is not answered", - Explanation: fmt.Sprintf("The drop of the superseded step's ask %s was asked of the seat %s as a kept call and no holder answered it within %s, so the build it names may still run.", ask, buildSeat, pipeline.AskBound), + Explanation: fmt.Sprintf("The drop of the superseded step's ask %s was asked of the seat %s as a kept call and no holder answered it within %s, so the build it names may still run.", ask, buildSeat, pipeline.DropBound), Source: module + "." + mcc.KindCallUnanswered, Observer: module, At: now().UTC(), Subject: condseat.Subject{Scope: mcc.ScopeCall, ID: id}}) } } -// boundExpired takes the expiry of a key of the state bounds: a drop's says the drop unanswered, an ask's ends the step no -// holder took (§32, §535, A-77); another key is none of the runner's. -func boundExpired(h *pipeline.Hearer, unanswered func(buildSeat, ask string) error, logf func(string, ...any)) func(key string) { +// boundExpired takes the expiry of a key of the state bounds: a drop's says the drop unanswered (A-222); another key is none +// of the runner's, an ask having no bound of its own (§535, §647 as A-466y amends them). +func boundExpired(unanswered func(buildSeat, ask string) error, logf func(string, ...any)) func(key string) { return func(key string) { if s, ask, ok := pipeline.ParseDropBound(key); ok { if err := unanswered(s, ask); err != nil { logf("the drop of ask %s on %s, unanswered, is not said: %v", ask, s, err) } - return - } - if !pipeline.IsAskKey(key) { - return - } - if err := h.NotTaken(key, pipeline.AskBound); err != nil { - logf("the step of ask %s, which no holder took, is not ended: %v", key, err) } } } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops_test.go index 8b8321b0..d10bfce0 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops_test.go @@ -31,7 +31,7 @@ func (f *fakeFindings) Clear(_ context.Context, scope, id, kind, _ string) error func TestADropBoundsExpirySaysCallUnanswered(t *testing.T) { w := &fakeFindings{} var logged int - expired := boundExpired(nil, dropUnanswered(context.Background(), w, "the-runner", func() time.Time { return time.Unix(100, 0) }), func(string, ...any) { logged++ }) + expired := boundExpired(dropUnanswered(context.Background(), w, "the-runner", func() time.Time { return time.Unix(100, 0) }), func(string, ...any) { logged++ }) expired("not-a-bound") if len(w.written) != 0 { t.Fatalf("a key of no drop was said: %+v", w.written) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 17c35f90..1551038a 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -67,8 +67,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - // The bound of each ask a build seat's holder must take: a key with its own time to live, its expiry heard (§535, A-77). - bounds, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "bounds"), "bound an ask no holder takes") + // The bound of each superseded step's drop: a key with its own time to live, its expiry heard (A-222, A-77). + bounds, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "bounds"), "bound a drop no holder answers") if err != nil { return err } @@ -151,14 +151,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // for its step. starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf, Say: say, Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, js: js, cred: cred}, Name: cred.Module, - Arm: armBound(ctx, bounds), Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains), - Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has} + Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains), + Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders} placeHalf := func(ev catalog.CatalogRevisionEvent, half pipeline.Half) error { n, err := starter.PlaceRun(ev, half, rv.Catalog.View) logger.Printf("the candidate %s's revision %d placed %d pipelines of its run", ev.Group, ev.Revision, n) return err } resume := func(branch string) { + rv.Holders.refresh() // a half held may name the claimant of the seat-holders view (§647) half, ok := rv.Catalog.View(branch) if !ok { return @@ -182,6 +183,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro resumer.Store(&resume) // The view reaching a newer revision places the steps of the pipelines that waited for it, and asks nothing again (A-448, §200). advance := func(branch string) { + rv.Holders.refresh() if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 { logger.Printf("the pipelines of %s that waited for a newer revision of its half: %d placed (%v)", branch, n, err) } @@ -227,7 +229,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // A newer commit or publish supersedes the older running pipelines it replaces, their asks dropped as kept calls (§31, // §350; A-182, A-202). starter.Supersede = func(newer seat.Pipeline) { - n, err := hearer.Supersede(newer, pipeline.BoundDrop(dropAsk(calls, cred.Module), armDropBound(ctx, bounds), dropAnswered(ctx, deleteBound(ctx, bounds), rv.Findings)), dropBound(ctx, bounds)) + n, err := hearer.Supersede(newer, pipeline.BoundDrop(dropAsk(calls, cred.Module), armDropBound(ctx, bounds), dropAnswered(ctx, deleteBound(ctx, bounds), rv.Findings))) if err != nil { logger.Printf("the pipelines %s supersedes: %d ended, then %v", newer.Pipeline, n, err) } else if n > 0 { @@ -258,10 +260,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } return hearer.RunTaken(ask, node) } - if err := hearAskTaken(ctx, heard, bounds, taken, logger.Printf); err != nil { + if err := hearAskTaken(heard, taken, logger.Printf); err != nil { return err } - expired := boundExpired(hearer, dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf) + expired := boundExpired(dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf) if err := watchBounds(ctx, bounds, expired, logger.Printf); err != nil { return err } @@ -295,7 +297,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // mesh-wants), never on EVENTS: bound here, or no step of any pipeline is ever heard ending (G-1 §24). logger.Printf("the kept seat events heard: %d streams bound", hub.StartKept(ctx, js, cred)) // The seat's write verbs are answered here, by the process that holds the pipelines (A-107). - stopVerbs, err := answerVerbs(calls, seats, writeVerbs(ctx, hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls, bounds)) + stopVerbs, err := answerVerbs(calls, seats, writeVerbs(hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls)) if err != nil { return err } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go index 4e01276d..d19683fe 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go @@ -137,11 +137,10 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any)) return nil } -// hearAskTaken deletes the bound of each ask a build seat's holder took (§602, §647) and keeps on its step the node that took -// it (A-182): the ask is answered, so its five minutes end with no finding. -func hearAskTaken(ctx context.Context, hub handlers, bounds jetstream.KeyValue, taken func(ask, node string) error, logf func(string, ...any)) error { +// hearAskTaken keeps on its step the node whose build seat holder took an ask (§602, §647; A-182), so a drop of it is asked of +// that node's holder, and begins the clock of a test run's ask (§725). +func hearAskTaken(hub handlers, taken func(ask, node string) error, logf func(string, ...any)) error { for _, ts := range pipeline.TakenSeats { - ts := ts err := hub.Handle(ts.Subject, func(m jetstream.Msg) { var ev nodebuildgo.AskTaken if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" { @@ -149,12 +148,6 @@ func hearAskTaken(ctx context.Context, hub handlers, bounds jetstream.KeyValue, _ = m.Term() return } - if err := bounds.Delete(ctx, ev.Ask); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { - logf("the bound of ask %s, taken on %s, is not deleted, heard again: %v", ev.Ask, ev.Node, err) - _ = m.Nak() - return - } - // The step keeps the node that took its ask, so a drop of it is asked of that node's holder (A-182). if err := taken(ev.Ask, ev.Node); err != nil { logf("the node %s that took ask %s is not kept on its step, heard again: %v", ev.Node, ev.Ask, err) _ = m.Nak() @@ -169,19 +162,8 @@ func hearAskTaken(ctx context.Context, hub handlers, bounds jetstream.KeyValue, return nil } -// armBound is the bound of an ask: a key of the state bounds with its own time to live, named for the ask. -func armBound(ctx context.Context, bounds jetstream.KeyValue) pipeline.Arm { - return func(ask string) error { - _, err := bounds.Create(ctx, ask, []byte(`{"ask":"`+ask+`"}`), jetstream.KeyTTL(pipeline.AskBound)) - if errors.Is(err, jetstream.ErrKeyExists) { - return nil - } - return err - } -} - -// watchBounds hears the end of each ask's bound, the bus's marker of the key's expiry, and ends the step no holder took (§32, -// §535, A-77), and says a drop no holder answered unanswered (A-222): no timer in the process. +// watchBounds hears the end of each drop's bound, the bus's marker of the key's expiry, and says a drop no holder answered +// unanswered (A-222, A-77): no timer in the process. func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(key string), logf func(string, ...any)) error { w, err := bounds.WatchAll(ctx, jetstream.UpdatesOnly()) if err != nil { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go index 27105fb6..3224a5a9 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go @@ -1,13 +1,10 @@ package main import ( - "context" "encoding/json" "errors" "fmt" - "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" @@ -54,21 +51,10 @@ func writeHandlers(v pipeline.Verbs) map[string]process.SeatHandler { } } -// dropBound deletes the bound of an ask a cancelled pipeline no longer waits on. -func dropBound(ctx context.Context, bounds jetstream.KeyValue) func(string) error { - return func(ask string) error { - if err := bounds.Delete(ctx, ask); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { - return err - } - return nil - } -} - // writeVerbs are the verbs over the hearer and the starter, their hand acts recorded through the seat mesh-hand-acts. -func writeVerbs(ctx context.Context, h *pipeline.Hearer, s *pipeline.Starter, view func(string) (pipeline.Half, bool), - calls *process.Conn, bounds jetstream.KeyValue) pipeline.Verbs { +func writeVerbs(h *pipeline.Hearer, s *pipeline.Starter, view func(string) (pipeline.Half, bool), calls *process.Conn) pipeline.Verbs { candidate := func(group, branch string, revision int64) (pipeline.Half, error) { return readCandidate(calls.CallKept, catalog.CatalogRevisionEvent{Kind: "candidate", Group: group, Branch: branch, Revision: revision}) } - return pipeline.Verbs{Hearer: h, Starter: s, View: view, Candidate: candidate, Acts: meshhandactsclient.Recorder{Ask: calls.Call}, Drop: dropBound(ctx, bounds)} + return pipeline.Verbs{Hearer: h, Starter: s, View: view, Candidate: candidate, Acts: meshhandactsclient.Recorder{Ask: calls.Call}} } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go index dac58bb4..3a620f52 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "sync" "sync/atomic" "time" @@ -12,6 +13,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/seatlist" "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-pipeline/internal/pipeline" @@ -26,6 +28,70 @@ type runnerViews struct { // Findings is the runner's held finding writer: it raises the finding pipeline-failed (§33) and says the runner's own // findings (A-206). Findings *findingsWriter + // Holders follows the seat-holders view of the seat mesh-module-assigner's claimant (§535, §647; A-466z). + Holders *holderView +} + +// holderView follows the view seat-holders of the module the catalog halves name as the claimant of the seat +// mesh-module-assigner (seatlist.HolderSeat), as the runner's manifest consumes it, `seat:mesh-module-assigner.views.seat-holders` +// (design 47 G §535, §647 as A-466z amends them; §717): a claimant named anew drops the view followed and follows the new one's. +// Nothing is asked on a timer: it is read again whenever a branch's half is held or advances. +type holderView struct { + reader *views.ProcessReader + catalog *pipeline.Views + logf func(string, ...any) + mu sync.Mutex + owner string + pic *views.Picture +} + +// refresh follows the claimant's seat-holders view when the claimant is another than the one followed. +func (h *holderView) refresh() { + var halves []pipeline.Half + for _, b := range h.catalog.Branches() { + if half, ok := h.catalog.View(b); ok { + halves = append(halves, half) + } + } + claimant, ok := pipeline.ClaimantOf(halves, seatlist.HolderSeat) + h.mu.Lock() + defer h.mu.Unlock() + if !ok || claimant == h.owner { + return + } + if h.owner != "" { + if err := h.reader.Drop(h.owner, seatHoldersState); err != nil { + h.logf("the seat-holders view of %s is not dropped: %v", h.owner, err) + } + h.owner, h.pic = "", nil + } + pic, err := h.reader.Follow(claimant, seatHoldersState, claimant+"."+seatHoldersState) + if err != nil { + h.logf("the seat-holders view of %s is not followed: %v", claimant, err) + return + } + h.owner, h.pic = claimant, pic + h.logf("the seat-holders view of %s, the claimant of %s, is followed", claimant, seatlist.HolderSeat) +} + +// seatHoldersState is the seat mesh-module-assigner's view of who holds each seat (§616, §699). +const seatHoldersState = "seat-holders" + +// Holders are the modules the followed view names for seatName, and whether its first view is read: before it, and with +// no claimant followed, nothing is judged (A-466y). +func (h *holderView) Holders(seatName string) ([]string, bool) { + h.mu.Lock() + pic := h.pic + h.mu.Unlock() + if pic == nil { + return nil, false + } + select { + case <-pic.Follower().Ready(): + default: + return nil, false + } + return seatlist.HoldersIn(func() (map[string]json.RawMessage, error) { return pic.Follower().Values(), nil })(seatName), true } // startViews wires the runner's views on hub and asks each first view; register every other handler on hub before or @@ -88,7 +154,7 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cr return nil, err } reader.Start(ctx, js, cred, status) - return &runnerViews{Registrations: regs, Catalog: cv, Findings: findings}, nil + return &runnerViews{Registrations: regs, Catalog: cv, Findings: findings, Holders: &holderView{reader: reader, catalog: cv, logf: logf}}, nil } // catalogView is a branch's meshcatalogclient.Catalog as the runner reads it. diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/bounds.go b/modules/mesh-delivery-pipeline/internal/pipeline/bounds.go index bfb3063e..f3847b25 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/bounds.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/bounds.go @@ -1,20 +1,19 @@ package pipeline import ( - "errors" "fmt" "strings" "time" - seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" nodebuildimage "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage" nodebuildnode "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode" ) -// AskBound is how long an ask on a build seat's work queue may wait for its holder to take it (design 47 G §535, §647): five -// minutes, the bound of a key of the state `bounds` with its own time to live (A-77, A-104), never a timer. -const AskBound = 5 * time.Minute +// DropBound is how long a superseded step's drop may go unanswered before it is said (design 47 G A-222, A-77): five minutes, +// a key of the state `bounds` with its own time to live. An ask waiting for its holder has no such bound (§535, §647 as +// A-466y amends them): it is timed by §86's build-capacity finding and, for a test run, by run-untaken-bound (§725). +const DropBound = 5 * time.Minute // TakenSeat is a build seat whose holder says `ask-taken` when it takes an ask (design 47 G §602): where the runner hears it. type TakenSeat struct { @@ -23,15 +22,15 @@ type TakenSeat struct { Subject string } -// TakenSeats are the three build seats, from their contracts: the seat mesh-builds and the seat mesh-wants say no ask-taken, -// so no bound is armed for their asks. +// TakenSeats are the three build seats, from their contracts: the seat mesh-builds and the seat mesh-wants say no ask-taken. var TakenSeats = []TakenSeat{ {Seat: nodebuildgo.Seat, Subject: nodebuildgo.SubjectAskTakenAll}, {Seat: nodebuildnode.Seat, Subject: nodebuildnode.SubjectAskTakenAll}, {Seat: nodebuildimage.Seat, Subject: nodebuildimage.SubjectAskTakenAll}, } -// BoundSeat says whether the holder of a seat says ask-taken, so an ask put on its work queue has a bound. +// BoundSeat says whether the holder of a seat says ask-taken: a build seat, whose asks are dropped and whose test steps share +// a run (§724). func BoundSeat(name string) bool { for _, s := range TakenSeats { if s.Seat == name { @@ -41,63 +40,6 @@ func BoundSeat(name string) bool { return false } -// Arm puts a bound of one ask's wait for its holder: a key of the state `bounds` named for the ask, written with its own -// time to live. It is armed before the ask is put, so an ask-taken heard at once finds it. -type Arm func(ask string) error - -// NotTaken ends the step asked under ask that no holder took within its bound, the bound's expiry heard (design 47 G §32, -// §535, §647, A-77): the step ends `could not run` naming the seat, so its pipeline ends failed once the others end, never -// a silent wait. A step that ended or was never asked of a seat takes nothing. -func (h *Hearer) NotTaken(ask string, bound time.Duration) error { - pkeys, err := h.Store.Keys("*.*.*") - if err != nil { - return fmt.Errorf("the pipelines are not listed: %w", err) - } - for _, key := range pkeys { - p, _, err := h.Store.Get(key) - if err != nil || p.Outcome != seat.PipelineOutcomeRunning || stepOfAsk(p, ask) < 0 { - continue - } - return h.endNotTaken(key, ask, bound) - } - return nil -} - -func (h *Hearer) endNotTaken(key, ask string, bound time.Duration) error { - for { - p, rev, err := h.Store.Get(key) - if err != nil { - return fmt.Errorf("the pipeline %s is not read: %w", key, err) - } - i := stepOfAsk(p, ask) - if i < 0 || p.Steps[i].Outcome != seat.StepResultOutcomeRunning { - return nil - } - now := h.now() - s := &p.Steps[i] - s.Outcome, s.Ended = seat.StepResultOutcomeCouldNotRun, &now - s.Why = fmt.Sprintf("the seat %s has no holder that took ask %s within %s", s.Seat, ask, bound) - p.Unlogged = append(p.Unlogged, endEntry(key, *s, now)) - ended := endPipeline(&p, now) - if _, err := h.Store.Update(key, p, rev); errors.Is(err, ErrChanged) { - continue - } else if err != nil { - return fmt.Errorf("pipeline %s cannot be written: %w", key, err) - } - h.Failures.StepEnded(h.Store, p, *s) - if err := h.say(seat.EventStepEnded, p, s); err != nil { - return fmt.Errorf("pipeline %s's %s is written but not said: %w", key, seat.EventStepEnded, err) - } - h.appendUnlogged(key) - h.Ager.Age() - h.Fills.Check() - return h.said(p, false, ended) - } -} - -// IsAskKey says whether a key of the state `bounds` is an ask's bound. -func IsAskKey(key string) bool { return strings.HasPrefix(key, "step-") && !strings.Contains(key, ".") } - // DropBoundPrefix begins the key of a superseded step's drop in the state `bounds`, `drop..` (design 47 G // A-222, A-77): its five-minute expiry says the drop unanswered. const DropBoundPrefix = "drop." diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go index bee58513..a179ebd5 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go @@ -75,7 +75,7 @@ func TestAPipelinePinsItsCatalogRevisionWhileItRuns(t *testing.T) { } want(0, delivery.SubjectFreed, "pipeline."+pkey, 12) said = nil - if n, err := h.Supersede(seat.Pipeline{Pipeline: "main.6.app", Branch: "main", Source: 6, Module: "app", Commit: "c2", Fingerprint: "src2:bb", Outcome: seat.PipelineOutcomeRunning}, nil, nil); n != 1 || err != nil { + if n, err := h.Supersede(seat.Pipeline{Pipeline: "main.6.app", Branch: "main", Source: 6, Module: "app", Commit: "c2", Fingerprint: "src2:bb", Outcome: seat.PipelineOutcomeRunning}, nil); n != 1 || err != nil { t.Fatalf("supersede %d (%v)", n, err) } want(0, delivery.SubjectFreed, "pipeline."+pkey, 12) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index fd76d1bf..698e19b7 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -203,8 +203,9 @@ func stepAsk(p seat.Pipeline, e catalog.CatalogEntry, a catalog.Artifact, step, Module: p.Module, Pipeline: p.Pipeline, Revision: p.Source, Step: step}} } -// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. -func askAll(asker StepAsker, arm Arm, key string, asks []Asked, recovery bool) error { +// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. An ask waits for its +// holder with no bound of the runner's own (§535, §647 as A-466y amends them). +func askAll(asker StepAsker, key string, asks []Asked, recovery bool) error { var first error for _, a := range asks { accept, ok := acceptStep[a.Seat] @@ -212,15 +213,6 @@ func askAll(asker StepAsker, arm Arm, key string, asks []Asked, recovery bool) e accept = to } err := fmt.Errorf("the seat %s is no build seat the runner asks", a.Seat) - if ok { - // A build seat's holder says ask-taken: the ask's bound is armed before it is put (§535, §647). - if arm != nil && BoundSeat(a.Seat) { - if aerr := arm(a.ID); aerr != nil { - err = fmt.Errorf("its bound is not armed: %w", aerr) - ok = false - } - } - } if ok { err = asker.AskStep(accept, a.ID, a.Ask) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index 500075d4..d4347310 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -448,3 +448,55 @@ func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) { t.Errorf("a second revision asked %d test steps, want its own three", got) } } + +// A seat with no holder is read only from the seat-holders view (design 47 G §535, §647 as A-466y and A-466z amend them): +// once that view's first view is read, a build step whose seat it names no holder for ends could not run naming the seat +// and is asked of no one; before it is read, or with a holder named, the step is asked and waits with no bound of the +// runner's own; the claimant of the view is the one module the catalog halves name. +func TestASeatWithNoHolderIsReadFromTheSeatHoldersView(t *testing.T) { + reg, entries := placeFixture() + for _, c := range []struct { + name string + holders []string + read bool + asked bool + }{{"the view not read yet", nil, false, true}, {"a holder named", []string{"mesh-build-go"}, true, true}, {"no holder named", nil, true, false}} { + t.Run(c.name, func(t *testing.T) { + store, asks := memStore{}, &stepAsks{} + s := &Starter{Pipelines: memCreated{store}, Say: func(string, any) error { return nil }, Steps: asks, Name: "the-runner", + Registration: func(string) (catalog.Registration, bool) { return reg, true }, + Holders: func(string) ([]string, bool) { return c.holders, c.read }} + if ok, err := s.Start(Begin{Branch: "main", Source: 4, Module: "app", Commit: "c1", Fp: "src2:aa", Entries: entries}); !ok || err != nil { + t.Fatalf("start = %v, %v", ok, err) + } + p, _, _ := store.Get("main.4.app") + lint := stepNamed(p, "app:program", "lint") + asked := false + for _, id := range asks.ids { + asked = asked || id == lint.Ask + } + if c.asked && (lint.Outcome != seat.StepResultOutcomeRunning || !asked) { + t.Errorf("the lint step = %+v, asked %v; want it running and asked", lint, asked) + } + if !c.asked && (lint.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(lint.Why, "node-build-go has no holder") || len(asks.ids) != 2) { + t.Errorf("the lint step = %+v, %d asked; want could not run naming the seat, only the build and composition asked", lint, len(asks.ids)) + } + }) + } + claims := func(seats ...string) map[string]any { + var cs []any + for _, n := range seats { + cs = append(cs, map[string]any{"name": n, "scope": "mesh"}) + } + return map[string]any{"module": "x", "version": "0.1", "claims": cs} + } + half := Half{Entries: map[string]catalog.CatalogEntry{"assigner": {Module: "assigner", Manifest: claims("mesh-module-assigner", "mesh-wants")}, + "app": {Module: "app", Manifest: claims()}}} + if got, ok := ClaimantOf([]Half{half, {}}, "mesh-module-assigner"); !ok || got != "assigner" { + t.Errorf("the claimant = %q, %v", got, ok) + } + other := Half{Entries: map[string]catalog.CatalogEntry{"other": {Module: "other", Manifest: claims("mesh-module-assigner")}}} + if got, ok := ClaimantOf([]Half{half, other}, "mesh-module-assigner"); ok { + t.Errorf("halves that disagree name %q", got) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go b/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go index a2a2db7b..0016e988 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go @@ -2,12 +2,14 @@ package pipeline import ( "context" + "encoding/json" "fmt" "sort" "sync" catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-sdk/go/catalogue" ) // Status is a step's status for one artifact of a module, read from the module's registration and never from a @@ -212,3 +214,35 @@ func (r *Readiness) Observe(rev int64) (first, advanced bool) { } return false, false } + +// ClaimantOf is the module the catalog halves name as the claimant of seatName (design 47 G §614, A-82): the one module whose +// entry's manifest claims it, the same in every half that names one; none, more than one, or halves that disagree name none. +func ClaimantOf(halves []Half, seatName string) (string, bool) { + found := "" + for _, h := range halves { + var named []string + for _, e := range h.Entries { + raw, err := json.Marshal(e.Manifest) + if err != nil { + continue + } + m, err := catalogue.Decode(raw) + if err != nil { + continue + } + for _, c := range m.Claims { + if c.Name == seatName { + named = append(named, e.Module) + } + } + } + switch { + case len(named) == 0: + continue + case len(named) > 1, found != "" && found != named[0]: + return "", false + } + found = named[0] + } + return found, found != "" +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/run.go b/modules/mesh-delivery-pipeline/internal/pipeline/run.go index 61d35175..becc1be8 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/run.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/run.go @@ -232,6 +232,7 @@ func (s *Starter) placeRunPipeline(key, id string, revision int64, half Half) (b if s.Registration != nil { if reg, ok := s.Registration(p.Module); ok { p.Steps, asks = Place(p, reg, half.Entries, now, s.Name) + asks = s.holderless(p.Steps, asks, now) } else { s.logf("%s has no registration yet: its pipeline is placed with no step", key) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index e8ee14d2..449ce01b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -73,8 +73,9 @@ type Starter struct { // Toolchain is the toolchain of a language as the runner heard it agreed (§416, A-104): its language and version; nil when // none was heard, which asks the step with none. Toolchain func(language string) (*nodebuildgo.ToolchainUsed, error) - // Arm arms the bound of an ask put on a build seat's work queue (§535, §647); nil arms none. - Arm Arm + // Holders are the modules the seat mesh-module-assigner's view seat-holders names for a seat, and whether that view's + // first view is read (§535, §647 as A-466y, A-466z amend them); nil, or a view not read yet, judges no seat holderless. + Holders func(seat string) ([]string, bool) // Fills refuses a start while a store of the runner is full (§366); nil refuses none. Fills *Fills // Recovered says whether a commit is named a recovery (§81, §86): its steps are asked on the build seats' stream delivery. @@ -94,6 +95,38 @@ type Starter struct { EndReaders func(run string, readers []seat.TestRunReader, r seat.TestRun) error } +// holderless ends, as it is placed, each build step whose seat the seat mesh-module-assigner's view seat-holders names no holder +// for, `could not run` naming the seat, and asks it of no one (design 47 G §535, §647 as A-466y and A-466z amend them): judged +// only once that view's first view is read, never from the picture before it. It answers the asks still to put. +func (s *Starter) holderless(steps []seat.StepResult, asks []Asked, now time.Time) []Asked { + if s.Holders == nil { + return asks + } + none := map[string]bool{} + for i := range steps { + st := &steps[i] + if st.Outcome != seat.StepResultOutcomeRunning || !BoundSeat(st.Seat) { + continue + } + if holders, read := s.Holders(st.Seat); !read || len(holders) > 0 { + continue + } + st.Outcome, st.Ended, st.Ask, st.Shared = seat.StepResultOutcomeCouldNotRun, &now, "", nil + st.Why = fmt.Sprintf("the seat %s has no holder: the seat-holders view names none", st.Seat) + none[st.Seat] = true + } + if len(none) == 0 { + return asks + } + out := asks[:0:0] + for _, a := range asks { + if !none[a.Seat] { + out = append(out, a) + } + } + return out +} + // askSteps puts a placed pipeline's asks (§22, §24): a test run another step asked is read, not asked again (share), then // the kept results and toolchain are carried and each ask is put. func (s *Starter) askSteps(key, commit string, asks []Asked) error { @@ -102,7 +135,7 @@ func (s *Starter) askSteps(key, commit string, asks []Asked) error { return nil } s.carry(key, asks) - return askAll(s.Steps, s.Arm, key, asks, s.recovered(key, commit)) + return askAll(s.Steps, key, asks, s.recovered(key, commit)) } func newPipeline(b Begin, key string, now time.Time) seat.Pipeline { @@ -140,6 +173,7 @@ func (s *Starter) Start(b Begin) (bool, error) { } else if s.Registration != nil { if reg, ok := s.Registration(b.Module); ok { p.Steps, asks = Place(p, reg, b.Entries, now, s.Name) + asks = s.holderless(p.Steps, asks, now) } else { s.logf("%s has no registration yet: its pipeline starts with no step and waits on the registration of %s, which places them (A-327)", key, b.Module) } @@ -311,6 +345,7 @@ func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipe } var asks []Asked p.Steps, asks = Place(p, r, half.Entries, now, s.Name) + asks = s.holderless(p.Steps, asks, now) if _, err := store.Update(key, p, rev); err != nil { s.logf("%s is written meanwhile, so nothing is placed in it now: %v", key, err) continue diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go index edd4e860..114411d9 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go @@ -302,49 +302,6 @@ func TestAnOptionalStepStillWaitingHoldsNoEnd(t *testing.T) { } } -// An ask no holder took within its bound ends its step `could not run` naming the seat, and with the rest ended the pipeline -// failed (design 47 G §32, §535, §647, A-104); a step that ended, or ask nobody placed, ends nothing. -func TestAnAskNoHolderTookEndsItsStepNamingTheSeat(t *testing.T) { - h, store, said, _, _, _ := fixture() - e := store[pkey] - e.p.Steps[1].Outcome = seat.StepResultOutcomePassed // the unit step ended; only lint is waiting for a holder - e.p.Steps[2].Outcome = seat.StepResultOutcomeNotAsked - store[pkey] = e - if err := h.NotTaken("nobody-asked", AskBound); err != nil || store[pkey].p.Steps[0].Outcome != seat.StepResultOutcomeRunning { - t.Fatalf("an ask no step names ended something: %v", err) - } - if err := h.NotTaken("a1", AskBound); err != nil { - t.Fatal(err) - } - p := store[pkey].p - if p.Steps[0].Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(p.Steps[0].Why, nodebuildgo.Seat) || - p.Outcome != seat.PipelineOutcomeFailed || !strings.Contains(strings.Join(*said, " "), seat.EventPipelineEnded) { - t.Errorf("step %+v, pipeline %s (%s), said %v: want could not run naming %s and a failed pipeline", p.Steps[0], p.Outcome, p.Why, *said, nodebuildgo.Seat) - } - // Its end taken once: the bound's expiry heard again changes nothing. - n := len(*said) - if err := h.NotTaken("a1", AskBound); err != nil || len(*said) != n { - t.Errorf("a second expiry said %d more events: %v", len(*said)-n, err) - } -} - -// The bound is armed before the ask is put, only for the build seats whose holder says ask-taken, and an ask whose bound is -// not armed is not put (§535, §647). -func TestTheBoundIsArmedBeforeAnAskIsPut(t *testing.T) { - var order []string - asker := askerFunc(func(accept, id string, _ any) error { order = append(order, "put "+id); return nil }) - arm := Arm(func(ask string) error { order = append(order, "arm "+ask); return nil }) - asks := []Asked{{Seat: nodebuildgo.Seat, ID: "step-1", Ask: 1}, {Seat: wants.Seat, ID: "step-2", Ask: 2}} - if err := askAll(asker, arm, "k", asks, false); err != nil || strings.Join(order, ",") != "arm step-1,put step-1,put step-2" { - t.Errorf("order %v, %v; want the build seat's bound armed first and mesh-wants' ask with none", order, err) - } - order = nil - failing := Arm(func(string) error { return errors.New("bucket gone") }) - if err := askAll(asker, failing, "k", asks[:1], false); err == nil || len(order) != 0 { - t.Errorf("an ask whose bound cannot be armed was put or said nothing: %v %v", order, err) - } -} - type askerFunc func(accept, id string, body any) error func (f askerFunc) AskStep(accept, id string, body any) error { return f(accept, id, body) } @@ -359,7 +316,7 @@ func TestARecoverysStepsAreAskedOnTheStreamDelivery(t *testing.T) { }{{true, "mesh.seat.node-build-go.accept.delivery.mesh-pipeline-runner," + wants.AcceptSubjectStep}, {false, "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner," + wants.AcceptSubjectStep}} { var to []string asker := askerFunc(func(accept, _ string, _ any) error { to = append(to, accept); return nil }) - if err := askAll(asker, nil, "k", asks, c.recovery); err != nil || strings.Join(to, ",") != c.want { + if err := askAll(asker, "k", asks, c.recovery); err != nil || strings.Join(to, ",") != c.want { t.Errorf("recovery %v: asked on %v (%v), want %s", c.recovery, to, err, c.want) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go index 0a307702..f008149d 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go @@ -49,10 +49,10 @@ func (h *Hearer) Taken(ask, node string) error { // whose fingerprint newer's commit changed, and for a publish's pipeline each an older publish of the package started // (design 47 G §31, §119; ADR 0666 §9; A-202): the pipeline and each step not ended // `superseded`, naming the newer commit, and it says pipeline-ended. A module the newer commit left as it was finishes its -// pipeline. The ask of each superseded step on a build seat that serves `drop` is dropped (drop; its bound deleted through -// unbound); a step on mesh-builds or mesh-wants, whose seats serve no drop, is only ended superseded (A-182). A run on +// pipeline. The ask of each superseded step on a build seat that serves `drop` is dropped (drop); a step of a test run leaves +// it, its ask dropped only with its last reader (§726); a step on mesh-builds or mesh-wants, whose seats serve no drop, is only ended superseded (A-182). A run on // request's pipeline neither supersedes nor is superseded. It says how many pipelines it ended. -func (h *Hearer) Supersede(newer seat.Pipeline, drop Drop, unbound func(ask string) error) (int, error) { +func (h *Hearer) Supersede(newer seat.Pipeline, drop Drop) (int, error) { if newer.Run != "" || newer.Source <= 0 { return 0, nil } @@ -67,7 +67,7 @@ func (h *Hearer) Supersede(newer seat.Pipeline, drop Drop, unbound func(ask stri if key == newer.Pipeline { continue } - done, err := h.supersedeOne(key, newer, why, drop, unbound) + done, err := h.supersedeOne(key, newer, why, drop) if err != nil && first == nil { first = err } @@ -99,7 +99,7 @@ func supersededWhy(newer seat.Pipeline) string { return fmt.Sprintf("the newer commit %s of %s changed %s's fingerprint", newer.Commit, newer.Branch, newer.Module) } -func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop Drop, unbound func(string) error) (bool, error) { +func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop Drop) (bool, error) { for { p, rev, err := h.Store.Get(key) if err != nil || !superseded(p, newer) { @@ -146,11 +146,6 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop h.logf("the drop of ask %s of the superseded %s is not answered: %v", s.Ask, key, err) } } - if unbound != nil { - if err := unbound(s.Ask); err != nil { - h.logf("the bound of ask %s of the superseded %s is not dropped: %v", s.Ask, key, err) - } - } } if err := h.said(p, false, true); err != nil && first == nil { first = err diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/supersede_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/supersede_test.go index 8490be86..95a06e7c 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/supersede_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/supersede_test.go @@ -29,7 +29,6 @@ func TestSupersede(t *testing.T) { } type dropped struct{ seat, node, ask string } var drops []dropped - var unbound []string drop := func(s, node, ask, why string) error { if !strings.Contains(why, "c2") { t.Errorf("the drop of %s names no newer commit: %q", ask, why) @@ -37,7 +36,6 @@ func TestSupersede(t *testing.T) { drops = append(drops, dropped{s, node, ask}) return nil } - unbind := func(ask string) error { unbound = append(unbound, ask); return nil } newer := func(key string, source int64, fp string, run string, pub *seat.Published) seat.Pipeline { return seat.Pipeline{Pipeline: key, Branch: "main", Source: source, Module: "app", Commit: "c2", Fingerprint: fp, Run: run, Published: pub, Outcome: seat.PipelineOutcomeRunning} @@ -50,12 +48,12 @@ func TestSupersede(t *testing.T) { {"a run on request", newer("main.r1.app", 6, "src2:bb", "r1", nil)}, {"an older source revision", newer("main.4.app", 4, "src2:bb", "", nil)}, } { - if n, err := h.Supersede(c.newer, drop, unbind); n != 0 || err != nil { + if n, err := h.Supersede(c.newer, drop); n != 0 || err != nil { t.Errorf("%s superseded %d (%v)", c.name, n, err) } } *said = nil - if n, err := h.Supersede(newer("main.6.app", 6, "src2:bb", "", nil), drop, unbind); n != 1 || err != nil { + if n, err := h.Supersede(newer("main.6.app", 6, "src2:bb", "", nil), drop); n != 1 || err != nil { t.Fatalf("a changed fingerprint superseded %d (%v)", n, err) } p, _, _ := store.Get(pkey) @@ -68,13 +66,13 @@ func TestSupersede(t *testing.T) { } } want := []dropped{{nodebuildgo.Seat, "n7", "a1"}, {nodebuildgo.Seat, "", "a2"}} - if !slices.Equal(drops, want) || !slices.Equal(unbound, []string{"a1", "a2"}) { - t.Errorf("dropped %+v and unbound %v, want %+v (the mesh-builds step only ended)", drops, unbound, want) + if !slices.Equal(drops, want) { + t.Errorf("dropped %+v, want %+v (the mesh-builds step only ended)", drops, want) } if n := len(*said); n != 5 || (*said)[4] != seat.EventPipelineEnded+".app.main.c1" { t.Errorf("said %v, want four step-ended then pipeline-ended", *said) } - if n, _ := h.Supersede(newer("main.7.app", 7, "src2:cc", "", nil), drop, unbind); n != 0 { + if n, _ := h.Supersede(newer("main.7.app", 7, "src2:cc", "", nil), drop); n != 0 { t.Errorf("a superseded pipeline was superseded again: %d", n) } @@ -82,10 +80,10 @@ func TestSupersede(t *testing.T) { pub := memStore{"main.5.lib": {p: seat.Pipeline{Pipeline: "main.5.lib", Branch: "main", Source: 5, Module: "lib", Fingerprint: "src2:aa", Outcome: seat.PipelineOutcomeRunning, Published: &seat.Published{Artifact: "core:pkg", Version: "v1"}}, rev: 1}} h.Store = pub - if n, _ := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v1"}), drop, unbind); n != 0 { + if n, _ := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v1"}), drop); n != 0 { t.Errorf("a publish of the same version superseded %d", n) } - if n, err := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v2"}), drop, unbind); n != 1 || err != nil { + if n, err := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v2"}), drop); n != 1 || err != nil { t.Fatalf("a newer publish superseded %d (%v)", n, err) } if p, _, _ := pub.Get("main.5.lib"); p.Outcome != seat.PipelineOutcomeSuperseded || !strings.Contains(p.Why, "v2") { @@ -105,9 +103,6 @@ func TestBoundDrop(t *testing.T) { if _, _, ok := ParseDropBound("step-ab"); ok { t.Error("an ask's bound was read as a drop's") } - if IsAskKey(DropBoundKey("s", "step-ab")) { - t.Error("a drop's bound was taken for an ask's") - } err := BoundDrop(func(string, string, string, string) error { return nil }, arm, done)(nodebuildgo.Seat, "n1", "step-ab", "why") if err != nil || len(armed) != 1 || armed[0] != "drop."+nodebuildgo.Seat+".step-ab" || len(answered) != 1 { t.Errorf("an answered drop: %v, armed %v, answered %v", err, armed, answered) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go index ab79a96b..7d60ec64 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -262,13 +262,13 @@ func TestATestRunsEndMovesEveryReaderAndIsCountedOnce(t *testing.T) { } // a, superseded, leaves the lint run to b's readers; b, superseded after, leaves none: the ask is dropped and the run ends. lint := stepNamed(a, "app:program", "lint") - if _, err := f.h.Supersede(seat.Pipeline{Pipeline: "main.5.app", Branch: "main", Source: 5, Module: "app", Commit: "c2", Fingerprint: "src2:bb"}, nil, nil); err != nil { + if _, err := f.h.Supersede(seat.Pipeline{Pipeline: "main.5.app", Branch: "main", Source: 5, Module: "app", Commit: "c2", Fingerprint: "src2:bb"}, nil); err != nil { t.Fatal(err) } if r, _, _ := f.runs.Get(lint.Shared[0].Run); r.Ended != nil || len(r.Readers) != 2 || len(f.drops) != 0 { t.Fatalf("after a left the lint run = %+v, drops %v; want it running for b's two readers", r, f.drops) } - if _, err := f.h.Supersede(seat.Pipeline{Pipeline: "main.7.app", Branch: "main", Source: 7, Module: "app", Commit: "c3", Fingerprint: "src2:cc"}, nil, nil); err != nil { + if _, err := f.h.Supersede(seat.Pipeline{Pipeline: "main.7.app", Branch: "main", Source: 7, Module: "app", Commit: "c3", Fingerprint: "src2:cc"}, nil); err != nil { t.Fatal(err) } if r, _, _ := f.runs.Get(lint.Shared[0].Run); r.Outcome != seat.StepResultOutcomeCouldNotRun || r.Said != NoReaderLeft || len(f.drops) != 2 || f.drops[0] != "node-build-go "+lint.Ask { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go index 91cbdf6e..0c4ff543 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -27,8 +27,6 @@ type Verbs struct { Candidate func(group, branch string, revision int64) (Half, error) // Acts records a hand act before the verb does it (rerun, accept, cancel); a nil Acts refuses the verb, nothing recorded. Acts views.HandActs - // Drop drops the bound of an ask a cancelled pipeline no longer waits on. - Drop func(ask string) error } func (v Verbs) now() time.Time { return v.Hearer.now() } @@ -185,8 +183,8 @@ func (v Verbs) Accept(caller string, in seat.AcceptInput) (seat.AcceptAnswer, er return seat.AcceptAnswer{Result: p.Steps[i]}, nil } -// Cancel ends a running pipeline `cancelled`, each step not ended cancelled and the bound of each ask dropped, recorded in -// the hand-act log first (§394); it says pipeline-ended. +// Cancel ends a running pipeline `cancelled`, each step not ended cancelled and each step of a test run leaving it, recorded +// in the hand-act log first (§394, §726); it says pipeline-ended. func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, error) { h := v.Hearer recorded := false @@ -205,7 +203,6 @@ func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, er recorded = true } now := h.now() - var drop []string var leave []seat.StepResult for i := range p.Steps { s := &p.Steps[i] @@ -215,9 +212,6 @@ func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, er if sharedRun(*s) != "" && s.Seat != wants.Seat { leave = append(leave, *s) // it leaves its test run, dropped only with the last reader (§726) } - if s.Ask != "" { - drop = append(drop, s.Ask) - } s.Outcome, s.By, s.Why, s.Ended = seat.StepResultOutcomeCancelled, caller, in.Why, &now } p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeCancelled, in.Why, &now @@ -230,13 +224,6 @@ func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, er for _, s := range leave { h.leaveRun(in.Pipeline, s, "the pipeline "+in.Pipeline+" is cancelled: "+in.Why) } - if v.Drop != nil { - for _, ask := range drop { - if err := v.Drop(ask); err != nil { - h.logf("the bound of ask %s of the cancelled %s is not dropped: %v", ask, in.Pipeline, err) - } - } - } return seat.CancelAnswer{Pipeline: p}, h.said(p, false, true) } } @@ -296,7 +283,7 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error v.Hearer.pin(p) // a pipeline that runs again keeps its revision again (A-449) if ask != nil { v.Starter.carry(in.Pipeline, []Asked{*ask}) - if err := askAll(v.Starter.Steps, v.Starter.Arm, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil { + if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil { return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go index 6b6ed3df..bb355a07 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go @@ -35,8 +35,9 @@ func verbsFixture() (Verbs, memStore, *[]string, *stepAsks, *actLog, *[]string) asks, acts, dropped := &stepAsks{}, &actLog{}, &[]string{} s := &Starter{Steps: asks, Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }, Now: h.Now} - return Verbs{Hearer: h, Starter: s, View: func(string) (Half, bool) { return Half{Entries: entries}, true }, Acts: acts, - Drop: func(ask string) error { *dropped = append(*dropped, ask); return nil }}, store, said, asks, acts, dropped + h.DropRun = func(_, _, ask, _ string) error { *dropped = append(*dropped, ask); return nil } + return Verbs{Hearer: h, Starter: s, View: func(string) (Half, bool) { return Half{Entries: entries}, true }, Acts: acts}, + store, said, asks, acts, dropped } func stepNamed(p seat.Pipeline, artifact, step string) seat.StepResult { @@ -81,7 +82,8 @@ func TestReviewEndsThePlacedReviewStepWithItsProvenCaller(t *testing.T) { // accept takes only a failed optional step, writes its hand act first, and refuses without one; rerun asks the step again // under a new ask saying it is a rerun and its caller, reopens a failed pipeline and asks nothing of a refused one; cancel ends -// a running pipeline cancelled, each step cancelled and each ask's bound dropped (design 47 G §22, §30, §325, §394). +// a running pipeline cancelled, each step cancelled, no ask's bound left to drop (design 47 G §22, §30, §325, §394; §647 as +// A-466y amends it). func TestAcceptRerunAndCancelWriteByCompareAndSetAndRecordTheirHandAct(t *testing.T) { v, store, said, asks, acts, dropped := verbsFixture() end := func(step, outcome string) { @@ -134,7 +136,7 @@ func TestAcceptRerunAndCancelWriteByCompareAndSetAndRecordTheirHandAct(t *testin } // A cancel ends the pipeline cancelled and drops the bounds of the asks still held. c, err := v.Cancel("operator", seat.CancelInput{Pipeline: pkey, Why: "wrong branch"}) - if err != nil || c.Pipeline.Outcome != "cancelled" || c.Pipeline.EndCount != 1 || len(*dropped) == 0 { + if err != nil || c.Pipeline.Outcome != "cancelled" || c.Pipeline.EndCount != 1 || len(*dropped) != 0 { t.Fatalf("cancel = %+v, %v, dropped %v", c.Pipeline, err, *dropped) } if s := stepNamed(c.Pipeline, "app:program", "lint"); s.Outcome != "cancelled" || s.By != "operator" { diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index cc545315..99523c6d 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -85,7 +85,9 @@ func TestManifest(t *testing.T) { "node-build-go.step-progress", "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked", "mesh-delivery.recovery-merged", // A-186: the seat list its write verbs' callers are checked against, heard as it changes. - "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed"})}, + "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", + // §647 as A-466z amends it: no holder is read from the seat-holders view. + "seat:mesh-module-assigner.views.seat-holders"})}, // RN2 (§21, §195, §200, §77): the registrations and the catalog half asked by their verbs, the first views read and // asked again by hand (a hand act). {"invokes", equalSets(m.Invokes, []string{"seat:mesh-catalog.registrations", diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index eec137ad..46ed603b 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -50,7 +50,8 @@ "mesh-conditions.findings-asked", "mesh-delivery.recovery-merged", "mesh-catalog.seat-definition-changed", - "mesh-catalog.seat-definition-removed" + "mesh-catalog.seat-definition-removed", + "seat:mesh-module-assigner.views.seat-holders" ], "invokes": [ "seat:mesh-catalog.seat-definitions", From 2b3a9e4a6abc43eef013ccb8fd2b05fc9f0273fc Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 11:42:09 +0100 Subject: [PATCH 05/14] runner: asking a test run again moves every reader that did not pass (R5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A rerun of one artifact's failed test step left the other artifact's step failed on a result no longer true; the rerun now joins, reads or starts the key's run, carrying the former readers marked in the key, and the new run's end moves the marked ones, said again as a rerun by its caller and naming the new ask, the pipeline reopened; an accepted step does not move. Builds: design 47 G §727 (test part); A-469l, A-470d, A-470e --- .../cmd/mesh-delivery-pipeline/testruns.go | 14 +- .../internal/pipeline/place_test.go | 16 +- .../internal/pipeline/runends.go | 36 +++-- .../internal/pipeline/steps.go | 51 +++++- .../internal/pipeline/testruns.go | 85 ++++++++-- .../internal/pipeline/testruns_test.go | 64 ++++++++ .../internal/pipeline/verbs.go | 149 +++++++++++++++++- 7 files changed, 371 insertions(+), 44 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go index ca1091ea..e26358bb 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go @@ -34,7 +34,7 @@ type testRunsKV struct { kv jetstream.KeyValue } -func (t testRunsKV) Create(key string, r seat.TestRun) error { +func (t testRunsKV) Create(key string, r pipeline.Run) error { raw, err := json.Marshal(r) if err != nil { return err @@ -47,22 +47,22 @@ func (t testRunsKV) Create(key string, r seat.TestRun) error { return nil } -func (t testRunsKV) Get(key string) (seat.TestRun, uint64, error) { +func (t testRunsKV) Get(key string) (pipeline.Run, uint64, error) { e, err := t.kv.Get(t.ctx, key) if errors.Is(err, jetstream.ErrKeyNotFound) { - return seat.TestRun{}, 0, pipeline.ErrNoRun + return pipeline.Run{}, 0, pipeline.ErrNoRun } if err != nil { - return seat.TestRun{}, 0, err + return pipeline.Run{}, 0, err } - var r seat.TestRun + var r pipeline.Run if err := json.Unmarshal(e.Value(), &r); err != nil { - return seat.TestRun{}, 0, fmt.Errorf("the test run %s is no test run: %w", key, err) + return pipeline.Run{}, 0, fmt.Errorf("the test run %s is no test run: %w", key, err) } return r, e.Revision(), nil } -func (t testRunsKV) Update(key string, r seat.TestRun, revision uint64) (uint64, error) { +func (t testRunsKV) Update(key string, r pipeline.Run, revision uint64) (uint64, error) { raw, err := json.Marshal(r) if err != nil { return 0, err diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index d4347310..f516c158 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -331,38 +331,38 @@ func TestAnArtifactWithoutALanguageIsPlacedByItsKind(t *testing.T) { // testRunsMem is the state test-runs in memory, a write by compare-and-set on the revision read. type testRunsMem map[string]struct { - r seat.TestRun + r Run rev uint64 } -func (m testRunsMem) Create(key string, r seat.TestRun) error { +func (m testRunsMem) Create(key string, r Run) error { if _, ok := m[key]; ok { return ErrExists } m[key] = struct { - r seat.TestRun + r Run rev uint64 }{r, 1} return nil } -func (m testRunsMem) Get(key string) (seat.TestRun, uint64, error) { +func (m testRunsMem) Get(key string) (Run, uint64, error) { e, ok := m[key] if !ok { - return seat.TestRun{}, 0, ErrNoRun + return Run{}, 0, ErrNoRun } - e.r.Readers = append([]seat.TestRunReader(nil), e.r.Readers...) + e.r.Readers = append([]RunReader(nil), e.r.Readers...) return e.r, e.rev, nil } func (m testRunsMem) Delete(key string) error { delete(m, key); return nil } -func (m testRunsMem) Update(key string, r seat.TestRun, rev uint64) (uint64, error) { +func (m testRunsMem) Update(key string, r Run, rev uint64) (uint64, error) { if m[key].rev != rev { return 0, ErrChanged } m[key] = struct { - r seat.TestRun + r Run rev uint64 }{r, rev + 1} return rev + 1, nil diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go index ade028a9..8d98ba7e 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go @@ -231,10 +231,10 @@ func (h *Hearer) runSettings() RunSettings { } // EndRun ends the test run under run with end's outcome, words, counts, node and output, written once by compare-and-set, -// and ends each of its readers with it but skip, the step whose own end ended it (§725, §726). A run ended already is -// not ended again. +// and ends each of its readers with it but skip, the step whose own end ended it (§725, §726). A run ended already is not +// ended again. func (h *Hearer) EndRun(run string, end seat.TestRun, skip *seat.TestRunReader) error { - var r seat.TestRun + var r Run for { got, rev, err := h.TestRuns.Get(run) if errors.Is(err, ErrNoRun) { @@ -271,7 +271,7 @@ func (h *Hearer) EndRun(run string, end seat.TestRun, skip *seat.TestRunReader) } var first error for _, reader := range r.Readers { - if skip != nil && reader == *skip { + if skip != nil && reader.Of() == *skip { continue } if err := h.endReader(reader, run, r); err != nil && first == nil { @@ -282,17 +282,24 @@ func (h *Hearer) EndRun(run string, end seat.TestRun, skip *seat.TestRunReader) } // endReader ends reader's step of run with the run's result (§726): its outcome, words, counts and node, the step naming -// the run in Shared as it did. A step ended already takes nothing. -func (h *Hearer) endReader(reader seat.TestRunReader, run string, r seat.TestRun) error { +// the run in Shared as it did. A step ended already takes nothing, but one the rerun that asked the run marked (A-470d, +// A-470e): it read another ask of the key, and it moves with this one, said again marked a rerun by that caller, naming +// the new ask, its pipeline reopened (§727). +func (h *Hearer) endReader(reader RunReader, run string, r Run) error { for { p, rev, err := h.Store.Get(reader.Pipeline) if err != nil { return nil // a pipeline gone has no reader left to end } - i := -1 + i, moved := -1, false for j, s := range p.Steps { - if s.Artifact == reader.Artifact && sharedRun(s) == run && s.Outcome == seat.StepResultOutcomeRunning { - i = j + if s.Artifact != reader.Artifact || sharedRun(s) != run { + continue + } + if s.Outcome == seat.StepResultOutcomeRunning { + i, moved = j, false + } else if reader.Rerun && s.Shared[0].Ask != r.Ask && s.Accepted == nil { + i, moved = j, true } } if i < 0 { @@ -300,7 +307,14 @@ func (h *Hearer) endReader(reader seat.TestRunReader, run string, r seat.TestRun } now := h.now() s := &p.Steps[i] - s.Outcome, s.Why, s.Node, s.Ended = r.Outcome, r.Said, r.Node, &now + if moved { + s.Rerun, s.By = true, reader.By + s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: s.Shared[0].AskedBy}} + if p.Outcome != seat.PipelineOutcomeRunning { + p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeRunning, "", nil + } + } + s.Outcome, s.Why, s.Node, s.Ended, s.Tests = r.Outcome, r.Said, r.Node, &now, nil if r.Tests != nil { t := *r.Tests s.Tests = &t @@ -350,7 +364,7 @@ func (h *Hearer) leaveRun(key string, s seat.StepResult, why string) { } left := r.Readers[:0:0] for _, o := range r.Readers { - if o != reader { + if o.Of() != reader { left = append(left, o) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 88bbacc7..4f32dbb1 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -154,7 +154,7 @@ type Hearer struct { func (h *Hearer) EndReaders(run string, readers []seat.TestRunReader, r seat.TestRun) error { var first error for _, reader := range readers { - if err := h.endReader(reader, run, r); err != nil && first == nil { + if err := h.endReader(RunReader{Pipeline: reader.Pipeline, Artifact: reader.Artifact}, run, Run{TestRun: r}); err != nil && first == nil { first = err } } @@ -236,6 +236,8 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh } unlogged := endEntry(ev.Pipeline, *s, now, keptKeys...) p.Unlogged = append(p.Unlogged, unlogged) + // The steps of this pipeline reading the run end in the same write, so the pipeline ends once (§726, §727). + readers := h.sameRunReaders(&p, i, ev, now) from := now if ev.Ended != nil { from = ev.Ended.UTC() @@ -268,6 +270,11 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh if err := h.say(seat.EventStepEnded, p, s); err != nil { return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", ev.Pipeline, seat.EventStepEnded, err) } + for _, j := range readers { + if err := h.say(seat.EventStepEnded, p, &p.Steps[j]); err != nil { + return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", ev.Pipeline, seat.EventStepEnded, err) + } + } // The step's end is written and said before its log entry is appended, so a log that refuses ends no step (§486, §606). h.appendUnlogged(ev.Pipeline) h.Ager.Age() @@ -279,6 +286,48 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh } } +// sameRunReaders ends, in p, every other step reading the test run step i asked with ev, its ask's end: a running reader, +// and one the rerun that asked the run marked, which moves (A-470d, A-470e), said again marked a rerun by that caller. It +// answers their indexes. +func (h *Hearer) sameRunReaders(p *seat.Pipeline, i int, ev StepEnded, now time.Time) []int { + asker := p.Steps[i] + run := sharedRun(asker) + if run == "" || h.TestRuns == nil || asker.Ask != ev.ID { + return nil + } + marks := map[string]RunReader{} + if r, _, err := h.TestRuns.Get(run); err == nil { + for _, o := range r.Readers { + if o.Pipeline == p.Pipeline { + marks[o.Artifact] = o + } + } + } + var out []int + for j := range p.Steps { + s := &p.Steps[j] + if j == i || sharedRun(*s) != run { + continue + } + switch m := marks[s.Artifact]; { + case s.Outcome == seat.StepResultOutcomeRunning && s.Ask == "": + case m.Rerun && s.Shared[0].Ask != ev.ID && s.Accepted == nil: + s.Rerun, s.By = true, m.By + s.Shared = []seat.Shared{{Run: run, Ask: ev.ID, AskedBy: &seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: asker.Artifact}}} + default: + continue + } + s.Outcome, s.Why, s.Node, s.Ended, s.Tests = asker.Outcome, asker.Why, asker.Node, &now, nil + if asker.Tests != nil { + t := *asker.Tests + s.Tests = &t + } + p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *s, now)) + out = append(out, j) + } + return out +} + // endRunOf ends the test run step s of p asked, with the end ev of its ask (§725, §726): the run's output kept in // test-runs-objects under the run's key, its outcome, words, counts and node written, and every other reader ended with it. // A step that asked no run ends nothing more. diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go index f382d652..be004716 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go @@ -18,15 +18,46 @@ import ( // packages, written create-only by the step that asks the run and changed by compare-and-set. type TestRuns interface { // Create writes key when it is absent; ErrExists when it is written already. - Create(key string, r seat.TestRun) error + Create(key string, r Run) error // Get reads key and its revision; ErrNoRun when it is absent. - Get(key string) (seat.TestRun, uint64, error) + Get(key string) (Run, uint64, error) // Update writes key at the revision read; ErrChanged when another write came between. - Update(key string, r seat.TestRun, revision uint64) (uint64, error) + Update(key string, r Run, revision uint64) (uint64, error) // Delete deletes key; a key gone is no fault. Delete(key string) error } +// Run is a key's value in test-runs (§725): the test run, its readers each marked whether a rerun's new run moves it and by +// whom (A-470d), so the marks are in the key and a restart takes them up. +type Run struct { + seat.TestRun + Readers []RunReader `json:"readers,omitempty"` +} + +// RunReader is a step reading a test run, its pipeline key and artifact (§725), marked rerun with the caller of the rerun +// whose new run moves it (A-470d). +type RunReader struct { + Pipeline string `json:"pipeline"` + Artifact string `json:"artifact"` + Rerun bool `json:"rerun,omitempty"` + By string `json:"by,omitempty"` +} + +// Of is the reader as the seat names it. +func (r RunReader) Of() seat.TestRunReader { + return seat.TestRunReader{Pipeline: r.Pipeline, Artifact: r.Artifact} +} + +// Answer is the run as the seat's read answers it, its readers named by pipeline and artifact. +func (r Run) Answer() seat.TestRun { + t := r.TestRun + t.Readers = nil + for _, o := range r.Readers { + t.Readers = append(t.Readers, o.Of()) + } + return t +} + // ErrNoRun is a Get of a test-runs key that is not written. var ErrNoRun = errors.New("no test run under the key") @@ -79,14 +110,14 @@ func readsRun(s seat.StepResult, run string) bool { } // addReaders adds each reader not in r's readers yet, in order. -func addReaders(r *seat.TestRun, readers []seat.TestRunReader) { +func addReaders(r *Run, readers []seat.TestRunReader) { for _, n := range readers { have := false for _, o := range r.Readers { - have = have || o == n + have = have || o.Of() == n } if !have { - r.Readers = append(r.Readers, n) + r.Readers = append(r.Readers, RunReader{Pipeline: n.Pipeline, Artifact: n.Artifact}) } } } @@ -122,7 +153,7 @@ func (s *Starter) share(key string, asks []Asked) []Asked { return asks } var out []Asked - joined := map[string]seat.TestRun{} + joined := map[string]Run{} for _, a := range asks { if a.Run == "" { out = append(out, a) @@ -162,7 +193,7 @@ func (s *Starter) share(key string, asks []Asked) []Asked { readers = a.Readers } } - if err := s.EndReaders(run, readers, r); err != nil { + if err := s.EndReaders(run, readers, r.TestRun); err != nil { s.logf("the steps of %s reading the ended test run %s are not ended: %v", key, run, err) } } @@ -219,9 +250,10 @@ func (s *Starter) started(a Asked, write func() error) error { } // claimRun writes a's test run create-only; ok says the ask is put (the key is a's own), else run is the run it reads. -func (s *Starter) claimRun(a Asked) (seat.TestRun, bool, error) { +func (s *Starter) claimRun(a Asked) (Run, bool, error) { now := s.now() - value := seat.TestRun{Level: a.Level, Digest: a.Digest, Seat: a.Seat, Ask: a.ID, Started: &now, Readers: a.Readers} + value := Run{TestRun: seat.TestRun{Level: a.Level, Digest: a.Digest, Seat: a.Seat, Ask: a.ID, Started: &now}} + addReaders(&value, a.Readers) for { err := s.started(a, func() error { return s.TestRuns.Create(a.Run, value) }) if err == nil { @@ -229,17 +261,17 @@ func (s *Starter) claimRun(a Asked) (seat.TestRun, bool, error) { } var full *fullErr if errors.As(err, &full) { - return seat.TestRun{}, false, err + return Run{}, false, err } if !errors.Is(err, ErrExists) { - return seat.TestRun{}, false, err + return Run{}, false, err } r, rev, err := s.TestRuns.Get(a.Run) if errors.Is(err, ErrNoRun) { continue // deleted meanwhile: written again } if err != nil { - return seat.TestRun{}, false, err + return Run{}, false, err } if r.Ask == a.ID { return r, true, nil @@ -252,7 +284,7 @@ func (s *Starter) claimRun(a Asked) (seat.TestRun, bool, error) { if errors.Is(err, ErrChanged) { continue } else if err != nil { - return seat.TestRun{}, false, err + return Run{}, false, err } return value, true, nil } @@ -260,7 +292,7 @@ func (s *Starter) claimRun(a Asked) (seat.TestRun, bool, error) { if _, err := s.TestRuns.Update(a.Run, r, rev); errors.Is(err, ErrChanged) { continue } else if err != nil { - return seat.TestRun{}, false, err + return Run{}, false, err } return r, false, nil } @@ -268,7 +300,7 @@ func (s *Starter) claimRun(a Asked) (seat.TestRun, bool, error) { // readOthers writes pipeline key's steps that read another step's run: no ask of their own, the run's ask and its asker // in Shared (A-469m, A-469q). -func (s *Starter) readOthers(key string, joined map[string]seat.TestRun) { +func (s *Starter) readOthers(key string, joined map[string]Run) { if s.Store == nil { return } @@ -303,3 +335,24 @@ func (s *Starter) readOthers(key string, joined map[string]seat.TestRun) { return } } + +// joinRun adds reader to run's readers by compare-and-set. +func (s *Starter) joinRun(run string, reader seat.TestRunReader) { + if s.TestRuns == nil { + return + } + for { + r, rev, err := s.TestRuns.Get(run) + if err != nil { + s.logf("the test run %s is not read, so %s does not join it: %v", run, reader.Pipeline, err) + return + } + addReaders(&r, []seat.TestRunReader{reader}) + if _, err := s.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + s.logf("%s does not join the test run %s: %v", reader.Pipeline, run, err) + } + return + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go index 7d60ec64..e80d6291 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -292,3 +292,67 @@ func TestATestRunsEndMovesEveryReaderAndIsCountedOnce(t *testing.T) { t.Errorf("the composition step's shared = %+v, want mesh-wants' as it came", s.Shared) } } + +// Asking a test run again moves every reader (design 47 G §727; A-469l, A-470d, A-470e): a rerun of one artifact's failed +// step starts a new run under the same key, carrying the former readers, the one that did not pass marked; the new run's +// passing end moves the other artifact's step to passed, marked a rerun by the caller and naming the new ask, and the +// pipeline says pipeline-ended again; a step a person accepted does not move. +func TestARerunOfATestRunMovesEveryReaderThatDidNotPass(t *testing.T) { + for _, accepted := range []bool{false, true} { + f := newRunFixture() + reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Policy: catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "required", + Integration: "bypass", Composition: "bypass", Review: "bypass", Approval: "bypass"}}} + f.s.Registration = func(string) (catalog.Registration, bool) { return reg, true } + p := f.start(t, 4, 9) + e := f.store[p.Pipeline] + e.p.Steps[0].Outcome = seat.StepResultOutcomePassed // the build + f.store[p.Pipeline] = e + unit := stepNamed(p, "app:program", "unit") + end := func(ask string, outcome string, rerun bool) { + t.Helper() + ended := time.Now() + if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: ask, + Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: outcome, + Rerun: rerun, By: map[bool]string{true: "op@shanks"}[rerun], Ended: &ended}); err != nil { + t.Fatal(err) + } + } + end(unit.Ask, "failed", false) + if q := f.pipeline(p); q.Outcome != seat.PipelineOutcomeFailed || q.EndCount != 1 || stepNamed(q, "app:tools", "unit").Outcome != "failed" { + t.Fatalf("after the failing run the pipeline = %s (%d), tools %s", q.Outcome, q.EndCount, stepNamed(q, "app:tools", "unit").Outcome) + } + if accepted { + e := f.store[p.Pipeline] + i := stepIndex(e.p, "unit", "app:program") + e.p.Steps[i].Outcome, e.p.Steps[i].Accepted = seat.StepResultOutcomeAccepted, &seat.Accepted{By: "op@shanks", Why: "known"} + f.store[p.Pipeline] = e + } + v := Verbs{Hearer: f.h, Starter: f.s, View: func(string) (Half, bool) { return Half{Entries: map[string]catalog.CatalogEntry{ + "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, + "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}, true }, Acts: &actLog{}} + a, err := v.Rerun("op@shanks", seat.RerunInput{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:tools", Why: "flaky"}) + if err != nil { + t.Fatal(err) + } + again := a.Result.Ask + r, _, _ := f.runs.Get(unit.Shared[0].Run) + if again == "" || again == unit.Ask || r.Ask != again || r.Ended != nil || len(r.Readers) != 2 || r.Readers[0].Rerun == accepted || (!accepted && r.Readers[0].By != "op@shanks") { + t.Fatalf("the rerun's run = %+v (ask %s), want a new run under the same key carrying the program, marked unless accepted", r, again) + } + end(again, "passed", true) + q := f.pipeline(p) + prog := stepNamed(q, "app:program", "unit") + if accepted { + if prog.Outcome != seat.StepResultOutcomeAccepted { + t.Errorf("an accepted step moved to %s", prog.Outcome) + } + continue + } + if prog.Outcome != seat.StepResultOutcomePassed || !prog.Rerun || prog.By != "op@shanks" || prog.Shared[0].Ask != again { + t.Errorf("the program's step = %+v, want passed, a rerun by the caller, naming the new ask", prog) + } + if q.Outcome != seat.PipelineOutcomePassed || q.EndCount != 2 { + t.Errorf("the pipeline = %s (%d), want passed at its second end", q.Outcome, q.EndCount) + } + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go index 0c4ff543..554bf8f3 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -236,6 +236,11 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error if in.Step == seat.StepResultStepApproval { return seat.RerunAnswer{}, errors.New("an approval is asked again by ask-again, not rerun") } + if p, _, err := v.Hearer.Store.Get(in.Pipeline); err == nil { + if i, err := stepOf(p, in.Step, in.Artifact); err == nil && v.Hearer.TestRuns != nil && sharedRun(p.Steps[i]) != "" && BoundSeat(p.Steps[i].Seat) { + return v.rerunTest(caller, in, p.Steps[i]) + } + } var ask *Asked p, _, err := v.change(in.Pipeline, seat.EventStepStarted, func() error { return v.record(seat.VerbRerun, fmt.Sprintf("asked the %s step of %s again", in.Step, in.Pipeline), in.Why) @@ -295,6 +300,148 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error return seat.RerunAnswer{Pipeline: in.Pipeline, Result: &r}, nil } +// rerunTest asks a test step of a test run again (design 47 G §727, A-469l): it joins its key's run while that runs, reads one +// that started after the step's last end, or else starts a new run under a new ask, replacing the ended key by compare-and-set +// and carrying its readers into the new value, so the new run's end moves every former reader that did not pass. +func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult) (seat.RerunAnswer, error) { + h := v.Hearer + run := sharedRun(st) + if st.Outcome == seat.StepResultOutcomeRunning { + return seat.RerunAnswer{}, fmt.Errorf("the %s step of %s is running: it is not asked again", in.Step, in.Pipeline) + } + r, rev, err := h.TestRuns.Get(run) + if err != nil && !errors.Is(err, ErrNoRun) { + return seat.RerunAnswer{}, fmt.Errorf("the test run %s is not read: %w", run, err) + } + exists := err == nil + reader := seat.TestRunReader{Pipeline: in.Pipeline, Artifact: st.Artifact} + read := exists && r.Ended != nil && r.Outcome != seat.StepResultOutcomeCouldNotRun && st.Ended != nil && r.Started != nil && r.Started.After(*st.Ended) + join := exists && r.Ended == nil + var ask *Asked + if !join && !read { + p, _, err := h.Store.Get(in.Pipeline) + if err != nil { + return seat.RerunAnswer{}, err + } + if ask, err = v.askOf(p, st, caller, h.now()); err != nil { + return seat.RerunAnswer{}, err + } + } + var asker *seat.SharedAskedBy + if join || read { + if asker, err = askerOf(h.Store, r.Ask); err != nil { + return seat.RerunAnswer{}, err + } + if asker != nil && *asker == (seat.SharedAskedBy{Pipeline: reader.Pipeline, Artifact: reader.Artifact}) { + asker = nil + } + } + event := seat.EventStepStarted + if read { + event = seat.EventStepEnded + } + p, _, err := v.change(in.Pipeline, event, func() error { + return v.record(seat.VerbRerun, fmt.Sprintf("asked the %s step of %s again", in.Step, in.Pipeline), in.Why) + }, func(p *seat.Pipeline, now time.Time) (int, error) { + switch p.Outcome { + case seat.PipelineOutcomeCancelled, seat.PipelineOutcomeSuperseded: + return -1, fmt.Errorf("the pipeline %s is %s: a step of it is not asked again", p.Pipeline, p.Outcome) + } + i, err := stepOf(*p, in.Step, in.Artifact) + if err != nil { + return -1, err + } + s := &p.Steps[i] + *s = seat.StepResult{Step: s.Step, Artifact: s.Artifact, Status: s.Status, Fingerprint: s.Fingerprint, Seat: s.Seat, + Outcome: seat.StepResultOutcomeRunning, By: caller, Rerun: true, Started: &now} + switch { + case ask != nil: + s.Ask, s.Shared = ask.ID, []seat.Shared{{Run: run, Ask: ask.ID}} + case read: + s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: asker}} + s.Outcome, s.Why, s.Node, s.Ended = r.Outcome, r.Said, r.Node, &now + if r.Tests != nil { + t := *r.Tests + s.Tests = &t + } + default: + s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: asker}} + } + if p.Outcome != seat.PipelineOutcomeRunning { + p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeRunning, "", nil + } + for j := range p.Steps { + if p.Steps[j].Step == seat.StepResultStepApproval && p.Steps[j].Outcome == seat.StepResultOutcomeNotAsked { + p.Steps[j].Outcome, p.Steps[j].Why = seat.StepResultOutcomeWaiting, "" + } + } + return i, nil + }) + if err != nil { + return seat.RerunAnswer{}, err + } + h.pin(p) + switch { + case join || read: + v.Starter.joinRun(run, reader) + default: + now := h.now() + value := Run{TestRun: seat.TestRun{Level: st.Step, Digest: r.Digest, Seat: st.Seat, Ask: ask.ID, Started: &now}} + // Every former reader is carried, each that did not pass marked to move with the new run by this caller (A-470d). + for _, o := range r.Readers { + if o.Of() == reader { + continue + } + o.Rerun, o.By = !v.passedReader(o, run), "" + if o.Rerun { + o.By = caller + } + value.Readers = append(value.Readers, o) + } + addReaders(&value, []seat.TestRunReader{reader}) + a := *ask + a.Run = run + if err := v.Starter.started(a, func() error { + if !exists { + return h.TestRuns.Create(run, value) + } + _, err := h.TestRuns.Update(run, value, rev) + return err + }); err != nil { + return seat.RerunAnswer{}, fmt.Errorf("the step is written running but its test run %s is not: %w", run, err) + } + v.Starter.carry(in.Pipeline, []Asked{*ask}) + if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil { + return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err) + } + } + i, err := stepOf(p, in.Step, in.Artifact) + if err != nil { + return seat.RerunAnswer{}, err + } + res := p.Steps[i] + return seat.RerunAnswer{Pipeline: in.Pipeline, Result: &res}, nil +} + +// passedReader says whether reader's step of run passed, or a person accepted it (§30): a step that moves with no new run. +func (v Verbs) passedReader(reader RunReader, run string) bool { + p, _, err := v.Hearer.Store.Get(reader.Pipeline) + if err != nil { + return true // a pipeline gone has no step to move + } + for _, s := range p.Steps { + if s.Artifact == reader.Artifact && sharedRun(s) == run { + switch s.Outcome { + case seat.StepResultOutcomePassed, seat.StepResultOutcomeCarried, seat.StepResultOutcomeAccepted, + seat.StepResultOutcomeSuperseded, seat.StepResultOutcomeCancelled: + return true + } + return s.Accepted != nil + } + } + return true +} + // askOf is the ask to put on the serving seat for step s again: the one Place makes for it, under a new id (an ask put twice // under one id is kept once), saying it is a rerun and by whom. func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time.Time) (*Asked, error) { @@ -319,7 +466,7 @@ func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time } else { return nil, fmt.Errorf("the catalog half of %s is not held yet: the step is asked when it is", p.Branch) } - _, placed := Place(p, reg, half.Entries, now, v.Starter.Name) + _, placed := place(p, reg, half.Entries, now, v.Starter.Name, false) base := AskID(startOf(p, now), p.Pipeline, s.Artifact, s.Step) for _, a := range placed { if a.ID != base { From ad40520e89a3add702e710cae1019e658a9c250c Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 11:43:52 +0100 Subject: [PATCH 06/14] runner: stands names a waiting step's ask and its run, and every step asked says step-started (R6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The watchman could not time a step reading a run: stands named no ask, and step-started was said only on rerun and ask-again. A waiting step now stands with its own ask and, for a test step, the run's key and ask; each step asked at a start, a resume, a placing or a run says step-started, naming its run in shared with no asker. Builds: design 47 G §731, §729, §726 (step-started of a reading step); A-468e, A-469a, A-469q, A-469r, A-469v --- .../internal/pipeline/read.go | 18 ++++++- .../internal/pipeline/read_test.go | 53 +++++++++++++++++++ .../internal/pipeline/run_test.go | 8 ++- .../internal/pipeline/start.go | 42 +++++++++++++-- .../internal/pipeline/steps.go | 25 ++++++++- .../internal/pipeline/testruns_test.go | 8 +-- 6 files changed, 143 insertions(+), 11 deletions(-) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read.go b/modules/mesh-delivery-pipeline/internal/pipeline/read.go index b244afbb..63d31d5a 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read.go @@ -100,7 +100,7 @@ func (r Reads) Stands() (seat.StandsAnswer, error) { } out = append(out, seat.Wait{Pipeline: p.Pipeline, Module: p.Module, Branch: p.Branch, Commit: p.Commit, Group: p.Group, Run: p.Run, Source: p.Source, Revision: p.Revision, Fingerprint: p.Fingerprint, Step: s.Step, - Artifact: s.Artifact, On: s.On, Since: *s.Started, V: 1}) + Artifact: s.Artifact, On: s.On, Since: *s.Started, V: 1, Ask: askOfWait(s), Shared: sharedOfWait(s)}) } } sort.SliceStable(out, func(i, j int) bool { return out[i].Since.Before(out[j].Since) }) @@ -219,3 +219,19 @@ func startedOf(p seat.Pipeline) time.Time { } return *p.Started } + +// askOfWait is a waiting step's own ask (design 47 G §731, §496): a step reading another's test run keeps none (A-469m). +func askOfWait(s seat.StepResult) string { return s.Ask } + +// sharedOfWait is the run a waiting step waits on, where the runner knows it (§731; A-469r, A-469v): a test step's run key +// and ask, as one entry with no machine and no asker. +func sharedOfWait(s seat.StepResult) []seat.Shared { + if len(s.Shared) == 0 { + return nil + } + out := make([]seat.Shared, 0, len(s.Shared)) + for _, x := range s.Shared { + out = append(out, seat.Shared{Run: x.Run, Ask: x.Ask, Machine: x.Machine}) + } + return out +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go index 20c544ab..8dad23b1 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" ) @@ -150,3 +151,55 @@ func TestShowAndKeptCarryTheStoresFill(t *testing.T) { t.Errorf("show with the fill unread: %v", err) } } + +// A step that reads a test run says its own step-started at every place it is asked, naming the run's key and ask in shared +// and no asker (design 47 G §726, §729; A-469a, A-469q), and stands names each waiting step's own ask and, for a test step, +// the run it waits on (§731; A-469r, A-469v). +func TestStandsAndStepStartedNameWhatAReadingStepWaitsOn(t *testing.T) { + f := newRunFixture() + p := f.start(t, 4, 9) + asker, reader := stepNamed(p, "app:program", "unit"), stepNamed(p, "app:tools", "unit") + started := map[string]seat.StepResult{} + for _, e := range f.said { + started[e.Result.Artifact+" "+e.Result.Step] = e.Result + } + for _, s := range []seat.StepResult{asker, reader} { + got, ok := started[s.Artifact+" unit"] + if !ok || len(got.Shared) != 1 || got.Shared[0] != (seat.Shared{Run: asker.Shared[0].Run, Ask: asker.Ask}) { + t.Errorf("the step-started of %s = %+v (said %v), want the run's key and ask, no asker", s.Artifact, got, ok) + } + } + if _, ok := started[" composition"]; !ok { + t.Error("the composition step's step-started is not said") + } + pipes := memKept{} + put(t, pipes, p.Pipeline, f.pipeline(p)) + a, err := Reads{Pipelines: pipes, Kept: memKept{}}.Stands() + if err != nil { + t.Fatal(err) + } + waits := map[string]seat.Wait{} + for _, w := range a.Stands { + waits[w.Artifact+" "+w.Step] = w + } + want := []seat.Shared{{Run: asker.Shared[0].Run, Ask: asker.Ask}} + if w := waits["app:program unit"]; w.Ask != asker.Ask || len(w.Shared) != 1 || w.Shared[0] != want[0] { + t.Errorf("the asking step stands as %+v, want its ask and its run", w) + } + if w := waits["app:tools unit"]; w.Ask != "" || len(w.Shared) != 1 || w.Shared[0] != want[0] { + t.Errorf("the reading step stands as %+v, want no ask of its own and the run's key and ask", w) + } + if w := waits[" composition"]; w.Ask == "" || len(w.Shared) != 0 { + t.Errorf("the composition step stands as %+v, want its own ask and no run", w) + } + // Resume asks the steps again and says their step-started again (A-469a). + n := len(f.said) + if _, err := f.s.Resume(f.store, "main", Half{Entries: map[string]catalog.CatalogEntry{ + "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, + "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}); err != nil { + t.Fatal(err) + } + if len(f.said) == n { + t.Error("resume said no step-started") + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/run_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/run_test.go index 6dbc0dc1..d2670bc5 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/run_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/run_test.go @@ -79,7 +79,13 @@ func TestRunOnRequest(t *testing.T) { if _, _, err := store.Get(RunKey("fix/1", a.Run, "same")); err == nil { t.Error("an entry the members do not change got a pipeline") } - if len(said) != 2 || !strings.HasPrefix(said[0], seat.EventPipelineStarted+".") || len(asks.ids) == 0 { + // Each pipeline placed says pipeline-started, then step-started for each step it asks (A-469a). + started, steps := 0, 0 + for _, x := range said { + started += strings.Count(x, seat.EventPipelineStarted+".") + steps += strings.Count(x, seat.EventStepStarted+".") + } + if started != 2 || !strings.HasPrefix(said[0], seat.EventPipelineStarted+".") || len(asks.ids) == 0 || steps < len(asks.ids) { t.Errorf("placing said %v and asked %d steps", said, len(asks.ids)) } if ids, _ := s.UnplacedRuns(); len(ids) != 0 { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 449ce01b..e3916834 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -130,12 +130,46 @@ func (s *Starter) holderless(steps []seat.StepResult, asks []Asked, now time.Tim // askSteps puts a placed pipeline's asks (§22, §24): a test run another step asked is read, not asked again (share), then // the kept results and toolchain are carried and each ask is put. func (s *Starter) askSteps(key, commit string, asks []Asked) error { + placed := asks asks = s.share(key, asks) - if len(asks) == 0 { - return nil + var err error + if len(asks) > 0 { + s.carry(key, asks) + err = askAll(s.Steps, key, asks, s.recovered(key, commit)) + } + s.sayStarted(key, placed) + return err +} + +// sayStarted says step-started for each step of pipeline key that asks were placed for (design 47 G §726, §729; A-469a): +// the step asked, and each step reading the test run it asks, naming the run's key and ask in shared, so the watchman times +// it from that ask (§496, §732). A composition step's, said before any run is chosen, names none (§729). +func (s *Starter) sayStarted(key string, asks []Asked) { + if s.Store == nil || s.Say == nil || len(asks) == 0 { + return + } + ids, runs := map[string]bool{}, map[string]bool{} + for _, a := range asks { + ids[a.ID] = true + if a.Run != "" { + runs[a.Run] = true + } + } + p, _, err := s.Store.Get(key) + if err != nil { + s.logf("the pipeline %s is not read, so its steps' step-started is not said: %v", key, err) + return + } + at := s.now() + for _, st := range p.Steps { + if st.Outcome != seat.StepResultOutcomeRunning || !(ids[st.Ask] || (st.Ask == "" && runs[sharedRun(st)])) { + continue + } + subject, ev := stepEvent(seat.EventStepStarted, p, st, at) + if err := s.Say(subject, ev); err != nil { + s.logf("step-started of %s's %s %s is not said: %v", key, st.Artifact, st.Step, err) + } } - s.carry(key, asks) - return askAll(s.Steps, key, asks, s.recovered(key, commit)) } func newPipeline(b Begin, key string, now time.Time) seat.Pipeline { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 4f32dbb1..c31f6639 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -432,8 +432,8 @@ func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error at := h.now() switch event { case seat.EventStepEnded, seat.EventStepStarted: - return h.Say(subject, seat.StepEvent{V: 1, At: &at, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit, - Fingerprint: p.Fingerprint, Group: p.Group, Revision: p.Revision, Source: p.Source, Result: *step}) + _, ev := stepEvent(event, p, *step, at) + return h.Say(subject, ev) case seat.EventPipelineEnded: e := pipelineEnded(p) if h.Widens != nil { @@ -446,6 +446,27 @@ func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error } } +// stepEvent is a step's step-started or step-ended of p, on the runner's subject for it: its result as it stands, with +// its shared when it reads a run (§729; A-468e, A-469q). +func stepEvent(event string, p seat.Pipeline, step seat.StepResult, at time.Time) (string, seat.StepEvent) { + last := p.Commit + if p.Group != "" { + last = p.Group + } + if event == seat.EventStepStarted { + // A step-started names the run and its ask, never the step that asked it (A-469q). + var shared []seat.Shared + for _, x := range step.Shared { + x.AskedBy = nil + shared = append(shared, x) + } + step.Shared = shared + } + subject := "mesh.seat." + seat.Seat + ".event." + event + "." + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Branch) + "." + keys.EscapeName(last) + return subject, seat.StepEvent{V: 1, At: &at, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit, + Fingerprint: p.Fingerprint, Group: p.Group, Revision: p.Revision, Source: p.Source, Result: step} +} + // The test budget per catalog module (design 47 G §24, §419; the test budget): at most 500 unit and 50 integration tests. const ( budgetUnit = 500 diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go index e80d6291..41185109 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -327,9 +327,11 @@ func TestARerunOfATestRunMovesEveryReaderThatDidNotPass(t *testing.T) { e.p.Steps[i].Outcome, e.p.Steps[i].Accepted = seat.StepResultOutcomeAccepted, &seat.Accepted{By: "op@shanks", Why: "known"} f.store[p.Pipeline] = e } - v := Verbs{Hearer: f.h, Starter: f.s, View: func(string) (Half, bool) { return Half{Entries: map[string]catalog.CatalogEntry{ - "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, - "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}, true }, Acts: &actLog{}} + v := Verbs{Hearer: f.h, Starter: f.s, View: func(string) (Half, bool) { + return Half{Entries: map[string]catalog.CatalogEntry{ + "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, + "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}, true + }, Acts: &actLog{}} a, err := v.Rerun("op@shanks", seat.RerunInput{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:tools", Why: "flaky"}) if err != nil { t.Fatal(err) From 7923ab63b1bdd1c0df016e2e5e81cb1566c0e3e9 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 11:47:05 +0100 Subject: [PATCH 07/14] runner: the test runs are read and dropped by the seat's verbs test-runs and test-runs-drop (R7) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A test run had no tool to read or recover it; test-runs answers each run with its packages from its output object, or says the output gone, with the state's fill, and test-runs-drop, answered by the process, records the hand act first, ends the readers naming the drop and deletes the key and its object; runner_test_runs and runner_test_runs_drop call them. Builds: design 47 G §728, §725 (the output object), §733 (unit); A-466v, A-469f, A-469g, A-469u, A-469v --- .../cmd/mesh-delivery-pipeline-tools/main.go | 17 +- .../mesh-delivery-pipeline-tools/testruns.go | 71 ++++++++ .../cmd/mesh-delivery-pipeline/verbs.go | 2 + .../cmd/mesh-delivery-pipeline/verbs_test.go | 2 +- .../internal/pipeline/steps.go | 13 +- .../internal/pipeline/testruns_read.go | 160 ++++++++++++++++++ .../internal/pipeline/testruns_test.go | 48 +++++- .../mesh-delivery-pipeline/manifest_test.go | 3 +- modules/mesh-delivery-pipeline/module.json | 8 +- 9 files changed, 314 insertions(+), 10 deletions(-) create mode 100644 modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/testruns.go create mode 100644 modules/mesh-delivery-pipeline/internal/pipeline/testruns_read.go diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go index eb5f47fb..9523c3ce 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go @@ -46,8 +46,8 @@ func as[T any](args map[string]any) (T, error) { return in, nil } -// Tools are the runner's read verbs over its states pipelines and kept. -func Tools(r pipeline.Reads) []stdio.Tool { +// Tools are the runner's read verbs over its states pipelines, kept and test-runs. +func Tools(r pipeline.Reads, testRuns ...pipeline.RunsRead) []stdio.Tool { runs := map[string]func(map[string]any) (any, error){ seat.VerbShow: func(a map[string]any) (any, error) { in, err := as[seat.ShowInput](a) @@ -79,6 +79,15 @@ func Tools(r pipeline.Reads) []stdio.Tool { return r.KeptResults(in) }, } + for _, t := range testRuns { + runs[seat.VerbTestRuns] = func(a map[string]any) (any, error) { + in, err := as[seat.TestRunsInput](a) + if err != nil { + return nil, err + } + return t.TestRuns(in) + } + } var out []stdio.Tool for _, v := range seat.Verbs() { if run, ok := runs[v]; ok { @@ -148,10 +157,12 @@ func main() { func(n string) (buscalls.Fill, error) { return busstate.ObjectsOf(n).Fill() }, func(n string) (*stdio.LogInfo, error) { return stdio.Log(n).Info() }, time.Now) r := pipeline.Reads{Pipelines: kept{stdio.State(seat.StatePipelines)}, Kept: kept{stdio.State(seat.StateKept)}, Fill: fill} - tools := append(Tools(r), FirstViewTools(os.Getenv(stdio.ServedAs), stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...) + runs := RunsRead(stdio.State(testRunsState), stdio.State(testRunsState).Info, busstate.ObjectsOf(testRunsObjects)) + tools := append(Tools(r, runs), FirstViewTools(os.Getenv(stdio.ServedAs), stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...) tools = append(tools, UnloggedTools(stdio.State(seat.StatePipelines), stdio.Log(seat.StateStepLog), meshhandactsclient.Recorder{Ask: stdio.Ask}, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) })...) tools = append(tools, CancelTool(r.List, stdio.Ask), RerunTool(stdio.Ask)) + tools = append(tools, TestRunsTools(stdio.Ask)...) tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now)) if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] %v\n", err) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/testruns.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/testruns.go new file mode 100644 index 00000000..de196cab --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/testruns.go @@ -0,0 +1,71 @@ +package main + +import ( + "encoding/json" + "errors" + "strings" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/busstate" + "git.novox.be/novox/mesh-sdk/go/catalogue" + + "mesh-delivery-pipeline/internal/pipeline" +) + +// testRunsState is the runner's state of its test runs (§725), and testRunsObjects its own object store (A-469e). +const testRunsState = "test-runs" + +var testRunsObjects = catalogue.StateObjectsName(testRunsState) + +// RunsRead is the verb test-runs' reader over the runner's state test-runs and its object store, through the bundle's calls. +func RunsRead(state stdio.KeptState, info func() (*stdio.StateInfo, error), objects busstate.Objects) pipeline.RunsRead { + return pipeline.RunsRead{Runs: kept{state}, + Object: func(name string) ([]byte, error) { + o, err := objects.Get(name) + if err != nil && strings.Contains(err.Error(), "not found") { + return nil, pipeline.ErrNoRun + } + return o.Data, err + }, + Fill: func() (seat.Fill, error) { + i, err := info() + if err != nil { + return seat.Fill{}, err + } + return seat.Fill{Store: testRunsState, Bytes: int64(i.Bytes), Cap: i.MaxBytes}, nil + }} +} + +// TestRunsTools are runner_test_runs and runner_test_runs_drop (design 47 G §728; A-469f, A-469u): each calls its seat verb +// over the bus, test-runs answered by this bundle and test-runs-drop by the runner's process (A-469g). +func TestRunsTools(ask func(key string, body any) (json.RawMessage, error)) []stdio.Tool { + return []stdio.Tool{ + {Name: Prefix + "_test_runs", + Description: "The pipeline runner's test runs, one per candidate, level and tested packages (the state test-runs): each with its packages read from its output object, its ask, node, readers, outcome and counts, and the state's fill against its cap; filtered by a candidate, a level, a digest or a module, or none.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "candidate": map[string]any{"type": "string"}, "level": map[string]any{"type": "string"}, + "digest": map[string]any{"type": "string"}, "module": map[string]any{"type": "string"}}}, + Run: func(args map[string]any) (any, error) { + in, err := as[seat.TestRunsInput](args) + if err != nil { + return nil, err + } + return ask("seat:"+seat.Seat+"."+seat.VerbTestRuns, in) + }}, + {Name: Prefix + "_test_runs_drop", + Description: "Drop one test run by the seat verb test-runs-drop, a hand act recorded first: its ask is dropped, each reading step ended could not run naming the drop, and its key and output object deleted. It takes the run's key and why.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "key": map[string]any{"type": "string"}, "why": map[string]any{"type": "string"}}, "required": []string{"key", "why"}}, + Run: func(args map[string]any) (any, error) { + in, err := as[seat.TestRunsDropInput](args) + if err != nil { + return nil, err + } + if strings.TrimSpace(in.Key) == "" || strings.TrimSpace(in.Why) == "" { + return nil, errors.New("test-runs-drop names the run's key and why") + } + return ask("seat:"+seat.Seat+"."+seat.VerbTestRunsDrop, in) + }}, + } +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go index 3224a5a9..bda4f80b 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go @@ -48,6 +48,8 @@ func writeHandlers(v pipeline.Verbs) map[string]process.SeatHandler { seat.VerbCancel: verbOf(func(c string, in seat.CancelInput) (any, error) { return v.Cancel(c, in) }), seat.VerbStart: verbOf(func(c string, in seat.StartInput) (any, error) { return v.Start(c, in) }), seat.VerbAskAgain: verbOf(func(c string, in seat.AskAgainInput) (any, error) { return v.AskAgain(c, in) }), + // §728, A-469g: a test run's drop is answered by the process, the only writer of pipelines and test-runs. + seat.VerbTestRunsDrop: verbOf(func(c string, in seat.TestRunsDropInput) (any, error) { return v.TestRunsDrop(c, in) }), } } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go index 9e2d5d28..a0993929 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go @@ -23,7 +23,7 @@ func TestEveryWriteVerbIsServed(t *testing.T) { got = append(got, v) } slices.Sort(got) - want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbRerun, seat.VerbReview, seat.VerbStart, seat.VerbRun} + want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbRerun, seat.VerbReview, seat.VerbStart, seat.VerbRun, seat.VerbTestRunsDrop} slices.Sort(want) if !slices.Equal(got, want) { t.Errorf("the process answers %v, want %v", got, want) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index c31f6639..db8f1f1c 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -340,8 +340,17 @@ func (h *Hearer) endRunOf(p seat.Pipeline, s seat.StepResult, ev StepEnded) erro if ev.Tests != nil { end.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration} } - if ev.Output != "" && h.RunOutputs != nil { - if err := h.RunOutputs.Put(run, []byte(ev.Output)); err != nil { + if h.RunOutputs != nil { + // The run's output object holds the packages it ran with their digests and the build seat's output (§725). + out := RunOutput{Output: ev.Output, Packages: []seat.TestRunPackage{}} + for _, k := range append(append([]nodebuildgo.KeptResult(nil), ev.Kept...), ev.Carried...) { + out.Packages = append(out.Packages, seat.TestRunPackage{Package: k.Package, Digest: k.Digest}) + } + raw, err := json.Marshal(out) + if err == nil { + err = h.RunOutputs.Put(run, raw) + } + if err != nil { return fmt.Errorf("the output of the test run %s cannot be kept: %w", run, err) } end.Output = run diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_read.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_read.go new file mode 100644 index 00000000..0e863d17 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_read.go @@ -0,0 +1,160 @@ +package pipeline + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" +) + +// RunOutput is a test run's output object in test-runs-objects (§725): the packages the run ran with their digests, the +// passing ones it kept and the carried ones it skipped, and the build seat's output. +type RunOutput struct { + Packages []seat.TestRunPackage `json:"packages"` + Output string `json:"output,omitempty"` +} + +// Gone is what a run whose output object is gone answers in place of its packages (§725, §733). +const Gone = "the run's output is gone" + +// RunsRead answers the seat verb test-runs (design 47 G §728; A-469u) from the runner's state test-runs and its object +// store: each run with its packages read from its output object, and the state's fill. +type RunsRead struct { + Runs Kept + // Object reads an object of test-runs-objects; ErrNoRun when it is gone. + Object func(name string) ([]byte, error) + Fill func() (seat.Fill, error) +} + +// TestRuns answers each test run of a candidate, a level, a digest or a module, or every one. +func (r RunsRead) TestRuns(in seat.TestRunsInput) (seat.TestRunsAnswer, error) { + all, err := r.Runs.Keys() + if err != nil { + return seat.TestRunsAnswer{}, fmt.Errorf("the test runs are not listed: %w", err) + } + sort.Strings(all) + a := seat.TestRunsAnswer{Runs: []seat.TestRun{}} + for _, key := range all { + if IsRunBound(key) || (in.Candidate != "" && !strings.HasPrefix(key, in.Candidate+".")) { + continue + } + raw, err := r.Runs.Value(key) + if err != nil || raw == nil { + continue + } + var run Run + if err := json.Unmarshal(raw, &run); err != nil { + continue + } + if (in.Level != "" && run.Level != in.Level) || (in.Digest != "" && run.Digest != in.Digest) || (in.Module != "" && !readsModule(run, in.Module)) { + continue + } + t := run.Answer() + t.Key = key + r.packages(&t) + a.Runs = append(a.Runs, t) + } + if r.Fill != nil { + f, err := r.Fill() + if err != nil { + return seat.TestRunsAnswer{}, fmt.Errorf("the fill of test-runs cannot be read: %w", err) + } + a.Fill = f + } + return a, nil +} + +// packages names t's packages from its output object, or says it gone (§725, A-466v). +func (r RunsRead) packages(t *seat.TestRun) { + if t.Output == "" || r.Object == nil { + return + } + data, err := r.Object(t.Output) + if errors.Is(err, ErrNoRun) { + t.Gone = Gone + return + } + var out RunOutput + if err == nil { + err = json.Unmarshal(data, &out) + } + if err != nil { + t.Gone = "the run's output is not read: " + err.Error() + return + } + t.Packages = out.Packages +} + +// readsModule says whether a reader of run is a step of module, its artifact `:`. +func readsModule(run Run, module string) bool { + for _, o := range run.Readers { + if strings.HasPrefix(o.Artifact, module+":") { + return true + } + } + return false +} + +// TestRunsDrop is the seat verb test-runs-drop (design 47 G §728; A-469g, A-469v), answered by the runner's process: the +// hand act recorded first, the run's ask dropped, each reader ended could not run naming the drop, then the run's output +// object and its key deleted. It answers the run deleted and the readers it ended. +func (v Verbs) TestRunsDrop(caller string, in seat.TestRunsDropInput) (seat.TestRunsDropAnswer, error) { + h := v.Hearer + if h.TestRuns == nil { + return seat.TestRunsDropAnswer{}, errors.New("the runner keeps no test runs") + } + r, _, err := h.TestRuns.Get(in.Key) + if errors.Is(err, ErrNoRun) { + return seat.TestRunsDropAnswer{}, fmt.Errorf("no test run %q in test-runs", in.Key) + } + if err != nil { + return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not read: %w", in.Key, err) + } + if err := v.record(seat.VerbTestRunsDrop, "dropped the test run "+in.Key, in.Why); err != nil { + return seat.TestRunsDropAnswer{}, err + } + why := fmt.Sprintf("the test run %s was dropped by %s: %s", in.Key, caller, in.Why) + var ended []seat.TestRunReader + if r.Ended == nil { + if h.DropRun != nil && r.Ask != "" { + if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil { + h.logf("the drop of ask %s of the dropped test run %s is not answered: %v", r.Ask, in.Key, err) + } + } + if err := h.EndRun(in.Key, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil); err != nil { + return seat.TestRunsDropAnswer{}, err + } + for _, o := range r.Readers { + ended = append(ended, o.Of()) + } + } + if h.RunOutputs != nil { + // The run's object is named by its key (endRunOf); one the run named otherwise is deleted too. + for _, name := range []string{in.Key, r.Output} { + if name == "" { + continue + } + if err := h.RunOutputs.Delete(name); err != nil { + return seat.TestRunsDropAnswer{}, fmt.Errorf("the output object %s of the test run is not deleted, so its key is kept: %w", name, err) + } + } + } + if err := h.TestRuns.Delete(in.Key); err != nil { + return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not deleted: %w", in.Key, err) + } + if h.Clock != nil { + for _, k := range []string{BoundPrefix + in.Key, UntakenPrefix + in.Key} { + _ = h.Clock.Unbound(k) + } + } + gone := r.Answer() + gone.Key = in.Key + now := h.now() + if gone.Ended == nil { + gone.Ended, gone.Outcome, gone.Said = &now, seat.StepResultOutcomeCouldNotRun, why + } + return seat.TestRunsDropAnswer{Run: gone, Ended: ended}, nil +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go index 41185109..f37047d2 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -248,7 +248,7 @@ func TestATestRunsEndMovesEveryReaderAndIsCountedOnce(t *testing.T) { t.Errorf("%s counts %+v, want the run's 5 unit tests once", p.Pipeline, c) } } - if r, _, _ := f.runs.Get(unit.Shared[0].Run); r.Outcome != "failed" || r.Output != unit.Shared[0].Run || string(f.objects[r.Output]) != "--- FAIL: TestX" { + if r, _, _ := f.runs.Get(unit.Shared[0].Run); r.Outcome != "failed" || r.Output != unit.Shared[0].Run || !strings.Contains(string(f.objects[r.Output]), `"output":"--- FAIL: TestX"`) { t.Errorf("the ended run = %+v, objects %v; want its output kept under its key", r, f.objects) } saidReaders := 0 @@ -358,3 +358,49 @@ func TestARerunOfATestRunMovesEveryReaderThatDidNotPass(t *testing.T) { } } } + +// The test runs' tools (design 47 G §728, §733 unit; A-466v, A-469g, A-469u, A-469v): test-runs answers each run with its +// packages read from its output object, and a run whose object is gone without them, saying so; test-runs-drop records the +// hand act first, ends the readers could not run naming the drop, and deletes the key and its object; with no hand act +// recorded it does nothing. +func TestTheTestRunsAreReadAndDroppedByTheirVerbs(t *testing.T) { + f := newRunFixture() + p := f.start(t, 4, 9) + unit, lint := stepNamed(p, "app:program", "unit"), stepNamed(p, "app:program", "lint") + ended := time.Now() + if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask, + Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "passed", + Kept: []nodebuildgo.KeptResult{{Package: "app/x", Digest: "d1", Level: "unit"}}, Ended: &ended}); err != nil { + t.Fatal(err) + } + pipes := memKept{} + for k, e := range f.runs { + put(t, pipes, k, e.r) + } + read := RunsRead{Runs: pipes, Object: f.objects.Get, Fill: func() (seat.Fill, error) { return seat.Fill{Store: "test-runs", Bytes: 9, Cap: 16 << 20}, nil }} + a, err := read.TestRuns(seat.TestRunsInput{Level: "unit", Module: "app"}) + if err != nil || len(a.Runs) != 1 || len(a.Runs[0].Packages) != 1 || a.Runs[0].Packages[0] != (seat.TestRunPackage{Package: "app/x", Digest: "d1"}) || + a.Runs[0].Key != unit.Shared[0].Run || len(a.Runs[0].Readers) != 2 || a.Fill.Cap != 16<<20 { + t.Fatalf("test-runs = %+v, %v; want the unit run with its package and readers, and the fill", a, err) + } + delete(f.objects, unit.Shared[0].Run) // gone, whatever step-outputs' age: the run answers without its packages + if a, _ := read.TestRuns(seat.TestRunsInput{Level: "unit"}); len(a.Runs) != 1 || a.Runs[0].Gone != Gone || len(a.Runs[0].Packages) != 0 { + t.Errorf("a run whose output is gone = %+v", a.Runs) + } + acts := &actLog{} + v := Verbs{Hearer: f.h, Starter: f.s, Acts: acts} + if _, err := (Verbs{Hearer: f.h, Starter: f.s}).TestRunsDrop("op@shanks", seat.TestRunsDropInput{Key: lint.Shared[0].Run, Why: "stuck"}); err == nil { + t.Fatal("a drop with no hand act recorded was done") + } + f.objects[lint.Shared[0].Run] = []byte("{}") + d, err := v.TestRunsDrop("op@shanks", seat.TestRunsDropInput{Key: lint.Shared[0].Run, Why: "stuck"}) + if err != nil || len(*acts) != 1 || len(d.Ended) != 2 || d.Run.Key != lint.Shared[0].Run || len(f.drops) != 1 { + t.Fatalf("drop = %+v, %v; acts %v, drops %v", d, err, *acts, f.drops) + } + if _, _, err := f.runs.Get(lint.Shared[0].Run); err != ErrNoRun || f.objects[lint.Shared[0].Run] != nil { + t.Errorf("after the drop the key is %v and the object %q, want both gone", err, f.objects[lint.Shared[0].Run]) + } + if s := stepNamed(f.pipeline(p), "app:tools", "lint"); s.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(s.Why, "dropped by op@shanks") { + t.Errorf("a reader of the dropped run = %+v", s) + } +} diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index 99523c6d..9b3cb607 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -113,7 +113,8 @@ func TestManifest(t *testing.T) { "seat:node-build-node.accept.delivery", "seat:node-build-image.accept.check", "seat:node-build-image.accept.delivery"})}, {"uses operator-channel", equalSets(m.Uses, []string{"operator-channel"})}, // RN25 (§539): the runner's tools carry the prefix runner_. - {"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_unlogged", "runner_unlogged_append"})}, + {"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_unlogged", "runner_unlogged_append", + "runner_test_runs", "runner_test_runs_drop"})}, // A-305 (ADR 0227 rule 8, ADR 0240 rule 2): the process is judged by its unit, as every process of mesh-delivery, // never by a tool of its own; runner_health stays a tool. {"health is the unit", healthTool(raw) == "unit"}, diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index 46ed603b..893dcac0 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -18,7 +18,9 @@ "rerun", "accept", "cancel", - "ask-again" + "ask-again", + "test-runs", + "test-runs-drop" ] } ], @@ -27,7 +29,9 @@ "runner_first_view_again", "runner_health", "runner_unlogged", - "runner_unlogged_append" + "runner_unlogged_append", + "runner_test_runs", + "runner_test_runs_drop" ], "consumes": [ "mesh-delivery-indexer.take-in.taken", From ebe49b9bdd496010e749959638abc0b2573cb17f Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 11:48:28 +0100 Subject: [PATCH 08/14] runner: a test run's steps are said waiting for a build node until the holder takes the run's ask (A-469b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A test step queued on a build seat was said running, so the watchman's build-capacity waits never saw it; every step of a test run now waits for a build node from the run's ask, and at the holder's ask-taken each is said started again, running. Builds: design 47 G §725, §732, §86; A-469b --- .../internal/pipeline/approval.go | 2 +- .../internal/pipeline/place.go | 5 ++- .../internal/pipeline/place_test.go | 2 +- .../internal/pipeline/read.go | 2 +- .../internal/pipeline/runends.go | 44 ++++++++++++++++++- .../internal/pipeline/start.go | 6 +-- .../internal/pipeline/steps.go | 6 +-- .../internal/pipeline/supersede.go | 4 +- .../internal/pipeline/testruns.go | 5 ++- .../internal/pipeline/testruns_test.go | 13 ++++++ .../internal/pipeline/verbs.go | 8 +++- 11 files changed, 80 insertions(+), 17 deletions(-) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/approval.go b/modules/mesh-delivery-pipeline/internal/pipeline/approval.go index beeff984..c3d32346 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/approval.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/approval.go @@ -38,7 +38,7 @@ func approvalDue(p seat.Pipeline) int { } continue } - if s.Outcome == seat.StepResultOutcomeWaiting || s.Outcome == seat.StepResultOutcomeRunning { + if s.Outcome == seat.StepResultOutcomeWaiting || open(s) { return -1 } if s.Status == seat.StepResultStatusRequired && s.Outcome != seat.StepResultOutcomePassed && s.Outcome != seat.StepResultOutcomeCarried { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index 698e19b7..a9bec3c3 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -139,12 +139,13 @@ func place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog // Another artifact's step of this pipeline asks the run: this one reads it. first := &asks[j] first.Readers = append(first.Readers, reader) - s.Ask = "" + s.Ask, s.Outcome, s.On = "", seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode s.Shared = []seat.Shared{{Run: run, Ask: first.ID, AskedBy: &seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: first.Readers[0].Artifact}}} steps = append(steps, s) continue } - s.Shared = []seat.Shared{{Run: run, Ask: s.Ask}} + // A test run's steps wait for a build node from its ask until the holder's ask-taken (A-469b). + s.Shared, s.Outcome, s.On = []seat.Shared{{Run: run, Ask: s.Ask}}, seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode ask.Run, ask.Level, ask.Digest, ask.Readers = run, step, digest, []seat.TestRunReader{reader} askOfRun[run] = len(asks) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index f516c158..eb49aa9a 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -407,7 +407,7 @@ func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) { if asker.Ask == "" || len(asker.Shared) != 1 || asker.Shared[0].Ask != asker.Ask || asker.Shared[0].AskedBy != nil { t.Errorf("the asking %s step = %+v, want its ask named in shared with no asked-by", level, asker) } - if reader.Ask != "" || reader.Outcome != seat.StepResultOutcomeRunning || len(reader.Shared) != 1 || reader.Shared[0] != (seat.Shared{ + if reader.Ask != "" || reader.Outcome != seat.StepResultOutcomeWaitingForABuildNode || reader.On != seat.WaitOnBuildNode || len(reader.Shared) != 1 || reader.Shared[0] != (seat.Shared{ Run: asker.Shared[0].Run, Ask: asker.Ask, AskedBy: reader.Shared[0].AskedBy}) || reader.Shared[0].AskedBy == nil || *reader.Shared[0].AskedBy != (seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: "app:program"}) { t.Errorf("the reading %s step = %+v, want no ask and the program's run named", level, reader) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read.go b/modules/mesh-delivery-pipeline/internal/pipeline/read.go index 63d31d5a..9fed9e40 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read.go @@ -95,7 +95,7 @@ func (r Reads) Stands() (seat.StandsAnswer, error) { if s.Ended != nil || s.Started == nil { continue } - if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting { + if !open(s) && s.Outcome != seat.StepResultOutcomeWaiting { continue } out = append(out, seat.Wait{Pipeline: p.Pipeline, Module: p.Module, Branch: p.Branch, Commit: p.Commit, diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go index 8d98ba7e..122936ae 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go @@ -116,6 +116,7 @@ func (h *Hearer) RunTaken(ask, node string) error { return fmt.Errorf("the test run %s is not written: %w", run, err) } } + h.runStarted(run, r) break } if err := h.Clock.Bound(BoundPrefix+run, h.runSettings().StepBound); err != nil { @@ -296,7 +297,7 @@ func (h *Hearer) endReader(reader RunReader, run string, r Run) error { if s.Artifact != reader.Artifact || sharedRun(s) != run { continue } - if s.Outcome == seat.StepResultOutcomeRunning { + if open(s) { i, moved = j, false } else if reader.Rerun && s.Shared[0].Ask != r.Ask && s.Accepted == nil { i, moved = j, true @@ -392,3 +393,44 @@ func (h *Hearer) leaveRun(key string, s seat.StepResult, why string) { return } } + +// open says whether a step is asked and not ended: running, or a test step waiting for a build node to take its run's ask +// (A-469b). +func open(s seat.StepResult) bool { + return s.Outcome == seat.StepResultOutcomeRunning || s.Outcome == seat.StepResultOutcomeWaitingForABuildNode +} + +// runStarted moves each step reading run from waiting for a build node to running, the holder's ask-taken heard (A-469b), +// and says its step-started again, so the watchman's build-capacity waits let it go (§86, §732). +func (h *Hearer) runStarted(run string, r Run) { + for _, reader := range r.Readers { + for { + p, rev, err := h.Store.Get(reader.Pipeline) + if err != nil { + break + } + var moved []int + for j, s := range p.Steps { + if s.Artifact == reader.Artifact && sharedRun(s) == run && s.Outcome == seat.StepResultOutcomeWaitingForABuildNode { + p.Steps[j].Outcome, p.Steps[j].On = seat.StepResultOutcomeRunning, "" + moved = append(moved, j) + } + } + if len(moved) == 0 { + break + } + if _, err := h.Store.Update(reader.Pipeline, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + h.logf("the steps of %s reading the taken test run %s are not written running: %v", reader.Pipeline, run, err) + break + } + for _, j := range moved { + if err := h.say(seat.EventStepStarted, p, &p.Steps[j]); err != nil { + h.logf("step-started of %s's %s is not said: %v", reader.Pipeline, p.Steps[j].Step, err) + } + } + break + } + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index e3916834..7a6617df 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -105,7 +105,7 @@ func (s *Starter) holderless(steps []seat.StepResult, asks []Asked, now time.Tim none := map[string]bool{} for i := range steps { st := &steps[i] - if st.Outcome != seat.StepResultOutcomeRunning || !BoundSeat(st.Seat) { + if !open(*st) || !BoundSeat(st.Seat) { continue } if holders, read := s.Holders(st.Seat); !read || len(holders) > 0 { @@ -162,7 +162,7 @@ func (s *Starter) sayStarted(key string, asks []Asked) { } at := s.now() for _, st := range p.Steps { - if st.Outcome != seat.StepResultOutcomeRunning || !(ids[st.Ask] || (st.Ask == "" && runs[sharedRun(st)])) { + if !open(st) || !(ids[st.Ask] || (st.Ask == "" && runs[sharedRun(st)])) { continue } subject, ev := stepEvent(seat.EventStepStarted, p, st, at) @@ -278,7 +278,7 @@ func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) { // runs and keeps its own ask, ending by the path it was asked under (§730, A-468k). running, own := map[string]bool{}, map[string]bool{} for _, st := range p.Steps { - if st.Outcome == seat.StepResultOutcomeRunning && st.Ask != "" { + if open(st) && st.Ask != "" { running[st.Ask], own[st.Ask] = true, len(st.Shared) == 0 } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index db8f1f1c..7f2989ee 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -201,7 +201,7 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh case ev.Commit != p.Commit || ev.Fingerprint != p.Fingerprint || ev.Module != p.Module || ev.Step != p.Steps[i].Step: return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s at %s (%s)", ErrRefused, ev.ID, ev.Step, ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Commit, p.Fingerprint) - case p.Steps[i].Outcome != seat.StepResultOutcomeRunning: + case !open(p.Steps[i]): if run := sharedRun(p.Steps[i]); run != "" && h.TestRuns != nil { if r, _, err := h.TestRuns.Get(run); err == nil && r.Ended == nil && r.Ask == ev.ID { // The asking step's end was written and its run's was not (a write failed): the run ends now, heard again. @@ -310,7 +310,7 @@ func (h *Hearer) sameRunReaders(p *seat.Pipeline, i int, ev StepEnded, now time. continue } switch m := marks[s.Artifact]; { - case s.Outcome == seat.StepResultOutcomeRunning && s.Ask == "": + case open(*s) && s.Ask == "": case m.Rerun && s.Shared[0].Ask != ev.ID && s.Accepted == nil: s.Rerun, s.By = true, m.By s.Shared = []seat.Shared{{Run: run, Ask: ev.ID, AskedBy: &seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: asker.Artifact}}} @@ -550,7 +550,7 @@ func endPipeline(p *seat.Pipeline, now time.Time) bool { // An optional step still waiting (a code review nobody has given) holds nothing: only a required one holds the // pipeline's end (A-104). continue - case s.Outcome == seat.StepResultOutcomeWaiting, s.Outcome == seat.StepResultOutcomeRunning: + case s.Outcome == seat.StepResultOutcomeWaiting, open(s): return false } if s.Status == seat.StepResultStatusRequired && s.Outcome != seat.StepResultOutcomePassed && s.Outcome != seat.StepResultOutcomeCarried { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go index f008149d..30be4d2f 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go @@ -30,7 +30,7 @@ func (h *Hearer) Taken(ask, node string) error { break } i := stepOfAsk(p, ask) - if i < 0 || p.Steps[i].Outcome != seat.StepResultOutcomeRunning || p.Steps[i].Node == node { + if i < 0 || !open(p.Steps[i]) || p.Steps[i].Node == node { break } p.Steps[i].Node = node @@ -110,7 +110,7 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop var changed []int for i := range p.Steps { s := &p.Steps[i] - if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting { + if !open(*s) && s.Outcome != seat.StepResultOutcomeWaiting { continue } // A step of a test run leaves it: its ask is dropped only with its last reader (§726, leaveRun). diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go index be004716..46ea8e7c 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go @@ -321,9 +321,12 @@ func (s *Starter) readOthers(key string, joined map[string]Run) { for i := range p.Steps { st := &p.Steps[i] r, ok := joined[sharedRun(*st)] - if !ok || st.Outcome != seat.StepResultOutcomeRunning { + if !ok || !open(*st) { continue } + if r.Node != "" { + st.Outcome, st.On = seat.StepResultOutcomeRunning, "" // its run's ask is taken already (A-469b) + } st.Ask = "" st.Shared = []seat.Shared{{Run: sharedRun(*st), Ask: r.Ask, AskedBy: by[sharedRun(*st)]}} } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go index f37047d2..b57f2352 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -114,9 +114,22 @@ func TestATestRunHasOneClockACapAndAnAge(t *testing.T) { if f.clock.bound[UntakenPrefix+run] != 24*time.Hour || len(f.clock.bound) != 3 { t.Fatalf("bounds at the start = %v, want the three runs' untaken bounds of 24h", f.clock.bound) } + // A-469b: the run's steps wait for a build node until the holder's ask-taken, then run, each said started again. + if unit.Outcome != seat.StepResultOutcomeWaitingForABuildNode || stepNamed(p, "app:tools", "unit").Outcome != seat.StepResultOutcomeWaitingForABuildNode { + t.Fatalf("the unit steps before ask-taken = %s, %s", unit.Outcome, stepNamed(p, "app:tools", "unit").Outcome) + } + n := len(f.said) if err := f.h.RunTaken(unit.Ask, "ace"); err != nil { t.Fatal(err) } + for _, art := range []string{"app:program", "app:tools"} { + if s := stepNamed(f.pipeline(p), art, "unit"); s.Outcome != seat.StepResultOutcomeRunning { + t.Errorf("the %s unit step after ask-taken is %s, want running", art, s.Outcome) + } + } + if len(f.said)-n != 2 || f.said[n].Result.Outcome != seat.StepResultOutcomeRunning { + t.Errorf("ask-taken said %d step events, want the two unit steps started running", len(f.said)-n) + } if r, _, _ := f.runs.Get(run); r.Node != "ace" || f.clock.bound[BoundPrefix+run] != 30*time.Minute { t.Fatalf("after ask-taken the run = %+v, bounds %v; want its node and its clock of 30m", r, f.clock.bound) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go index 554bf8f3..1608d60c 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -206,7 +206,7 @@ func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, er var leave []seat.StepResult for i := range p.Steps { s := &p.Steps[i] - if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting { + if !open(*s) && s.Outcome != seat.StepResultOutcomeWaiting { continue } if sharedRun(*s) != "" && s.Seat != wants.Seat { @@ -306,7 +306,7 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult) (seat.RerunAnswer, error) { h := v.Hearer run := sharedRun(st) - if st.Outcome == seat.StepResultOutcomeRunning { + if open(st) { return seat.RerunAnswer{}, fmt.Errorf("the %s step of %s is running: it is not asked again", in.Step, in.Pipeline) } r, rev, err := h.TestRuns.Get(run) @@ -357,6 +357,7 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult) switch { case ask != nil: s.Ask, s.Shared = ask.ID, []seat.Shared{{Run: run, Ask: ask.ID}} + s.Outcome, s.On = seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode // until ask-taken (A-469b) case read: s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: asker}} s.Outcome, s.Why, s.Node, s.Ended = r.Outcome, r.Said, r.Node, &now @@ -366,6 +367,9 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult) } default: s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: asker}} + if r.Node == "" { + s.Outcome, s.On = seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode + } } if p.Outcome != seat.PipelineOutcomeRunning { p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeRunning, "", nil From 1a924f2ca72676b39af05183345280b41958ef22 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 11:58:25 +0100 Subject: [PATCH 09/14] runner: a seat with no holder ends its pipeline failed at once, its other steps not asked (A-470g, A-470h) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A pipeline with a step whose seat the seat-holders view names no holder for was failed only once its other steps ended, after their asks ran for nothing; it now ends failed at placing, naming the seat, with every other step not asked and said, and none of its steps asked. Builds: design 47 G §32, §535, §647 (as A-466y and A-466z amend them); A-470g, A-470h --- .../cmd/mesh-delivery-pipeline/main.go | 2 +- .../internal/pipeline/place_test.go | 38 +++++++-- .../internal/pipeline/run.go | 6 +- .../internal/pipeline/start.go | 83 ++++++++++++++----- .../internal/pipeline/steps.go | 19 +++++ 5 files changed, 118 insertions(+), 30 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 1551038a..9d7187c9 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -152,7 +152,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf, Say: say, Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, js: js, cred: cred}, Name: cred.Module, Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains), - Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders} + Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing} placeHalf := func(ev catalog.CatalogRevisionEvent, half pipeline.Half) error { n, err := starter.PlaceRun(ev, half, rv.Catalog.View) logger.Printf("the candidate %s's revision %d placed %d pipelines of its run", ev.Group, ev.Revision, n) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index eb49aa9a..5c933433 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -450,9 +450,10 @@ func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) { } // A seat with no holder is read only from the seat-holders view (design 47 G §535, §647 as A-466y and A-466z amend them): -// once that view's first view is read, a build step whose seat it names no holder for ends could not run naming the seat -// and is asked of no one; before it is read, or with a holder named, the step is asked and waits with no bound of the -// runner's own; the claimant of the view is the one module the catalog halves name. +// once that view's first view is read, a build step whose seat it names no holder for ends could not run naming the seat, +// and the pipeline ends failed at once naming it, every other step `not asked`, each said, none asked (A-470g, A-470h); +// before it is read, or with a holder named, the step is asked and waits with no bound of the runner's own; the claimant of +// the view is the one module the catalog halves name. func TestASeatWithNoHolderIsReadFromTheSeatHoldersView(t *testing.T) { reg, entries := placeFixture() for _, c := range []struct { @@ -463,9 +464,11 @@ func TestASeatWithNoHolderIsReadFromTheSeatHoldersView(t *testing.T) { }{{"the view not read yet", nil, false, true}, {"a holder named", []string{"mesh-build-go"}, true, true}, {"no holder named", nil, true, false}} { t.Run(c.name, func(t *testing.T) { store, asks := memStore{}, &stepAsks{} + var said []int s := &Starter{Pipelines: memCreated{store}, Say: func(string, any) error { return nil }, Steps: asks, Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }, - Holders: func(string) ([]string, bool) { return c.holders, c.read }} + Holders: func(string) ([]string, bool) { return c.holders, c.read }, + Ended: func(_ seat.Pipeline, steps []int) error { said = steps; return nil }} if ok, err := s.Start(Begin{Branch: "main", Source: 4, Module: "app", Commit: "c1", Fp: "src2:aa", Entries: entries}); !ok || err != nil { t.Fatalf("start = %v, %v", ok, err) } @@ -478,8 +481,31 @@ func TestASeatWithNoHolderIsReadFromTheSeatHoldersView(t *testing.T) { if c.asked && (lint.Outcome != seat.StepResultOutcomeRunning || !asked) { t.Errorf("the lint step = %+v, asked %v; want it running and asked", lint, asked) } - if !c.asked && (lint.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(lint.Why, "node-build-go has no holder") || len(asks.ids) != 2) { - t.Errorf("the lint step = %+v, %d asked; want could not run naming the seat, only the build and composition asked", lint, len(asks.ids)) + if c.asked { + if p.Outcome != seat.PipelineOutcomeRunning || said != nil { + t.Errorf("the pipeline = %s (%s), ended steps said %v; want it running", p.Outcome, p.Why, said) + } + return + } + if lint.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(lint.Why, "node-build-go has no holder") || len(asks.ids) != 0 { + t.Errorf("the lint step = %+v, %d asked; want could not run naming the seat, nothing asked", lint, len(asks.ids)) + } + if want := "the seat node-build-go has no holder: the seat-holders view names none"; p.Outcome != seat.PipelineOutcomeFailed || p.Why != want || p.Ended == nil || p.EndCount != 1 { + t.Errorf("the pipeline = %s (%q), ended %v, count %d; want failed at once, %q", p.Outcome, p.Why, p.Ended, p.EndCount, want) + } + ended := 0 + for _, st := range p.Steps { + switch { + case st.Seat == "node-build-go", st.Outcome == seat.StepResultOutcomeNotAsked: + ended++ + case st.Outcome == seat.StepResultOutcomeBypassed, st.Outcome == seat.StepResultOutcomeCouldNotRun && st.Seat == "": + // ended by Place itself: a bypass, or a language no build seat names + default: + t.Errorf("the step %s %s = %s; want not asked", st.Artifact, st.Step, st.Outcome) + } + } + if ended < 3 || len(said) != ended { + t.Errorf("%d steps ended at placing, %d said; want each said", ended, len(said)) } }) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/run.go b/modules/mesh-delivery-pipeline/internal/pipeline/run.go index becc1be8..5063e81c 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/run.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/run.go @@ -229,10 +229,11 @@ func (s *Starter) placeRunPipeline(key, id string, revision int64, half Half) (b } p.Fingerprint, p.Repository, p.Revision = e.Fp, e.Repository, revision var asks []Asked + var ended []int if s.Registration != nil { if reg, ok := s.Registration(p.Module); ok { p.Steps, asks = Place(p, reg, half.Entries, now, s.Name) - asks = s.holderless(p.Steps, asks, now) + asks, ended = s.holderless(&p, asks, now) } else { s.logf("%s has no registration yet: its pipeline is placed with no step", key) } @@ -247,6 +248,9 @@ func (s *Starter) placeRunPipeline(key, id string, revision int64, half Half) (b if err := s.Say(runnerclient.Subject(seat.EventPipelineStarted, p.Module, p.Branch, p.Commit), ev); err != nil { s.logf("pipeline-started of %s is not said: %v", key, err) } + if len(ended) > 0 { + return true, s.endedAtPlacing(p, ended) + } if s.Steps != nil && len(asks) > 0 { if err := s.askSteps(key, p.Commit, asks); err != nil { return true, err diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 7a6617df..dcc1ab14 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -4,8 +4,10 @@ import ( "encoding/json" "errors" "fmt" + "slices" "sort" "strconv" + "strings" "time" runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient" @@ -76,6 +78,9 @@ type Starter struct { // Holders are the modules the seat mesh-module-assigner's view seat-holders names for a seat, and whether that view's // first view is read (§535, §647 as A-466y, A-466z amend them); nil, or a view not read yet, judges no seat holderless. Holders func(seat string) ([]string, bool) + // Ended says a pipeline that ended at its placing (holderless): the step-ended of each step of steps, then its end + // (Hearer.EndedAtPlacing); nil says it in the journal alone. + Ended func(p seat.Pipeline, steps []int) error // Fills refuses a start while a store of the runner is full (§366); nil refuses none. Fills *Fills // Recovered says whether a commit is named a recovery (§81, §86): its steps are asked on the build seats' stream delivery. @@ -95,36 +100,60 @@ type Starter struct { EndReaders func(run string, readers []seat.TestRunReader, r seat.TestRun) error } -// holderless ends, as it is placed, each build step whose seat the seat mesh-module-assigner's view seat-holders names no holder -// for, `could not run` naming the seat, and asks it of no one (design 47 G §535, §647 as A-466y and A-466z amend them): judged -// only once that view's first view is read, never from the picture before it. It answers the asks still to put. -func (s *Starter) holderless(steps []seat.StepResult, asks []Asked, now time.Time) []Asked { +// holderless ends at once, as it is placed, a pipeline with a build step whose seat the seat mesh-module-assigner's view +// seat-holders names no holder for (design 47 G §32, §535, §647 as A-466y and A-466z amend them; A-470g, A-470h): judged only +// once that view's first view is read, never from the picture before it. Each such step ends `could not run` naming the seat, +// every other step not ended ends `not asked`, each with its step log entry, and the pipeline ends `failed` naming the seat; +// none of its steps is asked. It answers the asks still to put and the steps it ended, none when no seat is holderless. +func (s *Starter) holderless(p *seat.Pipeline, asks []Asked, now time.Time) ([]Asked, []int) { if s.Holders == nil { - return asks + return asks, nil } - none := map[string]bool{} - for i := range steps { - st := &steps[i] - if !open(*st) || !BoundSeat(st.Seat) { + var none []string + for _, st := range p.Steps { + if !open(st) || !BoundSeat(st.Seat) || slices.Contains(none, st.Seat) { continue } - if holders, read := s.Holders(st.Seat); !read || len(holders) > 0 { - continue + if holders, read := s.Holders(st.Seat); read && len(holders) == 0 { + none = append(none, st.Seat) } - st.Outcome, st.Ended, st.Ask, st.Shared = seat.StepResultOutcomeCouldNotRun, &now, "", nil - st.Why = fmt.Sprintf("the seat %s has no holder: the seat-holders view names none", st.Seat) - none[st.Seat] = true } if len(none) == 0 { - return asks + return asks, nil } - out := asks[:0:0] - for _, a := range asks { - if !none[a.Seat] { - out = append(out, a) + sort.Strings(none) + whys := make([]string, 0, len(none)) + for _, n := range none { + whys = append(whys, fmt.Sprintf("the seat %s has no holder: the seat-holders view names none", n)) + } + var ended []int + for i := range p.Steps { + st := &p.Steps[i] + if !open(*st) && st.Outcome != seat.StepResultOutcomeWaiting { + continue } + st.Ended, st.Ask, st.Shared = &now, "", nil + if slices.Contains(none, st.Seat) { + st.Outcome, st.Why = seat.StepResultOutcomeCouldNotRun, fmt.Sprintf("the seat %s has no holder: the seat-holders view names none", st.Seat) + } else { + st.Outcome, st.Why = seat.StepResultOutcomeNotAsked, "the pipeline failed at once: "+strings.Join(whys, "; ") + } + p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now)) + ended = append(ended, i) } - return out + p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeFailed, strings.Join(whys, "; "), &now + p.EndCount++ + return nil, ended +} + +// endedAtPlacing says a pipeline holderless ended at placing (A-470h): each step it ended in its step-ended, then the +// pipeline's end, through Ended; with none given it is said in the journal alone. +func (s *Starter) endedAtPlacing(p seat.Pipeline, steps []int) error { + s.logf("%s ended failed at once: %s (A-470g)", p.Pipeline, p.Why) + if s.Ended == nil { + return nil + } + return s.Ended(p, steps) } // askSteps puts a placed pipeline's asks (§22, §24): a test run another step asked is read, not asked again (share), then @@ -202,12 +231,13 @@ func (s *Starter) Start(b Begin) (bool, error) { } p := newPipeline(b, key, now) var asks []Asked + var ended []int if b.unheld { s.logf("%s starts with no step: the catalog half of %s is not held yet, and its steps are placed once it is (A-336)", key, b.Branch) } else if s.Registration != nil { if reg, ok := s.Registration(b.Module); ok { p.Steps, asks = Place(p, reg, b.Entries, now, s.Name) - asks = s.holderless(p.Steps, asks, now) + asks, ended = s.holderless(&p, asks, now) } else { s.logf("%s has no registration yet: its pipeline starts with no step and waits on the registration of %s, which places them (A-327)", key, b.Module) } @@ -231,6 +261,9 @@ func (s *Starter) Start(b Begin) (bool, error) { if s.Supersede != nil { s.Supersede(p) } + if len(ended) > 0 { + return true, s.endedAtPlacing(p, ended) + } if s.Steps != nil && len(asks) > 0 { if err := s.askSteps(key, b.Commit, asks); err != nil { return true, err @@ -379,7 +412,7 @@ func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipe } var asks []Asked p.Steps, asks = Place(p, r, half.Entries, now, s.Name) - asks = s.holderless(p.Steps, asks, now) + asks, ended := s.holderless(&p, asks, now) if _, err := store.Update(key, p, rev); err != nil { s.logf("%s is written meanwhile, so nothing is placed in it now: %v", key, err) continue @@ -395,6 +428,12 @@ func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipe s.logf("pipeline-started of %s is not said: %v", key, err) } } + if len(ended) > 0 { + if err := s.endedAtPlacing(p, ended); err != nil { + note(err) + } + continue + } if s.Steps != nil && len(asks) > 0 { if err := s.askSteps(key, p.Commit, asks); err != nil { note(err) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 7f2989ee..718e322b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -571,3 +571,22 @@ func endPipeline(p *seat.Pipeline, now time.Time) bool { } return true } + +// EndedAtPlacing says a pipeline its starter ended at placing, a seat with no holder (A-470g, A-470h): the step-ended of each +// step of steps, its step log entries appended, the head's finding pipeline-failed (§33), then its pipeline-ended. +func (h *Hearer) EndedAtPlacing(p seat.Pipeline, steps []int) error { + var first error + for _, i := range steps { + if err := h.say(seat.EventStepEnded, p, &p.Steps[i]); err != nil && first == nil { + first = fmt.Errorf("pipeline %s's %s is written but not said: %w", p.Pipeline, seat.EventStepEnded, err) + } + } + h.appendUnlogged(p.Pipeline) + if len(steps) > 0 { + h.Failures.StepEnded(h.Store, p, p.Steps[steps[0]]) + } + if err := h.said(p, false, true); err != nil && first == nil { + first = err + } + return first +} From 19fbc076d672a10b0c65eeeb10ec0a71a9779786 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 12:00:35 +0100 Subject: [PATCH 10/14] runner: test-runs is answered by the process, its fill said against test-runs-cap-mib (A-470g, A-470i) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bundle said test-runs' fill against the bucket's 64 MiB ceiling, a cap that refuses nothing; the process, which holds test-runs-cap-mib and judges it, now answers the verb test-runs, its fill's cap being that setting, and runner_test_runs calls the verb. Builds: design 47 G §728, §725 (the cap); A-469d, A-470g, A-470i --- .../cmd/mesh-delivery-pipeline-tools/main.go | 16 ++-------- .../mesh-delivery-pipeline-tools/testruns.go | 32 ++----------------- .../cmd/mesh-delivery-pipeline/main.go | 6 ++-- .../cmd/mesh-delivery-pipeline/testruns.go | 25 +++++++++++++++ .../cmd/mesh-delivery-pipeline/verbs.go | 5 ++- .../cmd/mesh-delivery-pipeline/verbs_test.go | 2 +- .../internal/pipeline/testruns.go | 26 +++++++++++---- .../internal/pipeline/testruns_test.go | 10 ++++-- .../internal/pipeline/verbs.go | 2 ++ 9 files changed, 67 insertions(+), 57 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go index 9523c3ce..3a4f3e7b 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go @@ -46,8 +46,8 @@ func as[T any](args map[string]any) (T, error) { return in, nil } -// Tools are the runner's read verbs over its states pipelines, kept and test-runs. -func Tools(r pipeline.Reads, testRuns ...pipeline.RunsRead) []stdio.Tool { +// Tools are the runner's read verbs over its states pipelines and kept; test-runs is the process's (A-470i). +func Tools(r pipeline.Reads) []stdio.Tool { runs := map[string]func(map[string]any) (any, error){ seat.VerbShow: func(a map[string]any) (any, error) { in, err := as[seat.ShowInput](a) @@ -79,15 +79,6 @@ func Tools(r pipeline.Reads, testRuns ...pipeline.RunsRead) []stdio.Tool { return r.KeptResults(in) }, } - for _, t := range testRuns { - runs[seat.VerbTestRuns] = func(a map[string]any) (any, error) { - in, err := as[seat.TestRunsInput](a) - if err != nil { - return nil, err - } - return t.TestRuns(in) - } - } var out []stdio.Tool for _, v := range seat.Verbs() { if run, ok := runs[v]; ok { @@ -157,8 +148,7 @@ func main() { func(n string) (buscalls.Fill, error) { return busstate.ObjectsOf(n).Fill() }, func(n string) (*stdio.LogInfo, error) { return stdio.Log(n).Info() }, time.Now) r := pipeline.Reads{Pipelines: kept{stdio.State(seat.StatePipelines)}, Kept: kept{stdio.State(seat.StateKept)}, Fill: fill} - runs := RunsRead(stdio.State(testRunsState), stdio.State(testRunsState).Info, busstate.ObjectsOf(testRunsObjects)) - tools := append(Tools(r, runs), FirstViewTools(os.Getenv(stdio.ServedAs), stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...) + tools := append(Tools(r), FirstViewTools(os.Getenv(stdio.ServedAs), stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...) tools = append(tools, UnloggedTools(stdio.State(seat.StatePipelines), stdio.Log(seat.StateStepLog), meshhandactsclient.Recorder{Ask: stdio.Ask}, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) })...) tools = append(tools, CancelTool(r.List, stdio.Ask), RerunTool(stdio.Ask)) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/testruns.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/testruns.go index de196cab..39800750 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/testruns.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/testruns.go @@ -7,42 +7,14 @@ import ( seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" stdio "git.novox.be/novox/mesh-sdk/go" - "git.novox.be/novox/mesh-sdk/go/busstate" - "git.novox.be/novox/mesh-sdk/go/catalogue" - - "mesh-delivery-pipeline/internal/pipeline" ) -// testRunsState is the runner's state of its test runs (§725), and testRunsObjects its own object store (A-469e). -const testRunsState = "test-runs" - -var testRunsObjects = catalogue.StateObjectsName(testRunsState) - -// RunsRead is the verb test-runs' reader over the runner's state test-runs and its object store, through the bundle's calls. -func RunsRead(state stdio.KeptState, info func() (*stdio.StateInfo, error), objects busstate.Objects) pipeline.RunsRead { - return pipeline.RunsRead{Runs: kept{state}, - Object: func(name string) ([]byte, error) { - o, err := objects.Get(name) - if err != nil && strings.Contains(err.Error(), "not found") { - return nil, pipeline.ErrNoRun - } - return o.Data, err - }, - Fill: func() (seat.Fill, error) { - i, err := info() - if err != nil { - return seat.Fill{}, err - } - return seat.Fill{Store: testRunsState, Bytes: int64(i.Bytes), Cap: i.MaxBytes}, nil - }} -} - // TestRunsTools are runner_test_runs and runner_test_runs_drop (design 47 G §728; A-469f, A-469u): each calls its seat verb -// over the bus, test-runs answered by this bundle and test-runs-drop by the runner's process (A-469g). +// over the bus, both answered by the runner's process, which holds test-runs-cap-mib (A-469g, A-470i). func TestRunsTools(ask func(key string, body any) (json.RawMessage, error)) []stdio.Tool { return []stdio.Tool{ {Name: Prefix + "_test_runs", - Description: "The pipeline runner's test runs, one per candidate, level and tested packages (the state test-runs): each with its packages read from its output object, its ask, node, readers, outcome and counts, and the state's fill against its cap; filtered by a candidate, a level, a digest or a module, or none.", + Description: "The pipeline runner's test runs, one per candidate, level and tested packages (the state test-runs): each with its packages read from its output object, its ask, node, readers, outcome and counts, and the state's fill against its cap setting test-runs-cap-mib; filtered by a candidate, a level, a digest or a module, or none.", Input: map[string]any{"type": "object", "properties": map[string]any{ "candidate": map[string]any{"type": "string"}, "level": map[string]any{"type": "string"}, "digest": map[string]any{"type": "string"}, "module": map[string]any{"type": "string"}}}, diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 9d7187c9..11dcd1c6 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -238,7 +238,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } // The test runs (§724, §725): one run per candidate, level and digest, kept in the state test-runs with its clock. runDrop := pipeline.BoundDrop(dropAsk(calls, cred.Module), armDropBound(ctx, bounds), dropAnswered(ctx, deleteBound(ctx, bounds), rv.Findings)) - runsKV, _, err := openTestRuns(ctx, js, cred, getenv, starter, hearer, runDrop, logger.Printf) + runsKV, runObjs, err := openTestRuns(ctx, js, cred, getenv, starter, hearer, runDrop, logger.Printf) if err != nil { return err } @@ -297,7 +297,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // mesh-wants), never on EVENTS: bound here, or no step of any pipeline is ever heard ending (G-1 §24). logger.Printf("the kept seat events heard: %d streams bound", hub.StartKept(ctx, js, cred)) // The seat's write verbs are answered here, by the process that holds the pipelines (A-107). - stopVerbs, err := answerVerbs(calls, seats, writeVerbs(hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls)) + verbs := writeVerbs(hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls) + verbs.Runs = pipeline.RunsRead{Runs: runsKept{ctx: ctx, kv: runsKV}, Object: runObjs.Get, Fill: starter.TestRunsFill} + stopVerbs, err := answerVerbs(calls, seats, verbs) if err != nil { return err } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go index e26358bb..b8a6634c 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go @@ -112,6 +112,31 @@ func (t testRunsKV) Keys() ([]string, error) { return all, err } +// runsKept is the state test-runs as the verb test-runs reads it (A-470i): its keys, and each key's value as written. +type runsKept struct { + ctx context.Context + kv jetstream.KeyValue +} + +func (r runsKept) Keys() ([]string, error) { + ks, err := r.kv.Keys(r.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return ks, err +} + +func (r runsKept) Value(key string) (json.RawMessage, error) { + e, err := r.kv.Get(r.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return e.Value(), nil +} + // runObjects is the state test-runs' own object store, `test-runs-objects`, made with the state (A-469e), opened when // first used. type runObjects struct { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go index bda4f80b..5fd6c7dd 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go @@ -38,7 +38,8 @@ func answerVerbs(conn *process.Conn, seats *seatlist.Part, v pipeline.Verbs) (fu return conn.AnswerSeat(seat.Seat, seatlist.Definition(seats.Reader, seat.Seat), writeHandlers(v)) } -// writeHandlers are the seat's write verbs the process answers: run, review, rerun, accept, cancel, start, and ask-again. +// writeHandlers are the seat's write verbs the process answers: run, review, rerun, accept, cancel, start, and ask-again; and +// the test runs' verbs test-runs and test-runs-drop (A-469g, A-470i). func writeHandlers(v pipeline.Verbs) map[string]process.SeatHandler { return map[string]process.SeatHandler{ seat.VerbRun: verbOf(func(c string, in seat.RunInput) (any, error) { return v.Run(c, in) }), @@ -50,6 +51,8 @@ func writeHandlers(v pipeline.Verbs) map[string]process.SeatHandler { seat.VerbAskAgain: verbOf(func(c string, in seat.AskAgainInput) (any, error) { return v.AskAgain(c, in) }), // §728, A-469g: a test run's drop is answered by the process, the only writer of pipelines and test-runs. seat.VerbTestRunsDrop: verbOf(func(c string, in seat.TestRunsDropInput) (any, error) { return v.TestRunsDrop(c, in) }), + // A-470i: test-runs too, since the process holds test-runs-cap-mib, the cap its fill is said against. + seat.VerbTestRuns: verbOf(func(_ string, in seat.TestRunsInput) (any, error) { return v.Runs.TestRuns(in) }), } } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go index a0993929..a8f84fcd 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go @@ -23,7 +23,7 @@ func TestEveryWriteVerbIsServed(t *testing.T) { got = append(got, v) } slices.Sort(got) - want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbRerun, seat.VerbReview, seat.VerbStart, seat.VerbRun, seat.VerbTestRunsDrop} + want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbRerun, seat.VerbReview, seat.VerbStart, seat.VerbRun, seat.VerbTestRuns, seat.VerbTestRunsDrop} slices.Sort(want) if !slices.Equal(got, want) { t.Errorf("the process answers %v, want %v", got, want) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go index 46ea8e7c..23cf0685 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go @@ -208,22 +208,34 @@ func (e *fullErr) Error() string { return fmt.Sprintf("the state %s holds %d of its %d bytes, so no test run is asked (test-runs-cap-mib)", e.fill.Store, e.fill.Bytes, e.fill.Cap) } +// TestRunsFill is test-runs' fill against its cap setting test-runs-cap-mib, the cap that refuses a new run (A-469d, A-470i), +// never the manifest's ceiling. +func (s *Starter) TestRunsFill() (seat.Fill, error) { + if s.RunsFill == nil { + return seat.Fill{}, errors.New("the runner reads no fill of test-runs") + } + f, err := s.RunsFill() + if err != nil { + return seat.Fill{}, err + } + f.Cap = s.Runs.CapBytes + if f.Cap <= 0 { + f.Cap = DefaultRunSettings.CapBytes + } + return f, nil +} + // atCap is the error of test-runs at its cap setting, nil below it or when its fill is not read (said). func (s *Starter) atCap() error { if s.RunsFill == nil { return nil } - f, err := s.RunsFill() + f, err := s.TestRunsFill() if err != nil { s.logf("the fill of test-runs is not read, so its cap is not judged: %v", err) return nil } - cap := s.Runs.CapBytes - if cap <= 0 { - cap = DefaultRunSettings.CapBytes - } - if f.Bytes >= cap { - f.Cap = cap + if f.Bytes >= f.Cap { return &fullErr{fill: f} } return nil diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go index b57f2352..77cdf4e2 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -390,11 +390,15 @@ func TestTheTestRunsAreReadAndDroppedByTheirVerbs(t *testing.T) { for k, e := range f.runs { put(t, pipes, k, e.r) } - read := RunsRead{Runs: pipes, Object: f.objects.Get, Fill: func() (seat.Fill, error) { return seat.Fill{Store: "test-runs", Bytes: 9, Cap: 16 << 20}, nil }} + // The fill is said against the cap setting that refuses, never the bucket's 64 MiB ceiling (A-470g, A-470i). + f.s.Runs.CapBytes, f.fill = 1<<20, 9 + bucket := f.s.RunsFill + f.s.RunsFill = func() (seat.Fill, error) { fl, err := bucket(); fl.Cap = 64 << 20; return fl, err } + read := RunsRead{Runs: pipes, Object: f.objects.Get, Fill: f.s.TestRunsFill} a, err := read.TestRuns(seat.TestRunsInput{Level: "unit", Module: "app"}) if err != nil || len(a.Runs) != 1 || len(a.Runs[0].Packages) != 1 || a.Runs[0].Packages[0] != (seat.TestRunPackage{Package: "app/x", Digest: "d1"}) || - a.Runs[0].Key != unit.Shared[0].Run || len(a.Runs[0].Readers) != 2 || a.Fill.Cap != 16<<20 { - t.Fatalf("test-runs = %+v, %v; want the unit run with its package and readers, and the fill", a, err) + a.Runs[0].Key != unit.Shared[0].Run || len(a.Runs[0].Readers) != 2 || a.Fill.Cap != 1<<20 || a.Fill.Bytes != 9 { + t.Fatalf("test-runs = %+v, %v; want the unit run with its package and readers, and the fill against the cap setting", a, err) } delete(f.objects, unit.Shared[0].Run) // gone, whatever step-outputs' age: the run answers without its packages if a, _ := read.TestRuns(seat.TestRunsInput{Level: "unit"}); len(a.Runs) != 1 || a.Runs[0].Gone != Gone || len(a.Runs[0].Packages) != 0 { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go index 1608d60c..4626d260 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -27,6 +27,8 @@ type Verbs struct { Candidate func(group, branch string, revision int64) (Half, error) // Acts records a hand act before the verb does it (rerun, accept, cancel); a nil Acts refuses the verb, nothing recorded. Acts views.HandActs + // Runs answers the read verb test-runs, answered by the process that holds test-runs-cap-mib (A-470i). + Runs RunsRead } func (v Verbs) now() time.Time { return v.Hearer.now() } From a439b8da4787cc8e87c64728566fd8ebdf5845d4 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 12:16:01 +0100 Subject: [PATCH 11/14] runner: a carried reader with rerun false says no step-ended again at a take-up (A-470l) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runner's take-up of a rerun's test run already ends only the readers marked rerun and the open ones (endReader); this table case holds it there, so a reader that passed is never said again. Builds: design 47 G §725 (take-up), §727; A-470d, A-470l --- .../internal/pipeline/testruns_test.go | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go index 77cdf4e2..6000bd0b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -311,7 +311,10 @@ func TestATestRunsEndMovesEveryReaderAndIsCountedOnce(t *testing.T) { // passing end moves the other artifact's step to passed, marked a rerun by the caller and naming the new ask, and the // pipeline says pipeline-ended again; a step a person accepted does not move. func TestARerunOfATestRunMovesEveryReaderThatDidNotPass(t *testing.T) { - for _, accepted := range []bool{false, true} { + // takeUp: the rerun's run is ended by the restart's take-up, its bounds gone while the runner was away; a carried reader + // with rerun false, its step passed already, says no step-ended again (A-470l). + for _, c := range []struct{ accepted, takeUp bool }{{false, false}, {true, false}, {false, true}} { + accepted := c.accepted f := newRunFixture() reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Policy: catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "required", Integration: "bypass", Composition: "bypass", Review: "bypass", Approval: "bypass"}}} @@ -350,6 +353,35 @@ func TestARerunOfATestRunMovesEveryReaderThatDidNotPass(t *testing.T) { t.Fatal(err) } again := a.Result.Ask + if c.takeUp { + run := unit.Shared[0].Run + r, rev, _ := f.runs.Get(run) + r.Readers[0].Rerun, r.Readers[0].By = false, "" // the program's step passed already + if _, err := f.runs.Update(run, r, rev); err != nil { + t.Fatal(err) + } + e := f.store[p.Pipeline] + e.p.Steps[stepIndex(e.p, "unit", "app:program")].Outcome = seat.StepResultOutcomePassed + f.store[p.Pipeline] = e + for k := range f.clock.bound { + delete(f.clock.bound, k) + } + said := len(f.said) + if n, err := f.h.TakeUp(); err != nil || n != 1 { + t.Fatalf("take-up = %d, %v", n, err) + } + for _, ev := range f.said[said:] { + if ev.Result.Artifact == "app:program" { + t.Errorf("a carried reader with rerun false said %s again: %+v", ev.Result.Outcome, ev) + } + } + q := f.pipeline(p) + if stepNamed(q, "app:program", "unit").Outcome != seat.StepResultOutcomePassed || stepNamed(q, "app:tools", "unit").Outcome != seat.StepResultOutcomeCouldNotRun { + t.Errorf("after the take-up program %s, tools %s; want passed kept, tools could not run", + stepNamed(q, "app:program", "unit").Outcome, stepNamed(q, "app:tools", "unit").Outcome) + } + continue + } r, _, _ := f.runs.Get(unit.Shared[0].Run) if again == "" || again == unit.Ask || r.Ask != again || r.Ended != nil || len(r.Readers) != 2 || r.Readers[0].Rerun == accepted || (!accepted && r.Readers[0].By != "op@shanks") { t.Fatalf("the rerun's run = %+v (ask %s), want a new run under the same key carrying the program, marked unless accepted", r, again) From 2a90323af3ffd31603e9d0c8b1ebc729d56ae710 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 12:23:40 +0100 Subject: [PATCH 12/14] runner: no write of a test run acts on a read made before its latest change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The age and test-runs-drop deleted a run's key whatever was written after their read, an expiry or the last reader's leave ended whatever run the key held by then, a rerun answered a write between its read and its write as an error, and a step-progress heard as its run ended wrote a clock that later ended the run asked next. A delete of test-runs is now a compare-and-set at the revision read, retried from the latest read; a run's end ends only the run under the ask its caller judged; the last reader ends the run in its own leave; a rerun writes its new run from the latest read; a clock is read against the run after it is written, and a clock of a run not taken ends nothing. Builds: design 47 G §725, §726, §727, §728; A-469l, A-469z, A-470d --- .../cmd/mesh-delivery-pipeline/testruns.go | 18 ++- .../internal/pipeline/aged.go | 33 ++-- .../internal/pipeline/place_test.go | 8 +- .../internal/pipeline/runends.go | 50 ++++-- .../internal/pipeline/steps.go | 2 +- .../internal/pipeline/testruns.go | 5 +- .../internal/pipeline/testruns_read.go | 22 ++- .../internal/pipeline/testruns_test.go | 152 +++++++++++++++++- .../internal/pipeline/verbs.go | 61 ++++--- 9 files changed, 295 insertions(+), 56 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go index b8a6634c..da948880 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go @@ -74,7 +74,19 @@ func (t testRunsKV) Update(key string, r pipeline.Run, revision uint64) (uint64, return rev, err } -func (t testRunsKV) Delete(key string) error { +func (t testRunsKV) Delete(key string, revision uint64) error { + err := t.kv.Delete(t.ctx, key, jetstream.LastRevision(revision)) + if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) { + return pipeline.ErrChanged + } + if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + return err + } + return nil +} + +// deleteBound deletes a bound key, whose value is no run: a key gone is no fault. +func (t testRunsKV) deleteBound(key string) error { if err := t.kv.Delete(t.ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { return err } @@ -84,7 +96,7 @@ func (t testRunsKV) Delete(key string) error { // Bound writes key again with its time to live: deleted, then created with ttl, so its expiry is heard only ttl after // the last write (a delete is no expiry). func (t testRunsKV) Bound(key string, ttl time.Duration) error { - if err := t.Delete(key); err != nil { + if err := t.deleteBound(key); err != nil { return err } _, err := t.kv.Create(t.ctx, key, []byte(`{"bound":"`+key+`"}`), jetstream.KeyTTL(ttl)) @@ -94,7 +106,7 @@ func (t testRunsKV) Bound(key string, ttl time.Duration) error { return err } -func (t testRunsKV) Unbound(key string) error { return t.Delete(key) } +func (t testRunsKV) Unbound(key string) error { return t.deleteBound(key) } func (t testRunsKV) Has(key string) (bool, error) { _, err := t.kv.Get(t.ctx, key) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/aged.go b/modules/mesh-delivery-pipeline/internal/pipeline/aged.go index 948ac76a..bf9b7dda 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/aged.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/aged.go @@ -1,6 +1,7 @@ package pipeline import ( + "errors" "fmt" "time" @@ -139,23 +140,29 @@ func (a *Ager) ageRun(r Retention, e seat.LogEntry) { if run == "" { return } - t, _, err := a.Runs.Get(run) - // The entry is past the retention (ageAt), and a run ends no later than the reader's end it was read at: a run not - // ended yet is aged at a later reader's end. - if err != nil || t.Ended == nil { - return - } - if t.Output != "" && a.RunOutputs != nil { - if err := a.RunOutputs(t.Output); err != nil { - a.logf("aged: the output %s of the test run %s is not deleted, so the run is kept: %v", t.Output, run, err) + for { + t, rev, err := a.Runs.Get(run) + // The entry is past the retention (ageAt), and a run ends no later than the reader's end it was read at: a run not + // ended yet is aged at a later reader's end. + if err != nil || t.Ended == nil { return } - } - if err := a.Runs.Delete(run); err != nil { - a.logf("aged: the test run %s is not deleted: %v", run, err) + if t.Output != "" && a.RunOutputs != nil { + if err := a.RunOutputs(t.Output); err != nil { + a.logf("aged: the output %s of the test run %s is not deleted, so the run is kept: %v", t.Output, run, err) + return + } + } + // Deleted at the revision read: a rerun that wrote the key again since is read again, and a run it started is kept. + if err := a.Runs.Delete(run, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + a.logf("aged: the test run %s is not deleted: %v", run, err) + return + } + a.logf("aged: deleted the test run %s and its output (%s after its end)", run, r.Age) return } - a.logf("aged: deleted the test run %s and its output (%s after its end)", run, r.Age) } func (a *Ager) ageEntry(r Retention, e seat.LogEntry, now time.Time) { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index 5c933433..eec07992 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -355,7 +355,13 @@ func (m testRunsMem) Get(key string) (Run, uint64, error) { return e.r, e.rev, nil } -func (m testRunsMem) Delete(key string) error { delete(m, key); return nil } +func (m testRunsMem) Delete(key string, rev uint64) error { + if e, ok := m[key]; ok && e.rev != rev { + return ErrChanged + } + delete(m, key) + return nil +} func (m testRunsMem) Update(key string, r Run, rev uint64) (uint64, error) { if m[key].rev != rev { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go index 122936ae..d694bbb2 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go @@ -119,12 +119,29 @@ func (h *Hearer) RunTaken(ask, node string) error { h.runStarted(run, r) break } - if err := h.Clock.Bound(BoundPrefix+run, h.runSettings().StepBound); err != nil { + if err := h.bound(run, ask); err != nil { return fmt.Errorf("the clock of the test run %s is not begun: %w", run, err) } return h.Clock.Unbound(UntakenPrefix + run) } +// bound writes the clock of the run under ask, judged from a read of the run, then reads the run again: a run ended since, +// or asked again and not taken, has its clock deleted, so no bound written from the earlier read ends the run after it. +func (h *Hearer) bound(run, ask string) error { + if err := h.Clock.Bound(BoundPrefix+run, h.runSettings().StepBound); err != nil { + return err + } + r, _, err := h.TestRuns.Get(run) + if err != nil && !errors.Is(err, ErrNoRun) { + return err + } + // A run asked again and taken since keeps the clock: it is its own as well (its ask-taken writes it). + if errors.Is(err, ErrNoRun) || r.Ended != nil || (r.Ask != ask && r.Node == "") { + return h.Clock.Unbound(BoundPrefix + run) + } + return nil +} + // RunProgress takes a build seat's step-progress of a test run's ask (§725, A-466p): its clock begins again, so a run that // keeps saying its progress is never ended by it. An ask of no running, taken test run takes nothing. func (h *Hearer) RunProgress(ask string) error { @@ -139,7 +156,7 @@ func (h *Hearer) RunProgress(ask string) error { if err != nil { return fmt.Errorf("the test run %s is not read: %w", run, err) } - return h.Clock.Bound(BoundPrefix+run, h.runSettings().StepBound) + return h.bound(run, ask) } // RunExpired takes the expiry of a test run's bound key (§725, §720 *When the clock ends*, A-466s): the run ends `could not @@ -157,7 +174,8 @@ func (h *Hearer) RunExpired(key string) error { if err != nil { return fmt.Errorf("the test run %s is not read: %w", run, err) } - if r.Ended != nil || (untaken && r.Node != "") { + // An untaken bound of a taken run, and a clock of a run not taken (a clock left from an ask before a rerun's), end nothing. + if r.Ended != nil || (untaken && r.Node != "") || (!untaken && r.Node == "") { return nil } s := h.runSettings() @@ -170,7 +188,7 @@ func (h *Hearer) RunExpired(key string) error { h.logf("the drop of ask %s of the test run %s is not answered: %v", r.Ask, run, err) } } - return h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil) + return h.EndRun(run, seat.TestRun{Ask: r.Ask, Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil) } // TakeUp takes up again, at the runner's start, each test run with no end (§725): a taken run whose clock is gone, and an @@ -233,7 +251,8 @@ func (h *Hearer) runSettings() RunSettings { // EndRun ends the test run under run with end's outcome, words, counts, node and output, written once by compare-and-set, // and ends each of its readers with it but skip, the step whose own end ended it (§725, §726). A run ended already is not -// ended again. +// ended again, and end's ask, the one its caller judged, ends only a run under that ask: a run a rerun asked since is not +// ended by a judgement of the one before. func (h *Hearer) EndRun(run string, end seat.TestRun, skip *seat.TestRunReader) error { var r Run for { @@ -244,7 +263,7 @@ func (h *Hearer) EndRun(run string, end seat.TestRun, skip *seat.TestRunReader) if err != nil { return fmt.Errorf("the test run %s is not read: %w", run, err) } - if got.Ended != nil { + if got.Ended != nil || (end.Ask != "" && got.Ask != end.Ask) { return nil } now := h.now() @@ -373,23 +392,34 @@ func (h *Hearer) leaveRun(key string, s seat.StepResult, why string) { return } r.Readers = left + // The last to leave ends the run in the same write, so a step joining it after that read finds it ended and asks + // its own, never joining a run then ended for having no reader. + last := len(left) == 0 && r.Ended == nil + if last { + now := h.now() + r.Ended, r.Outcome, r.Said = &now, seat.StepResultOutcomeCouldNotRun, NoReaderLeft + } if _, err := h.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) { continue } else if err != nil { h.logf("the step %s of %s does not leave the test run %s: %v", s.Step, key, run, err) return } - if len(left) > 0 || r.Ended != nil { + if !last { return } + if h.Clock != nil { + for _, k := range []string{BoundPrefix + run, UntakenPrefix + run} { + if err := h.Clock.Unbound(k); err != nil { + h.logf("the bound %s of the ended test run is not deleted: %v", k, err) + } + } + } if h.DropRun != nil { if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil { h.logf("the drop of ask %s of the test run %s, which no reader is left of, is not answered: %v", r.Ask, run, err) } } - if err := h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: NoReaderLeft}, nil); err != nil { - h.logf("the test run %s, which no reader is left of, is not ended: %v", run, err) - } return } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 718e322b..6a1ce8ca 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -336,7 +336,7 @@ func (h *Hearer) endRunOf(p seat.Pipeline, s seat.StepResult, ev StepEnded) erro if run == "" || h.TestRuns == nil || s.Ask != ev.ID { return nil } - end := seat.TestRun{Outcome: ev.Outcome, Said: ev.Said, Node: ev.Node} + end := seat.TestRun{Ask: ev.ID, Outcome: ev.Outcome, Said: ev.Said, Node: ev.Node} if ev.Tests != nil { end.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration} } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go index 23cf0685..5287d4ae 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go @@ -23,8 +23,9 @@ type TestRuns interface { Get(key string) (Run, uint64, error) // Update writes key at the revision read; ErrChanged when another write came between. Update(key string, r Run, revision uint64) (uint64, error) - // Delete deletes key; a key gone is no fault. - Delete(key string) error + // Delete deletes key at the revision read; ErrChanged when another write came between, a key gone is no fault. No + // write of a run puts a value judged from an earlier read over a later one (the kind of the stale write). + Delete(key string, revision uint64) error } // Run is a key's value in test-runs (§725): the test run, its readers each marked whether a rerun's new run moves it and by diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_read.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_read.go index 0e863d17..b725c219 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_read.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_read.go @@ -124,7 +124,7 @@ func (v Verbs) TestRunsDrop(caller string, in seat.TestRunsDropInput) (seat.Test h.logf("the drop of ask %s of the dropped test run %s is not answered: %v", r.Ask, in.Key, err) } } - if err := h.EndRun(in.Key, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil); err != nil { + if err := h.EndRun(in.Key, seat.TestRun{Ask: r.Ask, Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil); err != nil { return seat.TestRunsDropAnswer{}, err } for _, o := range r.Readers { @@ -142,8 +142,24 @@ func (v Verbs) TestRunsDrop(caller string, in seat.TestRunsDropInput) (seat.Test } } } - if err := h.TestRuns.Delete(in.Key); err != nil { - return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not deleted: %w", in.Key, err) + // The key is deleted at the revision of the run dropped: a run a rerun asked under it since is not deleted with it. + for { + now, rev, err := h.TestRuns.Get(in.Key) + if errors.Is(err, ErrNoRun) { + break + } + if err != nil { + return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not read again, so it is not deleted: %w", in.Key, err) + } + if now.Ask != r.Ask { + return seat.TestRunsDropAnswer{}, fmt.Errorf("the key %s holds the run of ask %s, asked since the run of ask %s was dropped: it is not deleted", in.Key, now.Ask, r.Ask) + } + if err := h.TestRuns.Delete(in.Key, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not deleted: %w", in.Key, err) + } + break } if h.Clock != nil { for _, k := range []string{BoundPrefix + in.Key, UntakenPrefix + in.Key} { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go index 6000bd0b..cbbd531b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -15,11 +15,20 @@ import ( type clockMem struct { runs testRunsMem bound map[string]time.Duration + // before is called once at the next Bound, a write that comes between its caller's read and its write. + before func() } -func (c *clockMem) Bound(key string, ttl time.Duration) error { c.bound[key] = ttl; return nil } -func (c *clockMem) Unbound(key string) error { delete(c.bound, key); return nil } -func (c *clockMem) Has(key string) (bool, error) { _, ok := c.bound[key]; return ok, nil } +func (c *clockMem) Bound(key string, ttl time.Duration) error { + if b := c.before; b != nil { + c.before = nil + b() + } + c.bound[key] = ttl + return nil +} +func (c *clockMem) Unbound(key string) error { delete(c.bound, key); return nil } +func (c *clockMem) Has(key string) (bool, error) { _, ok := c.bound[key]; return ok, nil } func (c *clockMem) Keys() ([]string, error) { var out []string for k := range c.runs { @@ -453,3 +462,140 @@ func TestTheTestRunsAreReadAndDroppedByTheirVerbs(t *testing.T) { t.Errorf("a reader of the dropped run = %+v", s) } } + +// racingRuns is test-runs with a write coming between a caller's read and its next update or delete: each hook is called +// once, just before that write. +type racingRuns struct { + testRunsMem + beforeUpdate, beforeDelete func() +} + +func (r *racingRuns) Update(key string, v Run, rev uint64) (uint64, error) { + if b := r.beforeUpdate; b != nil { + r.beforeUpdate = nil + b() + } + return r.testRunsMem.Update(key, v, rev) +} + +func (r *racingRuns) Delete(key string, rev uint64) error { + if b := r.beforeDelete; b != nil { + r.beforeDelete = nil + b() + } + return r.testRunsMem.Delete(key, rev) +} + +// No write of test-runs or its bound keys puts back, ends or deletes a run from a read made before its latest change (design +// 47 G §725, §726, §727, §728; the kind test 109 met): each write is a compare-and-set at the revision read, retried from the +// latest read, and a judgement of a run ends only the run under the ask it judged. +func TestNoWriteOfATestRunActsOnAReadBeforeItsLatestChange(t *testing.T) { + asked := func(f *runFixture, run string) { + t.Helper() + r, rev, _ := f.runs.Get(run) + r.TestRun = seat.TestRun{Level: r.Level, Digest: r.Digest, Seat: r.Seat, Ask: "a2"} + if _, err := f.runs.Update(run, r, rev); err != nil { + t.Fatal(err) + } + } + for _, c := range []struct { + name string + race func(t *testing.T, f *runFixture, p seat.Pipeline, run string) + }{ + {"an expiry does not end a run asked again while its ask was dropped", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) { + unit := stepNamed(p, "app:program", "unit") + _ = f.h.RunTaken(unit.Ask, "ace") + f.h.DropRun = func(string, string, string, string) error { asked(f, run); return nil } + _ = f.h.RunExpired(BoundPrefix + run) + }}, + {"the last reader to leave ends the run in its own write, so a step joining then asks its own", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) { + f.h.DropRun = func(string, string, string, string) error { + _, _, _ = f.s.claimRun(Asked{Run: run, ID: "a2", Level: "unit", Seat: "node-build-go", + Readers: []seat.TestRunReader{{Pipeline: "main.5.app", Artifact: "app:program"}}}) + return nil + } + for _, a := range []string{"app:program", "app:tools"} { + f.h.leaveRun(p.Pipeline, stepNamed(p, a, "unit"), "cancelled") + } + }}, + {"the age does not delete a run asked again since it was read", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) { + _ = f.h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomePassed}, nil) + runs := &racingRuns{testRunsMem: f.runs, beforeDelete: func() { asked(f, run) }} + a := &Ager{Runs: runs, RunsKept: time.Hour, Pipelines: agedDeleter{f.store}, Aged: agedState{}, Now: func() time.Time { return agedNow }, + Log: func(from uint64, limit int) ([]LoggedEntry, error) { + if from > 1 { + return nil, nil + } + return []LoggedEntry{{Sequence: 1, Entry: seat.LogEntry{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:tools", + What: seat.LogEntryWhatEnded, At: agedNow.Add(-2 * time.Hour)}}}, nil + }} + a.Age() + }}, + {"test-runs-drop does not delete a run asked again under its key", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) { + f.h.TestRuns = &racingRuns{testRunsMem: f.runs, beforeDelete: func() { asked(f, run) }} + v := Verbs{Hearer: f.h, Starter: f.s, Acts: &actLog{}} + if _, err := v.TestRunsDrop("op@shanks", seat.TestRunsDropInput{Key: run, Why: "stuck"}); err == nil { + t.Error("a drop whose key holds a run asked since answers no error") + } + }}, + {"a step-progress heard as the run ends leaves no clock", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) { + unit := stepNamed(p, "app:program", "unit") + _ = f.h.RunTaken(unit.Ask, "ace") + f.clock.before = func() { _ = f.h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomePassed}, nil) } + _ = f.h.RunProgress(unit.Ask) + if _, ok := f.clock.bound[BoundPrefix+run]; ok { + t.Error("a clock written from a read before the run's end is kept") + } + asked(f, run) // the key's run is asked again, so the check below holds + }}, + {"a clock of a run not taken ends nothing", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) { + asked(f, run) + _ = f.h.RunExpired(BoundPrefix + run) + }}, + } { + t.Run(c.name, func(t *testing.T) { + f := newRunFixture() + p := f.start(t, 4, 9) + run := stepNamed(p, "app:program", "unit").Shared[0].Run + c.race(t, f, p, run) + if r, _, err := f.runs.Get(run); err != nil || r.Ask != "a2" || r.Ended != nil { + t.Errorf("the key's run = %+v, %v; want the run asked again under a2, running", r, err) + } + }) + } + // A rerun whose read of the key is changed before its write (a step joining the ended run) writes from the latest read, + // carrying that reader, never answering the change as an error nor putting back the earlier readers. + t.Run("a rerun writes its run from the latest read", func(t *testing.T) { + f := newRunFixture() + p := f.start(t, 4, 9) + unit := stepNamed(p, "app:program", "unit") + run := unit.Shared[0].Run + ended := time.Now() + if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask, + Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed", Ended: &ended}); err != nil { + t.Fatal(err) + } + f.h.TestRuns = &racingRuns{testRunsMem: f.runs, beforeUpdate: func() { + r, rev, _ := f.runs.Get(run) + r.Readers = append(r.Readers, RunReader{Pipeline: "main.9.app", Artifact: "app:program"}) + _, _ = f.runs.Update(run, r, rev) + }} + v := Verbs{Hearer: f.h, Starter: f.s, View: func(string) (Half, bool) { + return Half{Entries: map[string]catalog.CatalogEntry{ + "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, + "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}, true + }, Acts: &actLog{}} + a, err := v.Rerun("op@shanks", seat.RerunInput{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:program", Why: "fixed"}) + if err != nil { + t.Fatalf("a rerun whose read was changed before its write: %v", err) + } + r, _, _ := f.runs.Get(run) + joined := false + for _, o := range r.Readers { + joined = joined || o.Pipeline == "main.9.app" + } + if r.Ask != a.Result.Ask || r.Ended != nil || !joined || len(r.Readers) != 3 { + t.Errorf("the rerun's run = %+v; want the new ask running, carrying the reader that joined", r) + } + }) +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go index 4626d260..5ca58405 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -392,29 +392,50 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult) v.Starter.joinRun(run, reader) default: now := h.now() - value := Run{TestRun: seat.TestRun{Level: st.Step, Digest: r.Digest, Seat: st.Seat, Ask: ask.ID, Started: &now}} - // Every former reader is carried, each that did not pass marked to move with the new run by this caller (A-470d). - for _, o := range r.Readers { - if o.Of() == reader { - continue - } - o.Rerun, o.By = !v.passedReader(o, run), "" - if o.Rerun { - o.By = caller - } - value.Readers = append(value.Readers, o) - } - addReaders(&value, []seat.TestRunReader{reader}) a := *ask a.Run = run - if err := v.Starter.started(a, func() error { - if !exists { - return h.TestRuns.Create(run, value) + // The new run is written from the latest read of the key: a write between (a reader leaving, a join) is read again and + // its readers carried, never put back; a run another step started since is said, not replaced. + for { + value := Run{TestRun: seat.TestRun{Level: st.Step, Digest: r.Digest, Seat: st.Seat, Ask: ask.ID, Started: &now}} + // Every former reader is carried, each that did not pass marked to move with the new run by this caller (A-470d). + for _, o := range r.Readers { + if o.Of() == reader { + continue + } + o.Rerun, o.By = !v.passedReader(o, run), "" + if o.Rerun { + o.By = caller + } + value.Readers = append(value.Readers, o) } - _, err := h.TestRuns.Update(run, value, rev) - return err - }); err != nil { - return seat.RerunAnswer{}, fmt.Errorf("the step is written running but its test run %s is not: %w", run, err) + addReaders(&value, []seat.TestRunReader{reader}) + err := v.Starter.started(a, func() error { + if !exists { + return h.TestRuns.Create(run, value) + } + _, err := h.TestRuns.Update(run, value, rev) + return err + }) + if errors.Is(err, ErrChanged) || errors.Is(err, ErrExists) { + got, gotRev, gerr := h.TestRuns.Get(run) + switch { + case errors.Is(gerr, ErrNoRun): + r, rev, exists = Run{TestRun: seat.TestRun{Digest: r.Digest}}, 0, false + continue + case gerr != nil: + err = gerr + case got.Ended == nil: + err = fmt.Errorf("the run of ask %s was asked under the key since it was read", got.Ask) + default: + r, rev, exists = got, gotRev, true + continue + } + } + if err != nil { + return seat.RerunAnswer{}, fmt.Errorf("the step is written running but its test run %s is not: %w", run, err) + } + break } v.Starter.carry(in.Pipeline, []Asked{*ask}) if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil { From ab9d9604b40fa9f4b81c4ed69720332386d5ff5d Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 12:28:26 +0100 Subject: [PATCH 13/14] runner: a run's bound holds its ask, armed by compare-and-set, and its end is written before the step's finding (A-470n, A-470o) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A bound was armed by a delete and a create with no revision, so a late ask-taken or step-progress of a replaced run could arm a clock for the run asked next, an expiry heard after the bound was armed again ended the run, and a step-ended's run end waited behind its finding's event (3 s in test 109, past its bound). A bound key now holds the run's ask and is published at the key's latest revision with its own time to live, read against the run after it is written and deleted at its own revision when the run no longer names that ask; a late signal moves nothing and is journaled; an expiry ends the run only while its bound is still absent when read again; the run's end is written as its step-ended is heard, before the finding. Builds: design 47 G §720, §725, §726; A-470n, A-470o --- .../cmd/mesh-delivery-pipeline/testruns.go | 70 +++++++++++++--- .../internal/pipeline/runends.go | 54 +++++++++---- .../internal/pipeline/steps.go | 5 +- .../internal/pipeline/testruns.go | 2 +- .../internal/pipeline/testruns_test.go | 79 ++++++++++++++++++- 5 files changed, 178 insertions(+), 32 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go index da948880..22a724d1 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go @@ -31,6 +31,7 @@ const TestRunsState = "test-runs" // time to live (A-77, A-470). type testRunsKV struct { ctx context.Context + js jetstream.JetStream kv jetstream.KeyValue } @@ -93,21 +94,70 @@ func (t testRunsKV) deleteBound(key string) error { return nil } -// Bound writes key again with its time to live: deleted, then created with ttl, so its expiry is heard only ttl after -// the last write (a delete is no expiry). -func (t testRunsKV) Bound(key string, ttl time.Duration) error { - if err := t.deleteBound(key); err != nil { - return err +// boundValue is a bound key's value: the key and the ask of the run it bounds (A-470n). +type boundValue struct { + Bound string `json:"bound"` + Ask string `json:"ask"` +} + +// Bound writes key again holding ask, with ttl as its own time to live, published at the key's latest revision (a delete +// marker counted), so a write between is read again and never overwritten unseen (A-470n); the earlier message, replaced, +// takes its time to live with it, so the expiry is heard only ttl after the last write. +func (t testRunsKV) Bound(key, ask string, ttl time.Duration) (uint64, error) { + raw, err := json.Marshal(boundValue{Bound: key, Ask: ask}) + if err != nil { + return 0, err } - _, err := t.kv.Create(t.ctx, key, []byte(`{"bound":"`+key+`"}`), jetstream.KeyTTL(ttl)) - if errors.Is(err, jetstream.ErrKeyExists) { + subject := "$KV." + t.kv.Bucket() + "." + key + for { + var last uint64 + st, err := t.js.Stream(t.ctx, "KV_"+t.kv.Bucket()) + if err != nil { + return 0, err + } + m, err := st.GetLastMsgForSubject(t.ctx, subject) + switch { + case err == nil: + last = m.Sequence + case !errors.Is(err, jetstream.ErrMsgNotFound): + return 0, err + } + ack, err := t.js.Publish(t.ctx, subject, raw, jetstream.WithMsgTTL(ttl), jetstream.WithExpectLastSequencePerSubject(last)) + if err != nil && strings.Contains(err.Error(), "wrong last sequence") { + continue + } + if err != nil { + return 0, err + } + return ack.Sequence, nil + } +} + +// Bounded reads a bound key: the ask it holds and its revision. +func (t testRunsKV) Bounded(key string) (string, uint64, bool, error) { + e, err := t.kv.Get(t.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return "", 0, false, nil + } + if err != nil { + return "", 0, false, err + } + var v boundValue + _ = json.Unmarshal(e.Value(), &v) + return v.Ask, e.Revision(), true, nil +} + +func (t testRunsKV) Unbound(key string) error { return t.deleteBound(key) } + +// UnboundAt deletes a bound key at revision: a later write of it is kept. +func (t testRunsKV) UnboundAt(key string, revision uint64) error { + err := t.kv.Delete(t.ctx, key, jetstream.LastRevision(revision)) + if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence") || errors.Is(err, jetstream.ErrKeyNotFound)) { return nil } return err } -func (t testRunsKV) Unbound(key string) error { return t.deleteBound(key) } - func (t testRunsKV) Has(key string) (bool, error) { _, err := t.kv.Get(t.ctx, key) if errors.Is(err, jetstream.ErrKeyNotFound) { @@ -351,7 +401,7 @@ func openTestRuns(ctx context.Context, js jetstream.JetStream, cred process.Acco if err != nil { return nil, nil, err } - runs := testRunsKV{ctx: ctx, kv: kv} + runs := testRunsKV{ctx: ctx, js: js, kv: kv} objects := &runObjects{ctx: ctx, js: js, module: cred.Module} starter.TestRuns, starter.Clock, starter.Runs = runs, runs, settings starter.RunsFill = runsFill(ctx, js, cred.Module, time.Now) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go index d694bbb2..e686f14f 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go @@ -38,12 +38,18 @@ type RunObjects interface { Delete(name string) error } -// RunClock is the bound keys of the test runs, keys of test-runs with their own time to live. +// RunClock is the bound keys of the test runs, keys of test-runs with their own time to live, each holding the ask of the +// run it bounds (A-470n). type RunClock interface { - // Bound writes key again with ttl as its time to live. - Bound(key string, ttl time.Duration) error + // Bound writes key again holding ask, with ttl as its time to live, by compare-and-set on the key's latest revision + // (retried when another write came between); it answers the revision written. + Bound(key, ask string, ttl time.Duration) (uint64, error) + // Bounded reads key: the ask it holds and its revision; ok false when it is not written. + Bounded(key string) (ask string, revision uint64, ok bool, err error) // Unbound deletes key; a key gone is no fault. Unbound(key string) error + // UnboundAt deletes key at revision; a later write of it (another run's bound) is kept, and a key gone is no fault. + UnboundAt(key string, revision uint64) error // Has says whether key is written. Has(key string) (bool, error) // Keys are test-runs' keys. @@ -106,6 +112,7 @@ func (h *Hearer) RunTaken(ask, node string) error { return fmt.Errorf("the test run %s is not read: %w", run, err) } if r.Ask != ask || r.Ended != nil { + h.logf("the ask-taken of ask %s is late: the test run %s names ask %s (ended %v), so it moves nothing (A-470n)", ask, run, r.Ask, r.Ended != nil) return nil } if r.Node != node { @@ -119,25 +126,31 @@ func (h *Hearer) RunTaken(ask, node string) error { h.runStarted(run, r) break } - if err := h.bound(run, ask); err != nil { + if err := armBound(h.Clock, h.TestRuns, BoundPrefix+run, run, ask, h.runSettings().StepBound); err != nil { return fmt.Errorf("the clock of the test run %s is not begun: %w", run, err) } - return h.Clock.Unbound(UntakenPrefix + run) -} - -// bound writes the clock of the run under ask, judged from a read of the run, then reads the run again: a run ended since, -// or asked again and not taken, has its clock deleted, so no bound written from the earlier read ends the run after it. -func (h *Hearer) bound(run, ask string) error { - if err := h.Clock.Bound(BoundPrefix+run, h.runSettings().StepBound); err != nil { + // The untaken bound is deleted only while it holds this ask: a rerun's own untaken bound is kept. + held, rev, ok, err := h.Clock.Bounded(UntakenPrefix + run) + if err != nil || !ok || held != ask { return err } - r, _, err := h.TestRuns.Get(run) + return h.Clock.UnboundAt(UntakenPrefix+run, rev) +} + +// armBound writes the bound key of the run under ask holding that ask, judged from a read of the run, then reads the run +// again (A-470n): when it no longer names ask, or has ended, the bound just written is deleted at its revision, so no +// bound written from the earlier read ends the run after it, and a bound another run wrote since is kept. +func armBound(clock RunClock, runs TestRuns, key, run, ask string, ttl time.Duration) error { + rev, err := clock.Bound(key, ask, ttl) + if err != nil { + return err + } + r, _, err := runs.Get(run) if err != nil && !errors.Is(err, ErrNoRun) { return err } - // A run asked again and taken since keeps the clock: it is its own as well (its ask-taken writes it). - if errors.Is(err, ErrNoRun) || r.Ended != nil || (r.Ask != ask && r.Node == "") { - return h.Clock.Unbound(BoundPrefix + run) + if errors.Is(err, ErrNoRun) || r.Ended != nil || r.Ask != ask { + return clock.UnboundAt(key, rev) } return nil } @@ -151,12 +164,13 @@ func (h *Hearer) RunProgress(ask string) error { } r, _, err := h.TestRuns.Get(run) if errors.Is(err, ErrNoRun) || (err == nil && (r.Ask != ask || r.Ended != nil)) { + h.logf("the step-progress of ask %s is late: the test run %s names ask %q (ended %v), so it moves nothing (A-470n)", ask, run, r.Ask, r.Ended != nil) return nil } if err != nil { return fmt.Errorf("the test run %s is not read: %w", run, err) } - return h.bound(run, ask) + return armBound(h.Clock, h.TestRuns, BoundPrefix+run, run, ask, h.runSettings().StepBound) } // RunExpired takes the expiry of a test run's bound key (§725, §720 *When the clock ends*, A-466s): the run ends `could not @@ -178,6 +192,14 @@ func (h *Hearer) RunExpired(key string) error { if r.Ended != nil || (untaken && r.Node != "") || (!untaken && r.Node == "") { return nil } + // The marker of an expiry holds no value (A-470o): the run ends only while its bound is still absent when read again, + // so an expiry heard after the bound was armed again for the run's own ask ends nothing. + if _, _, armed, err := h.Clock.Bounded(key); err != nil { + return fmt.Errorf("the bound %s is not read again: %w", key, err) + } else if armed { + h.logf("the expiry of %s ends nothing: the bound is armed again (A-470o)", key) + return nil + } s := h.runSettings() why := fmt.Sprintf("the ask %s of %s was taken by %s and said no progress within %s (test-step-bound)", r.Ask, r.Seat, r.Node, s.StepBound) if untaken { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 6a1ce8ca..63caa13f 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -252,6 +252,9 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh } else if err != nil { return p, fmt.Errorf("pipeline %s cannot be written: %w", ev.Pipeline, err) } + // The run's end is written as its step-ended is heard, before any finding the step says, so no wait elsewhere (a + // finding's event the bus is slow to take) delays it past its bound (A-470o). + runErr := h.endRunOf(p, *s, ev) if h.Keep != nil { for _, k := range ev.Kept { kept := seat.KeptResult{Digest: k.Digest, Level: k.Level, Package: k.Package, Pipeline: k.Pipeline, LastRead: &now} @@ -282,7 +285,7 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh if err := h.said(p, asked, ended); err != nil { return p, err } - return p, h.endRunOf(p, *s, ev) + return p, runErr } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go index 5287d4ae..494a2cb7 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go @@ -255,7 +255,7 @@ func (s *Starter) started(a Asked, write func() error) error { if untaken <= 0 { untaken = DefaultRunSettings.Untaken } - if err := s.Clock.Bound(UntakenPrefix+a.Run, untaken); err != nil { + if err := armBound(s.Clock, s.TestRuns, UntakenPrefix+a.Run, a.Run, a.ID, untaken); err != nil { s.logf("the untaken bound of the test run %s is not written: %v", a.Run, err) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go index cbbd531b..ad9470ad 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -11,23 +11,41 @@ import ( nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" ) -// clockMem is the bound keys of test-runs in memory, each with the time to live it was last written with. +// clockMem is the bound keys of test-runs in memory, each with the time to live it was last written with, the ask it holds +// and its revision. type clockMem struct { runs testRunsMem bound map[string]time.Duration + asks map[string]string + revs map[string]uint64 + rev uint64 // before is called once at the next Bound, a write that comes between its caller's read and its write. before func() } -func (c *clockMem) Bound(key string, ttl time.Duration) error { +func (c *clockMem) Bound(key, ask string, ttl time.Duration) (uint64, error) { if b := c.before; b != nil { c.before = nil b() } - c.bound[key] = ttl + if c.asks == nil { + c.asks, c.revs = map[string]string{}, map[string]uint64{} + } + c.rev++ + c.bound[key], c.asks[key], c.revs[key] = ttl, ask, c.rev + return c.rev, nil +} +func (c *clockMem) Bounded(key string) (string, uint64, bool, error) { + _, ok := c.bound[key] + return c.asks[key], c.revs[key], ok, nil +} +func (c *clockMem) Unbound(key string) error { delete(c.bound, key); return nil } +func (c *clockMem) UnboundAt(key string, rev uint64) error { + if c.revs[key] == rev { + delete(c.bound, key) + } return nil } -func (c *clockMem) Unbound(key string) error { delete(c.bound, key); return nil } func (c *clockMem) Has(key string) (bool, error) { _, ok := c.bound[key]; return ok, nil } func (c *clockMem) Keys() ([]string, error) { var out []string @@ -156,6 +174,14 @@ func TestATestRunHasOneClockACapAndAnAge(t *testing.T) { if r, _, _ := f.runs.Get(run); r.Ended != nil { t.Fatal("a run is ended by its untaken bound after it was taken") } + // An expiry heard once the bound is armed again ends nothing (A-470o); the bound holds the run's ask (A-470n). + if err := f.h.RunExpired(BoundPrefix + run); err != nil { + t.Fatal(err) + } + if r, _, _ := f.runs.Get(run); r.Ended != nil || f.clock.asks[BoundPrefix+run] != unit.Ask { + t.Fatalf("an expiry of a bound armed again ended the run %+v, or the bound holds ask %q", r, f.clock.asks[BoundPrefix+run]) + } + delete(f.clock.bound, BoundPrefix+run) // the bus's expiry deletes it if err := f.h.RunExpired(BoundPrefix + run); err != nil { t.Fatal(err) } @@ -506,6 +532,7 @@ func TestNoWriteOfATestRunActsOnAReadBeforeItsLatestChange(t *testing.T) { unit := stepNamed(p, "app:program", "unit") _ = f.h.RunTaken(unit.Ask, "ace") f.h.DropRun = func(string, string, string, string) error { asked(f, run); return nil } + delete(f.clock.bound, BoundPrefix+run) // the bus's expiry deletes it _ = f.h.RunExpired(BoundPrefix + run) }}, {"the last reader to leave ends the run in its own write, so a step joining then asks its own", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) { @@ -563,6 +590,50 @@ func TestNoWriteOfATestRunActsOnAReadBeforeItsLatestChange(t *testing.T) { } }) } + // A late signal of a replaced run arms no bound and is said in the journal (A-470n). + t.Run("a late ask-taken or step-progress of a replaced run moves nothing", func(t *testing.T) { + f := newRunFixture() + p := f.start(t, 4, 9) + unit := stepNamed(p, "app:program", "unit") + run := unit.Shared[0].Run + asked(f, run) + delete(f.clock.bound, UntakenPrefix+run) + _ = f.h.RunTaken(unit.Ask, "ace") + _ = f.h.RunProgress(unit.Ask) + late := 0 + for _, l := range f.logged { + if strings.Contains(l, unit.Ask) && strings.Contains(l, "moves nothing") { + late++ + } + } + if _, ok := f.clock.bound[BoundPrefix+run]; ok || late != 2 { + t.Errorf("bounds %v, journal %q; want no clock and both late signals journaled", f.clock.bound, f.logged) + } + }) + // A run's end is written as its step-ended is heard, before the finding the step says (A-470o). + t.Run("the run ends before the step's finding is written", func(t *testing.T) { + f := newRunFixture() + reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Policy: catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "required", + Integration: "bypass", Composition: "bypass", Review: "bypass", Approval: "bypass"}}} + f.s.Registration = func(string) (catalog.Registration, bool) { return reg, true } + p := f.start(t, 4, 9) + unit := stepNamed(p, "app:program", "unit") + run := unit.Shared[0].Run + endedFirst := false + f.h.Failures = &Failures{Findings: newHeld(), Author: func(string, string) (string, error) { + r, _, _ := f.runs.Get(run) + endedFirst = r.Ended != nil + return "alice", nil + }} + ended := time.Now() + if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask, + Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed", Ended: &ended}); err != nil { + t.Fatal(err) + } + if !endedFirst { + t.Error("the step's finding was written before its run's end") + } + }) // A rerun whose read of the key is changed before its write (a step joining the ended run) writes from the latest read, // carrying that reader, never answering the change as an error nor putting back the earlier readers. t.Run("a rerun writes its run from the latest read", func(t *testing.T) { From a96f873e2146df0bba80cc800ee3e2ed414cf9ee Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 12:30:19 +0100 Subject: [PATCH 14/14] runner: test 109, the test runs whole against the real bus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runner whole against an embedded bus: one unit and one integration run per level read by both artifacts, a failing run counted once, a rerun moving the other reader and not the accepted one, a restart's take-up, progress past the bound, a replaced reader, a silent ask ending could not run with its readers and its ask dropped, a second revision's own runs, a seat with no holder failing its pipeline at once with its other steps not asked, test-runs and test-runs-drop over the bus, and the cap. The finding writer's stream is hosted, as on the mesh. Builds: design 47 G §733, §724, §725, §726, §727, §728, §535, §647; A-468f, A-470f, A-470g, A-470h, A-470i, A-470n --- .../testruns_integration_test.go | 536 ++++++++++++++++++ 1 file changed, 536 insertions(+) create mode 100644 modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go new file mode 100644 index 00000000..6ec49142 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go @@ -0,0 +1,536 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" + catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/seatlist" + "git.novox.be/novox/mesh-sdk/go/testbus" + "git.novox.be/novox/mesh-sdk/go/views" + "git.novox.be/novox/mesh-sdk/go/views/viewstest" + + "mesh-delivery-pipeline/internal/pipeline" +) + +// Test 109 (design 47 G §733 *Whole (runner…)*, §724 to §728, §535 and §647 as A-466y and A-466z amend them; A-468f: in +// mesh-delivery with a fixture build seat holder; A-470f: no reader-only refusal). The runner whole against the real bus: a +// module whose program and tools bundle test one directory asks one unit and one integration run, both artifacts' steps +// ending with its result and shared; a failing test fails once and is counted once; a rerun of one artifact's failed step +// that passes moves the other's, the pipeline ending again, and an accepted step does not move; a runner restarted during a +// run takes it up by its ask, its clock holding that ask (A-470n); a test run waiting with a holder present is not failed; +// a replaced reader leaves the ask to the other readers; a run that keeps saying its progress past its bound is not ended, +// and a taken ask silent for its bound ends could not run with its readers, its ask dropped; a second revision with the +// same digests asks its own runs; a seat the seat-holders view names no holder for ends its pipeline failed at once, its +// other steps not asked (A-470g, A-470h); the process answers test-runs with its fill against test-runs-cap-mib and +// test-runs-drop deletes the key and its output object (A-470i); at the cap a step ends could not run, naming the fill. +// The step bound is two seconds here, as the setting test-step-bound names it, where the mesh's is 30 minutes. +func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { + url := testbus.URL(t) + nc, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + const module = "mesh-delivery-pipeline" + for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries"} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}); err != nil { + t.Fatal(err) + } + } + for _, state := range []string{"bounds", TestRunsState} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1, LimitMarkerTTL: time.Minute}); err != nil { + t.Fatal(err) + } + } + // test-runs' own object store, made with the state (A-469e). + if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: catalogue.BusName(module, catalogue.StateObjectsName(TestRunsState))}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil { + t.Fatal(err) + } + events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>", "mesh.stream.*.event.>"}}) + if err != nil { + t.Fatal(err) + } + if _, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: "n1_" + module, AckPolicy: jetstream.AckExplicitPolicy}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy}); err != nil { + t.Fatal(err) + } + + // The seat-holders view of the seat mesh-module-assigner's claimant (§647, A-466z), served as its owner does: node-build-go + // has a holder, node-build-node none. + holders := viewstest.Serve(t, ctx, js, nc, "assigner", "seat-holders", "assigner.seat-holders", "n1_"+module) + if _, err := holders.Put(ctx, "node-build-go", []byte(`{"holders":[{"module":"mesh-build-go"}]}`)); err != nil { + t.Fatal(err) + } + + // The registrations: app tests its program and tools bundle from one directory, its unit step required and its + // integration step optional; web is built by node-build-node. + policy := catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "required", Integration: "optional", Composition: "bypass", Review: "bypass", Approval: "bypass"}} + regs := []catalog.Registration{{Module: "app", Branches: []string{"main"}, Origin: "o/app", Policy: policy}, + {Module: "web", Branches: []string{"main"}, Origin: "o/web", Policy: policy}} + answer := func(subject string, body any) { + if _, err := nc.Subscribe(subject, func(m *nats.Msg) { + raw, _ := json.Marshal(map[string]any{"result": body}) + _ = m.Respond(raw) + }); err != nil { + t.Fatal(err) + } + } + answer("mesh.seat.mesh-catalog.call.registrations.*", catalog.RegistrationsAnswer{Registrations: regs}) + answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{}) + entries := []catalog.CatalogEntry{ + {Module: "app", Commit: "c1", Fp: "src2:aa", Path: "modules/app", Repository: "o/app", + Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, + {Module: "web", Commit: "w1", Fp: "src2:ww", Path: "modules/web", Repository: "o/web", Artifacts: []catalog.Artifact{{Ref: "web:program", Language: "node"}}}, + {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, + {Module: "mesh-build-node", Acts: []catalog.ActingRule{{Seat: "node-build-node", Step: "build", On: "node"}}}, + {Module: "assigner", Manifest: map[string]any{"module": "assigner", "version": "0.1", "claims": []any{map[string]any{"name": seatlist.HolderSeat, "scope": "mesh"}}}}, + } + var revision atomic.Int64 + revision.Store(1) + if _, err := nc.Subscribe("mesh.seat.mesh-catalog.call.revision.*", func(m *nats.Msg) { + r := revision.Load() + raw, _ := json.Marshal(map[string]any{"result": catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: r, Kind: "trunk", Branch: "main", Source: 2 + r}, Entries: entries}}) + _ = m.Respond(raw) + }); err != nil { + t.Fatal(err) + } + def, err := os.ReadFile("../../../mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json") + if err != nil { + t.Fatal(err) + } + if _, err := nc.Subscribe("mesh.seat.mesh-catalog.call."+seatlist.Verb+".*", func(m *nats.Msg) { + var in struct { + Seat string `json:"seat"` + } + _ = json.Unmarshal(m.Data, &in) + entry := seatlist.Entry{Seat: seat.Seat, Source: seatlist.SourceTrunk, Digest: "sha256:runner"} + var result any = seatlist.Listing{Revision: 1, Seats: []seatlist.Entry{entry}} + if in.Seat != "" { + entry.Definition = def + result = seatlist.Answer{Seat: in.Seat, Found: in.Seat == seat.Seat, Entry: &entry, Revision: 1} + } + raw, _ := json.Marshal(map[string]any{"result": result}) + _ = m.Respond(raw) + }); err != nil { + t.Fatal(err) + } + // Every pipeline-ended said, and every drop asked of a build seat. + var mu sync.Mutex + var endings []seat.PipelineEnded + var drops []string + if _, err := nc.Subscribe("mesh.seat.mesh-pipeline-runner.event.pipeline-ended.>", func(m *nats.Msg) { + var e seat.PipelineEnded + if json.Unmarshal(m.Data, &e) == nil { + mu.Lock() + endings = append(endings, e) + mu.Unlock() + } + }); err != nil { + t.Fatal(err) + } + if _, err := nc.Subscribe("mesh.seat.*.call.drop.>", func(m *nats.Msg) { + mu.Lock() + drops = append(drops, string(m.Data)) + mu.Unlock() + raw, _ := json.Marshal(map[string]any{"result": map[string]any{}}) + _ = m.Respond(raw) + }); err != nil { + t.Fatal(err) + } + ended := func(key string) []seat.PipelineEnded { + mu.Lock() + defer mu.Unlock() + var out []seat.PipelineEnded + for _, e := range endings { + if e.Pipeline == key { + out = append(out, e) + } + } + return out + } + + // The settings, as the manifest's resources write them (A-469x): a cap of 1 MiB, a step bound of two seconds. + dir := t.TempDir() + files := map[string]string{envRunsCap: "1", envStepBound: "2s"} + env := map[string]string{} + for name, content := range files { + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + env[name] = path + } + broker := filepath.Join(dir, "broker") + raw, _ := json.Marshal(process.Account{URL: url, Node: "n1", Module: module, User: "runner", Password: "x"}) + if err := os.WriteFile(broker, raw, 0o600); err != nil { + t.Fatal(err) + } + env[process.BrokerFileEnv] = broker + logs := &syncBuffer{} + start := func() func() { + runCtx, stop := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { done <- run(runCtx, func(k string) string { return env[k] }, logs) }() + return func() { stop(); <-done } + } + stop := start() + defer func() { stop() }() + + pipelines, err := js.KeyValue(ctx, module+"_pipelines") + if err != nil { + t.Fatal(err) + } + runsKV, err := js.KeyValue(ctx, module+"_"+TestRunsState) + if err != nil { + t.Fatal(err) + } + firstViews, err := js.KeyValue(ctx, module+"_"+views.StatusState) + if err != nil { + t.Fatal(err) + } + // The seat-holders view's first view is read before any pipeline is placed, so a seat is judged from it (A-466y). + waitFor(t, ctx, logs, func() bool { + e, err := firstViews.Get(ctx, views.StatusKey("assigner", "seat-holders")) + var s views.Status + return err == nil && json.Unmarshal(e.Value(), &s) == nil && !s.Answered.IsZero() + }) + + takeIn := func(module, commit, fp string, source, rev int64) { + taken, _ := json.Marshal(catalog.TakeInTaken{Key: "m." + commit, Kind: "taken", Branch: "main", Source: source, Revision: rev, + Entries: []catalog.TakeInEntry{{Module: module, Commit: commit, Fingerprint: fp}}}) + if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", taken); err != nil { + t.Fatal(err) + } + } + placed := func(key string) seat.Pipeline { + var p seat.Pipeline + waitFor(t, ctx, logs, func() bool { return readPipeline(ctx, pipelines, key, &p) && len(p.Steps) > 1 }) + return p + } + read := func(key string) seat.Pipeline { + var p seat.Pipeline + readPipeline(ctx, pipelines, key, &p) + return p + } + step := func(p seat.Pipeline, artifact, name string) seat.StepResult { + for _, s := range p.Steps { + if s.Artifact == artifact && s.Step == name { + return s + } + } + return seat.StepResult{} + } + runOf := func(key string) (pipeline.Run, bool) { + e, err := runsKV.Get(ctx, key) + var r pipeline.Run + return r, err == nil && json.Unmarshal(e.Value(), &r) == nil + } + runKeys := func(candidate string) []string { + ks, _ := runsKV.Keys(ctx) + var out []string + for _, k := range ks { + if !pipeline.IsRunBound(k) && strings.HasPrefix(k, candidate+".") { + out = append(out, k) + } + } + return out + } + runOfStep := func(key, artifact, name string) (string, pipeline.Run) { + var run string + var r pipeline.Run + waitFor(t, ctx, logs, func() bool { + s := step(read(key), artifact, name) + if len(s.Shared) == 0 { + return false + } + run = s.Shared[0].Run + got, ok := runOf(run) + r = got + return ok && r.Ask != "" + }) + return run, r + } + publish := func(subject string, body any) { + raw, _ := json.Marshal(body) + if _, err := js.Publish(ctx, subject, raw); err != nil { + t.Fatal(err) + } + } + taken := func(ask string) { + publish(nodebuildgo.SubjectAskTaken("n1"), nodebuildgo.AskTaken{Ask: ask, Asker: module, Node: "n1", At: time.Now()}) + } + endAsk := func(key, ask, name, outcome string, tests *nodebuildgo.TestCounts) { + now := time.Now() + publish(nodebuildgo.SubjectStepEnded("n1"), nodebuildgo.StepEnded{ID: ask, Ask: ask, Asker: module, Pipeline: key, Module: "app", Commit: "c1", + Fingerprint: "src2:aa", Step: name, Node: "n1", Outcome: outcome, Tests: tests, At: now, Ended: &now}) + } + passBuild := func(key, module, commit, fp string) { + ask := step(read(key), "", seat.StepResultStepBuild).Ask + publish("mesh.seat.mesh-builds.event.step-ended.n1", builds.StepEnded{ID: ask, Pipeline: key, Step: "build", Node: "n1", Outcome: "passed", + Entries: []builds.BuildEntry{{Module: module, Commit: commit, Fingerprint: fp}}}) + } + call := func(verb string, in, out any) string { + var reply struct { + Result json.RawMessage `json:"result"` + Error string `json:"error"` + } + waitFor(t, ctx, logs, func() bool { + m, err := nc.Request("mesh.seat.mesh-pipeline-runner.call."+verb+".agent~dev", mustJSON(in), 2*time.Second) + reply.Error = "" + return err == nil && json.Unmarshal(m.Data, &reply) == nil && !strings.Contains(reply.Error, "not yet read") + }) + if out != nil && reply.Error == "" { + _ = json.Unmarshal(reply.Result, out) + } + return reply.Error + } + + // One unit run and one integration run for both artifacts (§724), each read by both, waiting for a build node until its + // ask is taken (A-469b), its holder present and nothing failing it (A-466y). + const p3 = "main.3.app" + takeIn("app", "c1", "src2:aa", 3, 1) + passBuild(p3, "app", "c1", "src2:aa") + unitRun, unit := runOfStep(p3, "app:program", "unit") + integRun, integ := runOfStep(p3, "app:program", "integration") + if got := runKeys("main.1"); len(got) != 2 || len(unit.Readers) != 2 || len(integ.Readers) != 2 { + t.Fatalf("runs %v, unit readers %+v, integration readers %+v; want one run per level read by both artifacts", got, unit.Readers, integ.Readers) + } + if s := step(read(p3), "app:tools", "unit"); s.Outcome != seat.StepResultOutcomeWaitingForABuildNode || s.Ask != "" || s.Shared[0].Ask != unit.Ask { + t.Fatalf("the tools' unit step = %+v; want it waiting for a build node, reading the program's run", s) + } + + // A runner restarted during a run takes it up again by its ask (§725): the run's end, heard after the restart, ends both. + taken(unit.Ask) + waitFor(t, ctx, logs, func() bool { r, _ := runOf(unitRun); return r.Node == "n1" }) + // Its clock holds the run's ask (A-470n). + waitFor(t, ctx, logs, func() bool { + e, err := runsKV.Get(ctx, pipeline.BoundPrefix+unitRun) + return err == nil && strings.Contains(string(e.Value()), `"ask":"`+unit.Ask+`"`) + }) + stop() + stop = start() + // It keeps saying its progress past its bound and is not ended (§725, §720). + // Each progress is heard before the next is said: the run's clock is written again (its bound key's revision rises). + boundRev := func() uint64 { + e, err := runsKV.Get(ctx, pipeline.BoundPrefix+unitRun) + if err != nil { + return 0 + } + return e.Revision() + } + for at := time.Now(); time.Since(at) < 3*time.Second; { + before := boundRev() + publish(nodebuildgo.SubjectStepProgress("n1"), nodebuildgo.StepProgress{Ask: unit.Ask, Asker: module, Phase: "run", At: time.Now()}) + waitFor(t, ctx, logs, func() bool { return boundRev() > before }) + } + if r, _ := runOf(unitRun); r.Ended != nil { + t.Fatalf("a run saying its progress past its bound ended: %+v", r) + } + // A failing test fails once and is counted once (§726). + endAsk(p3, unit.Ask, "unit", "failed", &nodebuildgo.TestCounts{Unit: 5}) + taken(integ.Ask) + endAsk(p3, integ.Ask, "integration", "failed", &nodebuildgo.TestCounts{Integration: 2}) + waitFor(t, ctx, logs, func() bool { return len(ended(p3)) == 1 }) + p := read(p3) + for _, a := range []string{"app:program", "app:tools"} { + if s := step(p, a, "unit"); s.Outcome != seat.StepResultOutcomeFailed || len(s.Shared) == 0 || s.Shared[0].Run != unitRun { + t.Errorf("%s's unit step = %+v; want failed with the run in shared", a, s) + } + } + for _, k := range []string{unitRun, integRun} { + if r, _ := runOf(k); r.Ended == nil || r.Outcome != seat.StepResultOutcomeFailed { + t.Errorf("the run %s at its end = %+v; want it ended failed", k, r) + } + } + if e := ended(p3)[0]; e.Outcome != seat.PipelineEndedOutcomeFailed || e.Tests == nil || e.Tests.Unit != 5 || e.Tests.Integration != 2 { + t.Fatalf("pipeline-ended = %+v; want failed, the tests counted once", e) + } + + // An accepted step does not move; a rerun of the program's unit step that passes moves the tools' to passed, and the + // pipeline says pipeline-ended again (§727, A-470d). + if e := call(seat.VerbAccept, seat.AcceptInput{Pipeline: p3, Step: "integration", Artifact: "app:program", Why: "flaky"}, nil); e != "" { + t.Fatalf("accept: %s", e) + } + if e := call(seat.VerbRerun, seat.RerunInput{Pipeline: p3, Step: "unit", Artifact: "app:program", Why: "fixed"}, nil); e != "" { + t.Fatalf("rerun: %s", e) + } + var again pipeline.Run + waitFor(t, ctx, logs, func() bool { r, ok := runOf(unitRun); again = r; return ok && r.Ended == nil && r.Ask != unit.Ask }) + taken(again.Ask) + endAsk(p3, again.Ask, "unit", "passed", &nodebuildgo.TestCounts{Unit: 5}) + if e := call(seat.VerbRerun, seat.RerunInput{Pipeline: p3, Step: "integration", Artifact: "app:tools", Why: "fixed"}, nil); e != "" { + t.Fatalf("rerun: %s", e) + } + var integAgain pipeline.Run + waitFor(t, ctx, logs, func() bool { + r, ok := runOf(integRun) + integAgain = r + return ok && r.Ended == nil && r.Ask != integ.Ask + }) + taken(integAgain.Ask) + endAsk(p3, integAgain.Ask, "integration", "passed", nil) + waitFor(t, ctx, logs, func() bool { + e := ended(p3) + return len(e) >= 2 && e[len(e)-1].Outcome == seat.PipelineEndedOutcomePassed + }) + p = read(p3) + if s := step(p, "app:tools", "unit"); s.Outcome != seat.StepResultOutcomePassed || !s.Rerun { + t.Errorf("the tools' unit step = %+v; want moved to passed, marked a rerun", s) + } + if s := step(p, "app:program", "integration"); s.Outcome != seat.StepResultOutcomeAccepted { + t.Errorf("the accepted step = %+v; want it accepted still", s) + } + for _, k := range []string{unitRun, integRun} { + if r, _ := runOf(k); r.Ended == nil || r.Outcome != seat.StepResultOutcomePassed { + t.Errorf("the run %s after its rerun = %+v; want it ended passed", k, r) + } + } + if e := ended(p3); e[len(e)-1].EndCount <= e[0].EndCount { + t.Errorf("pipeline-ended counts %d then %d; want a higher end count", e[0].EndCount, e[len(e)-1].EndCount) + } + + // A second revision with the same digests asks its own runs, so a failed test runs again (§724). + revision.Store(2) + publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 2, Kind: "trunk", Branch: "main", Source: 4}) + const p4, p5 = "main.4.app", "main.5.app" + takeIn("app", "c1", "src2:aa", 4, 2) + placed(p4) + run4, r4 := runOfStep(p4, "app:program", "integration") + if !strings.HasPrefix(run4, "main.2.") || r4.Ask == integ.Ask { + t.Fatalf("revision 2's integration run %s (%s); want its own", run4, r4.Ask) + } + // A replaced reader leaves the ask to the other readers (§726): a second pipeline of the same candidate and digest reads + // the run, the first is cancelled and leaves it, and the run's ask is not dropped. + takeIn("app", "c1", "src2:aa", 5, 2) + placed(p5) + waitFor(t, ctx, logs, func() bool { r, _ := runOf(run4); return len(r.Readers) == 4 }) + if e := call(seat.VerbCancel, seat.CancelInput{Pipeline: p4, Why: "replaced"}, nil); e != "" { + t.Fatalf("cancel: %s", e) + } + waitFor(t, ctx, logs, func() bool { + r, _ := runOf(run4) + return len(r.Readers) == 2 && r.Readers[0].Pipeline == p5 && r.Readers[1].Pipeline == p5 + }) + if r, _ := runOf(run4); r.Ended != nil || r.Ask != r4.Ask { + t.Fatalf("the run of the replaced readers = %+v; want it running under its ask", r) + } + mu.Lock() + dropped := append([]string(nil), drops...) + mu.Unlock() + if len(dropped) != 0 { + t.Fatalf("drops asked %v; want none while a reader is left\n%s", dropped, logs.String()) + } + // A taken ask silent for its bound ends could not run with its readers, its ask dropped (§725, §720). + taken(r4.Ask) + waitFor(t, ctx, logs, func() bool { + return step(read(p5), "app:tools", "integration").Outcome == seat.StepResultOutcomeCouldNotRun && + step(read(p5), "app:program", "integration").Outcome == seat.StepResultOutcomeCouldNotRun + }) + if r, _ := runOf(run4); r.Ended == nil || r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, "test-step-bound") { + t.Errorf("the silent run = %+v; want it ended could not run, naming its bound", r) + } + waitFor(t, ctx, logs, func() bool { + mu.Lock() + defer mu.Unlock() + return len(drops) == 1 && strings.Contains(drops[0], r4.Ask) + }) + + // A seat the seat-holders view names no holder for ends the pipeline failed at once, naming it, its other steps not + // asked (§535, §647; A-470g, A-470h). + const web = "main.5.web" + takeIn("web", "w1", "src2:ww", 5, 2) + waitFor(t, ctx, logs, func() bool { return len(ended(web)) == 1 }) + if e := ended(web)[0]; e.Outcome != seat.PipelineEndedOutcomeFailed || e.Why != "the seat node-build-node has no holder: the seat-holders view names none" { + t.Fatalf("pipeline-ended = %+v; want failed at once naming the seat", e) + } + // The steps of the seat with no holder end could not run, a bypassed step stays bypassed, every other ends not asked, + // and none is asked (A-470h). + for _, s := range read(web).Steps { + want := seat.StepResultOutcomeNotAsked + switch { + case s.Status == seat.StepResultStatusBypass: + want = seat.StepResultOutcomeBypassed + case s.Seat == "node-build-node": + want = seat.StepResultOutcomeCouldNotRun + } + if s.Outcome != want || s.Ask != "" { + t.Errorf("the step %s %s of the pipeline with no holder = %s (ask %q); want %s", s.Artifact, s.Step, s.Outcome, s.Ask, want) + } + } + + // test-runs is answered by the process, its fill against test-runs-cap-mib (A-470i), each run's packages from its output + // object; test-runs-drop deletes the key and its object (§728). + var runs seat.TestRunsAnswer + if e := call(seat.VerbTestRuns, seat.TestRunsInput{Candidate: "main.1"}, &runs); e != "" || len(runs.Runs) != 2 || runs.Fill.Cap != 1<<20 || runs.Fill.Store != TestRunsState { + t.Fatalf("test-runs = %+v (%s); want main.1's two runs, the fill against 1 MiB", runs, e) + } + objects, err := js.ObjectStore(ctx, catalogue.BusName(module, catalogue.StateObjectsName(TestRunsState))) + if err != nil { + t.Fatal(err) + } + if _, err := objects.GetInfo(ctx, unitRun); err != nil { + t.Fatalf("the unit run's output object: %v", err) + } + var drop seat.TestRunsDropAnswer + if e := call(seat.VerbTestRunsDrop, seat.TestRunsDropInput{Key: unitRun, Why: "test 109"}, &drop); e != "" || drop.Run.Key != unitRun { + t.Fatalf("test-runs-drop = %+v (%s)", drop, e) + } + if _, ok := runOf(unitRun); ok { + t.Error("the dropped run's key is kept") + } + if _, err := objects.GetInfo(ctx, unitRun); err == nil { + t.Error("the dropped run's output object is kept") + } + + // At the cap a step ends could not run, naming the fill (§725, A-469d). + pad := make([]byte, 600<<10) + for _, k := range []string{"pad.a", "pad.b"} { + if _, err := runsKV.Put(ctx, k, pad); err != nil { + t.Fatal(err) + } + } + revision.Store(3) + publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 3, Kind: "trunk", Branch: "main", Source: 5}) + const p6 = "main.6.app" + takeIn("app", "c1", "src2:aa", 6, 3) + waitFor(t, ctx, logs, func() bool { + s := step(read(p6), "app:program", "unit") + return s.Outcome == seat.StepResultOutcomeCouldNotRun && strings.Contains(s.Why, "test-runs-cap-mib") + }) +} + +func mustJSON(v any) []byte { + raw, _ := json.Marshal(v) + return raw +}