From b9c50ef0c5ab4338f982f4bc07d27d3276b1bca4 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 16 Nov 2026 21:14:46 +0100 Subject: [PATCH] pipeline runner: a pipeline for every module whose fingerprint has none, on a take-in and at start-up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A batched take-in names only its newest revision's movers; the other modules it moved never got a pipeline. Starter.TakenIn and the runner's first-half pickup now start one for each module of the branch's head with no pipeline at its fingerprint. Builds: design 47 G §44, §54, §88, §476 --- .../cmd/mesh-delivery-pipeline/main.go | 5 ++ .../internal/pipeline/start.go | 71 +++++++++++++++++++ .../internal/pipeline/start_test.go | 48 +++++++++++++ 3 files changed, 124 insertions(+) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index ef8ad6cf..7d973c3d 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -163,6 +163,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if !ok { return } + // Every module of the half whose fingerprint has no pipeline gets one at the half's source revision, each said in the + // journal: what a take-in batch, or a runner that was away, left unstarted. + if n, err := starter.StartUnstarted(branch, half.Source, half); err != nil || n > 0 { + logger.Printf("the modules of %s with no pipeline at their fingerprint: %d started (%v)", branch, n, err) + } // A pipeline of the branch whose registration was heard before this half was held gets its steps now (A-327). if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 { logger.Printf("the pipelines of %s that waited on its half for their steps: %d placed (%v)", branch, n, err) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index b95dfc29..9db155f9 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "sort" "strconv" "time" @@ -459,6 +460,15 @@ func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half started++ } } + // A batch of merges is taken in as one event naming the newest revision's movers only; every other module the batch moved + // has its fingerprint in the half and no pipeline yet, and starts one here at the take-in's source revision. + if held { + n, err := s.StartUnstarted(ev.Branch, ev.Source, half) + started += n + if err != nil && first == nil { + first = err + } + } // The view was read before the starts wrote: when pipelines were held back for it and it has reached the revision since, // they are placed now, not left for an advance that already came (A-448, §200). if !held && started > 0 && revision > 0 && s.Store != nil { @@ -477,3 +487,64 @@ func commitOrGroup(p seat.Pipeline) string { } return p.Commit } + +// Unstarted names the modules of half whose fingerprint has no pipeline on branch: no pipeline of any outcome, a published +// one included, carries that module at that fingerprint. A run on request is no pipeline of the branch's head. Sorted. +func (s *Starter) Unstarted(branch string, half Half) ([]string, error) { + if s.Store == nil { + return nil, errors.New("the runner holds no state pipelines to read") + } + pkeys, err := s.Store.Keys(keys.EscapeName(branch) + ".*.*") + if err != nil { + return nil, fmt.Errorf("the pipelines of %s are not listed: %w", branch, err) + } + have := map[string]bool{} + for _, k := range pkeys { + p, _, err := s.Store.Get(k) + if err != nil { + return nil, fmt.Errorf("the pipeline %s is not read: %w", k, err) + } + if p.Run == "" { + have[p.Module+"\x00"+p.Fingerprint] = true + } + } + var out []string + for module, e := range half.Entries { + if e.Fp != "" && !have[module+"\x00"+e.Fp] { + out = append(out, module) + } + } + sort.Strings(out) + return out, nil +} + +// StartUnstarted starts, at source, the pipeline of each module Unstarted names, from the entry half holds: the class fix of a +// batched take-in, which names only its newest revision's movers. It says how many it started. +func (s *Starter) StartUnstarted(branch string, source int64, half Half) (int, error) { + if s.Store == nil { + return 0, nil + } + if source <= 0 { + s.logf("the catalog half of %s names no source revision: the modules with no pipeline start none now", branch) + return 0, nil + } + modules, err := s.Unstarted(branch, half) + if err != nil { + return 0, err + } + started := 0 + var first error + for _, m := range modules { + e := half.Entries[m] + ok, err := s.Start(Begin{Branch: branch, Source: source, Module: m, Commit: e.Commit, Fp: e.Fp, Repository: e.Repository, + Revision: half.Revision, Entries: half.Entries}) + if err != nil && first == nil { + first = err + } + if ok { + started++ + s.logf("%s has no pipeline at its fingerprint %s: one is started at source revision %d", m, e.Fp, source) + } + } + return started, first +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go index b8a999e1..7428c5ad 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -266,3 +266,51 @@ func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T t.Errorf("the pipeline a runner before this one set aside: placed %d (%v), %d steps, %d asks more, want it started as a newly started one", n, err, len(p.Steps), len(asks.asks)-before) } } + +// A batch of merges taken in together is one take-in.taken naming only the newest revision's movers: every other module of the +// branch's head with no pipeline at its fingerprint gets one too, at the take-in's source revision; a module whose fingerprint +// has a pipeline already (of any source revision) and a run on request's pipeline start nothing. +func TestTakenInStartsEveryModuleWithNoPipelineAtItsFingerprint(t *testing.T) { + mem := memStore{} + store := memCreated{mem} + s := &Starter{Pipelines: store, Store: mem, Say: func(string, any) error { return nil }} + half := Half{Revision: 12, Source: 9, Entries: map[string]catalog.CatalogEntry{ + "moved": {Commit: "c1", Fp: "src2:aa", Repository: "o/moved"}, + "batched": {Commit: "c2", Fp: "src2:bb", Repository: "o/batched"}, + "done": {Commit: "c3", Fp: "src2:cc"}, + "run": {Commit: "c4", Fp: "src2:dd"}, + "nofp": {Commit: "c5"}, + }} + view := func(string) (Half, bool) { return half, true } + mem["main.3.done"] = struct { + p seat.Pipeline + rev uint64 + }{seat.Pipeline{Module: "done", Fingerprint: "src2:cc", Outcome: seat.PipelineOutcomePassed}, 1} + mem["main.4.run"] = struct { + p seat.Pipeline + rev uint64 + }{seat.Pipeline{Module: "run", Fingerprint: "src2:dd", Run: "r1"}, 1} + ev := catalog.TakeInTaken{Kind: "taken", Branch: "main", Source: 9, Revision: 12, + Entries: []catalog.TakeInEntry{{Module: "moved", Commit: "c1", Fingerprint: "src2:aa"}}} + n, err := s.TakenIn(ev, view) + if err != nil || n != 3 { + t.Fatalf("the take-in started %d (%v), want moved, batched and run (a run on request is no pipeline of the head)", n, err) + } + for _, k := range []string{"main.9.moved", "main.9.batched", "main.9.run"} { + if _, _, err := mem.Get(k); err != nil { + t.Errorf("%s is not started: %v", k, err) + } + } + for _, k := range []string{"main.9.done", "main.9.nofp"} { + if _, _, err := mem.Get(k); err == nil { + t.Errorf("%s is started, but its module has a pipeline at its fingerprint or none", k) + } + } + if n, err := s.TakenIn(ev, view); err != nil || n != 0 { + t.Errorf("the same take-in heard again started %d (%v), want none", n, err) + } + // A half naming no source revision starts none (the start-up sweep reads the half's own). + if n, err := s.StartUnstarted("main", 0, Half{Entries: map[string]catalog.CatalogEntry{"late": {Fp: "src2:ee"}}}); n != 0 || err != nil { + t.Errorf("a half with no source revision started %d (%v)", n, err) + } +} -- 2.54.0