pipeline runner: a pipeline for every module whose fingerprint has none, on a take-in and at start-up
A batched take-in names only its newest revision's movers; the other modules it moved never got a pipeline. Starter.TakenIn and the runner's first-half pickup now start one for each module of the branch's head with no pipeline at its fingerprint. Builds: design 47 G §44, §54, §88, §476
This commit is contained in:
@@ -163,6 +163,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
// Every module of the half whose fingerprint has no pipeline gets one at the half's source revision, each said in the
|
||||
// journal: what a take-in batch, or a runner that was away, left unstarted.
|
||||
if n, err := starter.StartUnstarted(branch, half.Source, half); err != nil || n > 0 {
|
||||
logger.Printf("the modules of %s with no pipeline at their fingerprint: %d started (%v)", branch, n, err)
|
||||
}
|
||||
// A pipeline of the branch whose registration was heard before this half was held gets its steps now (A-327).
|
||||
if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 {
|
||||
logger.Printf("the pipelines of %s that waited on its half for their steps: %d placed (%v)", branch, n, err)
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
@@ -459,6 +460,15 @@ func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half
|
||||
started++
|
||||
}
|
||||
}
|
||||
// A batch of merges is taken in as one event naming the newest revision's movers only; every other module the batch moved
|
||||
// has its fingerprint in the half and no pipeline yet, and starts one here at the take-in's source revision.
|
||||
if held {
|
||||
n, err := s.StartUnstarted(ev.Branch, ev.Source, half)
|
||||
started += n
|
||||
if err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
}
|
||||
// The view was read before the starts wrote: when pipelines were held back for it and it has reached the revision since,
|
||||
// they are placed now, not left for an advance that already came (A-448, §200).
|
||||
if !held && started > 0 && revision > 0 && s.Store != nil {
|
||||
@@ -477,3 +487,64 @@ func commitOrGroup(p seat.Pipeline) string {
|
||||
}
|
||||
return p.Commit
|
||||
}
|
||||
|
||||
// Unstarted names the modules of half whose fingerprint has no pipeline on branch: no pipeline of any outcome, a published
|
||||
// one included, carries that module at that fingerprint. A run on request is no pipeline of the branch's head. Sorted.
|
||||
func (s *Starter) Unstarted(branch string, half Half) ([]string, error) {
|
||||
if s.Store == nil {
|
||||
return nil, errors.New("the runner holds no state pipelines to read")
|
||||
}
|
||||
pkeys, err := s.Store.Keys(keys.EscapeName(branch) + ".*.*")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the pipelines of %s are not listed: %w", branch, err)
|
||||
}
|
||||
have := map[string]bool{}
|
||||
for _, k := range pkeys {
|
||||
p, _, err := s.Store.Get(k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the pipeline %s is not read: %w", k, err)
|
||||
}
|
||||
if p.Run == "" {
|
||||
have[p.Module+"\x00"+p.Fingerprint] = true
|
||||
}
|
||||
}
|
||||
var out []string
|
||||
for module, e := range half.Entries {
|
||||
if e.Fp != "" && !have[module+"\x00"+e.Fp] {
|
||||
out = append(out, module)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// StartUnstarted starts, at source, the pipeline of each module Unstarted names, from the entry half holds: the class fix of a
|
||||
// batched take-in, which names only its newest revision's movers. It says how many it started.
|
||||
func (s *Starter) StartUnstarted(branch string, source int64, half Half) (int, error) {
|
||||
if s.Store == nil {
|
||||
return 0, nil
|
||||
}
|
||||
if source <= 0 {
|
||||
s.logf("the catalog half of %s names no source revision: the modules with no pipeline start none now", branch)
|
||||
return 0, nil
|
||||
}
|
||||
modules, err := s.Unstarted(branch, half)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
started := 0
|
||||
var first error
|
||||
for _, m := range modules {
|
||||
e := half.Entries[m]
|
||||
ok, err := s.Start(Begin{Branch: branch, Source: source, Module: m, Commit: e.Commit, Fp: e.Fp, Repository: e.Repository,
|
||||
Revision: half.Revision, Entries: half.Entries})
|
||||
if err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
if ok {
|
||||
started++
|
||||
s.logf("%s has no pipeline at its fingerprint %s: one is started at source revision %d", m, e.Fp, source)
|
||||
}
|
||||
}
|
||||
return started, first
|
||||
}
|
||||
|
||||
@@ -266,3 +266,51 @@ func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T
|
||||
t.Errorf("the pipeline a runner before this one set aside: placed %d (%v), %d steps, %d asks more, want it started as a newly started one", n, err, len(p.Steps), len(asks.asks)-before)
|
||||
}
|
||||
}
|
||||
|
||||
// A batch of merges taken in together is one take-in.taken naming only the newest revision's movers: every other module of the
|
||||
// branch's head with no pipeline at its fingerprint gets one too, at the take-in's source revision; a module whose fingerprint
|
||||
// has a pipeline already (of any source revision) and a run on request's pipeline start nothing.
|
||||
func TestTakenInStartsEveryModuleWithNoPipelineAtItsFingerprint(t *testing.T) {
|
||||
mem := memStore{}
|
||||
store := memCreated{mem}
|
||||
s := &Starter{Pipelines: store, Store: mem, Say: func(string, any) error { return nil }}
|
||||
half := Half{Revision: 12, Source: 9, Entries: map[string]catalog.CatalogEntry{
|
||||
"moved": {Commit: "c1", Fp: "src2:aa", Repository: "o/moved"},
|
||||
"batched": {Commit: "c2", Fp: "src2:bb", Repository: "o/batched"},
|
||||
"done": {Commit: "c3", Fp: "src2:cc"},
|
||||
"run": {Commit: "c4", Fp: "src2:dd"},
|
||||
"nofp": {Commit: "c5"},
|
||||
}}
|
||||
view := func(string) (Half, bool) { return half, true }
|
||||
mem["main.3.done"] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
}{seat.Pipeline{Module: "done", Fingerprint: "src2:cc", Outcome: seat.PipelineOutcomePassed}, 1}
|
||||
mem["main.4.run"] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
}{seat.Pipeline{Module: "run", Fingerprint: "src2:dd", Run: "r1"}, 1}
|
||||
ev := catalog.TakeInTaken{Kind: "taken", Branch: "main", Source: 9, Revision: 12,
|
||||
Entries: []catalog.TakeInEntry{{Module: "moved", Commit: "c1", Fingerprint: "src2:aa"}}}
|
||||
n, err := s.TakenIn(ev, view)
|
||||
if err != nil || n != 3 {
|
||||
t.Fatalf("the take-in started %d (%v), want moved, batched and run (a run on request is no pipeline of the head)", n, err)
|
||||
}
|
||||
for _, k := range []string{"main.9.moved", "main.9.batched", "main.9.run"} {
|
||||
if _, _, err := mem.Get(k); err != nil {
|
||||
t.Errorf("%s is not started: %v", k, err)
|
||||
}
|
||||
}
|
||||
for _, k := range []string{"main.9.done", "main.9.nofp"} {
|
||||
if _, _, err := mem.Get(k); err == nil {
|
||||
t.Errorf("%s is started, but its module has a pipeline at its fingerprint or none", k)
|
||||
}
|
||||
}
|
||||
if n, err := s.TakenIn(ev, view); err != nil || n != 0 {
|
||||
t.Errorf("the same take-in heard again started %d (%v), want none", n, err)
|
||||
}
|
||||
// A half naming no source revision starts none (the start-up sweep reads the half's own).
|
||||
if n, err := s.StartUnstarted("main", 0, Half{Entries: map[string]catalog.CatalogEntry{"late": {Fp: "src2:ee"}}}); n != 0 || err != nil {
|
||||
t.Errorf("a half with no source revision started %d (%v)", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user