diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index f8976c2a..9f442ba7 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -166,6 +166,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if n, err := starter.Release(pipelineStore{ctx: ctx, kv: pipelines}, starter.SelfFp, "ran", rv.Catalog.View); err != nil { logger.Printf("the pipelines held for the runner at %s: %d released, then %v", starter.SelfFp, n, err) } + // A pipeline of the branch whose registration was heard before this half was held gets its steps now (A-327). + if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 { + logger.Printf("the pipelines of %s that waited on its half for their steps: %d placed (%v)", branch, n, err) + } n, err := starter.Resume(pipelineStore{ctx: ctx, kv: pipelines}, branch, half) if err != nil { logger.Printf("the steps placed on %s and not yet asked: %d asked, then %v", branch, n, err) @@ -190,6 +194,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return case <-rv.Registrations.Ready(): } + // A registration or a branch's half heard before the runner could act on it (its first views are asked before the + // starter is made) is not lost: each branch held is picked up now as its half becoming held would (A-327, §88). + for _, branch := range rv.Catalog.Branches() { + resume(branch) + } if err := placeLostRuns(starter.UnplacedRuns, calls.CallKept, placeHalf, logger.Printf); err != nil { logger.Printf("the runs not placed at the start: %v", err) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 420fa8ed..dcd0f0e6 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -234,15 +234,39 @@ func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) { // Registered places, on a registration-changed of reg's module, the steps of every running pipeline of that module that has // none, from the registration (design 47 G §21, §23, A-327): a pipeline started before its module had a registration. Each is // written by compare-and-set, so it is placed once, its steps asked, and said; one held for the runner's new version is left to -// its release, a run on request to its candidate, and one whose branch has no catalog half yet keeps waiting, the first error -// returned. It says how many it placed. +// its release, a run on request to its candidate, and one whose branch has no catalog half yet keeps waiting, said, until that +// half is held (Waiting). It says how many it placed. func (s *Starter) Registered(store Updater, reg catalog.Registration, view func(branch string) (Half, bool)) (int, error) { if reg.Module == "" { return 0, nil } - pkeys, err := store.Keys("*.*." + reg.Module) + return s.placeWaiting(store, "*.*."+reg.Module, func(p seat.Pipeline) (catalog.Registration, bool) { + return reg, p.Module == reg.Module + }, view, "the registration of "+reg.Module+", which arrived after the pipeline started") +} + +// Waiting places, from the registrations held now, the steps of every running pipeline of branch that has none (A-327): a +// registration heard before the runner could place from it, or before the branch's catalog half was held, is not lost, since +// the half becoming held and the registrations' first view being held each place what waited on them. An empty branch is +// every branch. It says how many it placed. +func (s *Starter) Waiting(store Updater, branch string, view func(branch string) (Half, bool)) (int, error) { + if s.Registration == nil { + return 0, nil + } + return s.placeWaiting(store, "*.*.*", func(p seat.Pipeline) (catalog.Registration, bool) { + if branch != "" && p.Branch != branch { + return catalog.Registration{}, false + } + return s.Registration(p.Module) + }, view, "the registration held when its branch's half or the registrations became held") +} + +// placeWaiting places the steps of each running pipeline under pattern that has none, is neither held nor a run on request, +// and whose registration reg names, from that registration and its branch's catalog half (Registered, Waiting). +func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipeline) (catalog.Registration, bool), view func(branch string) (Half, bool), from string) (int, error) { + pkeys, err := store.Keys(pattern) if err != nil { - return 0, fmt.Errorf("the pipelines of %s are not listed: %w", reg.Module, err) + return 0, fmt.Errorf("the pipelines %s are not listed: %w", pattern, err) } now := time.Now().UTC() if s.Now != nil { @@ -261,22 +285,26 @@ func (s *Starter) Registered(store Updater, reg catalog.Registration, view func( note(fmt.Errorf("the pipeline %s is not read: %w", key, err)) continue } - if p.Module != reg.Module || p.Outcome != seat.PipelineOutcomeRunning || p.Held != nil || p.Run != "" || len(p.Steps) > 0 { + if p.Outcome != seat.PipelineOutcomeRunning || p.Held != nil || p.Run != "" || len(p.Steps) > 0 { + continue + } + r, ok := reg(p) + if !ok { continue } half, ok := view(p.Branch) if !ok { - note(fmt.Errorf("the catalog half of %s is not held yet: %s still waits on its steps", p.Branch, key)) + s.logf("the catalog half of %s is not held yet: %s waits on it for its steps, placed once it is held", p.Branch, key) continue } var asks []Asked - p.Steps, asks = Place(p, reg, half.Entries, now, s.Name) + p.Steps, asks = Place(p, r, half.Entries, now, s.Name) if _, err := store.Update(key, p, rev); err != nil { - s.logf("%s is written meanwhile, so this registration places nothing in it: %v", key, err) + s.logf("%s is written meanwhile, so nothing is placed in it now: %v", key, err) continue } placed++ - s.logf("%s: %d steps placed from the registration of %s, which arrived after the pipeline started (A-327)", key, len(p.Steps), reg.Module) + s.logf("%s: %d steps placed from %s (A-327)", key, len(p.Steps), from) if s.Steps != nil && len(asks) > 0 { s.carry(key, asks) if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, p.Commit)); err != nil { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go index 62b668da..95d13a76 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -150,4 +150,32 @@ func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T if n, err := s.Registered(store.memStore, reg, view); n != 0 || err != nil || len(asks.asks) != before { t.Errorf("the registration heard again placed %d (%v), asked %d more, want nothing", n, err, len(asks.asks)-before) } + + // The registration heard before the branch's catalog half is held (or before the runner could place from it) is not + // lost: the half becoming held, or the registrations' first view, places the waiting pipeline once (Waiting). + held := false + late := func(string) (Half, bool) { return Half{Entries: entries}, held } + taken = catalog.TakeInTaken{Key: "m.2", Branch: "next", Source: 5, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c2", Fingerprint: "src2:bb"}}} + registered = false + if n, err := s.TakenIn(taken, view); n != 1 || err != nil { + t.Fatalf("the take-in on next started %d (%v)", n, err) + } + registered = true + if n, err := s.Registered(store.memStore, reg, late); n != 0 || err != nil { + t.Fatalf("the registration before the half placed %d (%v), want none and no error: it waits on the half", n, err) + } + before = len(asks.asks) + held = true + if n, err := s.Waiting(store.memStore, "main", late); n != 0 || err != nil { + t.Errorf("the half of main placed %d (%v) of next's pipelines", n, err) + } + n, err = s.Waiting(store.memStore, "next", late) + p, _, _ = store.Get("next.5.app") + if n != 1 || err != nil || len(p.Steps) == 0 || len(asks.asks) == before { + t.Fatalf("the half of next becoming held placed %d (%v): %d steps, %d asks more, want its steps placed and asked", n, err, len(p.Steps), len(asks.asks)-before) + } + before = len(asks.asks) + if n, err := s.Waiting(store.memStore, "", late); n != 0 || err != nil || len(asks.asks) != before { + t.Errorf("the registrations' first view after it placed %d (%v), asked %d more, want nothing", n, err, len(asks.asks)-before) + } }