From 8ef266ad6fc8e1105a8ac21b30546968d2eabe76 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 13 Nov 2026 02:30:13 +0100 Subject: [PATCH] Runner (A-327): a registration heard before the runner can place from it is not lost MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A registration-changed heard before the registrar was stored, or before the branch's catalog half was held, placed nothing and was gone: its pipelines kept no steps. The half becoming held and the registrations' first view now place every running pipeline with no steps whose registration is held (Waiting), and the first view also picks up each held branch whose readiness came before the resume callback was stored. Builds: design 47 G §21, §23, §54, §88, A-327 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-pipeline/main.go | 9 ++++ .../internal/pipeline/start.go | 46 +++++++++++++++---- .../internal/pipeline/start_test.go | 28 +++++++++++ 3 files changed, 74 insertions(+), 9 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index f8976c2a..9f442ba7 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -166,6 +166,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if n, err := starter.Release(pipelineStore{ctx: ctx, kv: pipelines}, starter.SelfFp, "ran", rv.Catalog.View); err != nil { logger.Printf("the pipelines held for the runner at %s: %d released, then %v", starter.SelfFp, n, err) } + // A pipeline of the branch whose registration was heard before this half was held gets its steps now (A-327). + if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 { + logger.Printf("the pipelines of %s that waited on its half for their steps: %d placed (%v)", branch, n, err) + } n, err := starter.Resume(pipelineStore{ctx: ctx, kv: pipelines}, branch, half) if err != nil { logger.Printf("the steps placed on %s and not yet asked: %d asked, then %v", branch, n, err) @@ -190,6 +194,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return case <-rv.Registrations.Ready(): } + // A registration or a branch's half heard before the runner could act on it (its first views are asked before the + // starter is made) is not lost: each branch held is picked up now as its half becoming held would (A-327, §88). + for _, branch := range rv.Catalog.Branches() { + resume(branch) + } if err := placeLostRuns(starter.UnplacedRuns, calls.CallKept, placeHalf, logger.Printf); err != nil { logger.Printf("the runs not placed at the start: %v", err) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 420fa8ed..dcd0f0e6 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -234,15 +234,39 @@ func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) { // Registered places, on a registration-changed of reg's module, the steps of every running pipeline of that module that has // none, from the registration (design 47 G §21, §23, A-327): a pipeline started before its module had a registration. Each is // written by compare-and-set, so it is placed once, its steps asked, and said; one held for the runner's new version is left to -// its release, a run on request to its candidate, and one whose branch has no catalog half yet keeps waiting, the first error -// returned. It says how many it placed. +// its release, a run on request to its candidate, and one whose branch has no catalog half yet keeps waiting, said, until that +// half is held (Waiting). It says how many it placed. func (s *Starter) Registered(store Updater, reg catalog.Registration, view func(branch string) (Half, bool)) (int, error) { if reg.Module == "" { return 0, nil } - pkeys, err := store.Keys("*.*." + reg.Module) + return s.placeWaiting(store, "*.*."+reg.Module, func(p seat.Pipeline) (catalog.Registration, bool) { + return reg, p.Module == reg.Module + }, view, "the registration of "+reg.Module+", which arrived after the pipeline started") +} + +// Waiting places, from the registrations held now, the steps of every running pipeline of branch that has none (A-327): a +// registration heard before the runner could place from it, or before the branch's catalog half was held, is not lost, since +// the half becoming held and the registrations' first view being held each place what waited on them. An empty branch is +// every branch. It says how many it placed. +func (s *Starter) Waiting(store Updater, branch string, view func(branch string) (Half, bool)) (int, error) { + if s.Registration == nil { + return 0, nil + } + return s.placeWaiting(store, "*.*.*", func(p seat.Pipeline) (catalog.Registration, bool) { + if branch != "" && p.Branch != branch { + return catalog.Registration{}, false + } + return s.Registration(p.Module) + }, view, "the registration held when its branch's half or the registrations became held") +} + +// placeWaiting places the steps of each running pipeline under pattern that has none, is neither held nor a run on request, +// and whose registration reg names, from that registration and its branch's catalog half (Registered, Waiting). +func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipeline) (catalog.Registration, bool), view func(branch string) (Half, bool), from string) (int, error) { + pkeys, err := store.Keys(pattern) if err != nil { - return 0, fmt.Errorf("the pipelines of %s are not listed: %w", reg.Module, err) + return 0, fmt.Errorf("the pipelines %s are not listed: %w", pattern, err) } now := time.Now().UTC() if s.Now != nil { @@ -261,22 +285,26 @@ func (s *Starter) Registered(store Updater, reg catalog.Registration, view func( note(fmt.Errorf("the pipeline %s is not read: %w", key, err)) continue } - if p.Module != reg.Module || p.Outcome != seat.PipelineOutcomeRunning || p.Held != nil || p.Run != "" || len(p.Steps) > 0 { + if p.Outcome != seat.PipelineOutcomeRunning || p.Held != nil || p.Run != "" || len(p.Steps) > 0 { + continue + } + r, ok := reg(p) + if !ok { continue } half, ok := view(p.Branch) if !ok { - note(fmt.Errorf("the catalog half of %s is not held yet: %s still waits on its steps", p.Branch, key)) + s.logf("the catalog half of %s is not held yet: %s waits on it for its steps, placed once it is held", p.Branch, key) continue } var asks []Asked - p.Steps, asks = Place(p, reg, half.Entries, now, s.Name) + p.Steps, asks = Place(p, r, half.Entries, now, s.Name) if _, err := store.Update(key, p, rev); err != nil { - s.logf("%s is written meanwhile, so this registration places nothing in it: %v", key, err) + s.logf("%s is written meanwhile, so nothing is placed in it now: %v", key, err) continue } placed++ - s.logf("%s: %d steps placed from the registration of %s, which arrived after the pipeline started (A-327)", key, len(p.Steps), reg.Module) + s.logf("%s: %d steps placed from %s (A-327)", key, len(p.Steps), from) if s.Steps != nil && len(asks) > 0 { s.carry(key, asks) if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, p.Commit)); err != nil { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go index 62b668da..95d13a76 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -150,4 +150,32 @@ func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T if n, err := s.Registered(store.memStore, reg, view); n != 0 || err != nil || len(asks.asks) != before { t.Errorf("the registration heard again placed %d (%v), asked %d more, want nothing", n, err, len(asks.asks)-before) } + + // The registration heard before the branch's catalog half is held (or before the runner could place from it) is not + // lost: the half becoming held, or the registrations' first view, places the waiting pipeline once (Waiting). + held := false + late := func(string) (Half, bool) { return Half{Entries: entries}, held } + taken = catalog.TakeInTaken{Key: "m.2", Branch: "next", Source: 5, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c2", Fingerprint: "src2:bb"}}} + registered = false + if n, err := s.TakenIn(taken, view); n != 1 || err != nil { + t.Fatalf("the take-in on next started %d (%v)", n, err) + } + registered = true + if n, err := s.Registered(store.memStore, reg, late); n != 0 || err != nil { + t.Fatalf("the registration before the half placed %d (%v), want none and no error: it waits on the half", n, err) + } + before = len(asks.asks) + held = true + if n, err := s.Waiting(store.memStore, "main", late); n != 0 || err != nil { + t.Errorf("the half of main placed %d (%v) of next's pipelines", n, err) + } + n, err = s.Waiting(store.memStore, "next", late) + p, _, _ = store.Get("next.5.app") + if n != 1 || err != nil || len(p.Steps) == 0 || len(asks.asks) == before { + t.Fatalf("the half of next becoming held placed %d (%v): %d steps, %d asks more, want its steps placed and asked", n, err, len(p.Steps), len(asks.asks)-before) + } + before = len(asks.asks) + if n, err := s.Waiting(store.memStore, "", late); n != 0 || err != nil || len(asks.asks) != before { + t.Errorf("the registrations' first view after it placed %d (%v), asked %d more, want nothing", n, err, len(asks.asks)-before) + } }