From db942e9204f812a530bb1c448913c09217c15442 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:40:22 +0100 Subject: [PATCH] =?UTF-8?q?=C2=A788:=20a=20restarted=20runner=20asks=20the?= =?UTF-8?q?=20steps=20placed=20and=20not=20yet=20asked?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Starter.Resume asks again each running step of a branch's running pipelines under the id it was placed under; the branch view tells Views.OnReady once when it holds its first half, and the runner resumes that branch then. Builds: design 47 G §22, §24, §88 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-pipeline/main.go | 23 +++++++- .../cmd/mesh-delivery-pipeline/views.go | 6 +- .../internal/pipeline/place_test.go | 42 ++++++++++++++ .../internal/pipeline/registrations.go | 15 ++++- .../internal/pipeline/registrations_test.go | 30 ++++++++++ .../internal/pipeline/start.go | 58 +++++++++++++++++++ 6 files changed, 170 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 9c874d18..1066135e 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -11,6 +11,7 @@ import ( "log" "os" "os/signal" + "sync/atomic" "syscall" "github.com/nats-io/nats.go" @@ -60,7 +61,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } defer calls.Close() hub := process.NewEventHub(logger.Printf) - rv, err := startViews(ctx, js, conn, cred, hub, calls.State("findings"), calls.Call, logger.Printf) + // A branch's view holding its first half asks again the steps placed in its running pipelines that were never asked + // (§88); the starter is made after the views, so it is reached through a pointer set once it is. + var resumer atomic.Pointer[func(string)] + rv, err := startViews(ctx, js, conn, cred, hub, calls.State("findings"), calls.Call, func(branch string) { + if f := resumer.Load(); f != nil { + (*f)(branch) + } + }, logger.Printf) if err != nil { return err } @@ -80,6 +88,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // for its step. starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf, Say: say, Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, js: js, cred: cred}, Name: cred.Module} + resume := func(branch string) { + half, ok := rv.Catalog.View(branch) + if !ok { + return + } + n, err := starter.Resume(pipelineStore{ctx: ctx, kv: pipelines}, branch, half) + if err != nil { + logger.Printf("the steps placed on %s and not yet asked: %d asked, then %v", branch, n, err) + return + } + logger.Printf("the steps placed on %s and not yet asked: %d asked again", branch, n) + } + resumer.Store(&resume) if err := hearStepEnded(hub, hearer, logger.Printf); err != nil { return err } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go index 3918bfd5..3228a10c 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go @@ -25,7 +25,7 @@ type runnerViews struct { // startViews wires the runner's views on hub and asks each first view; register every other handler on hub before or // after, then start hub once. func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cred process.Account, hub *process.EventHub, findingsState process.State, - call meshcatalogclient.Call, logf func(string, ...any)) (*runnerViews, error) { + call meshcatalogclient.Call, onReady func(branch string), logf func(string, ...any)) (*runnerViews, error) { findings, err := findingWriter(ctx, cred, conn, js, findingsState, logf) if err != nil { return nil, err @@ -37,7 +37,7 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cr // A watched branch first named by a registration gets its view of the catalog half (CT10), adopted through the reader (design // 47 G §532, A-26): the reader keeps its status, raises and clears its findings through the module's writer, hears the // seat's catalog-revision on the module's consumer and asks its first view at once. - cv := &pipeline.Views{Logf: logf, Open: func(branch string) (pipeline.BranchView, error) { + cv := &pipeline.Views{Logf: logf, OnReady: onReady, Open: func(branch string) (pipeline.BranchView, error) { c, _, err := meshcatalogclient.WireCatalogProcess(reader, call, branch, nil) if err != nil { return nil, err @@ -59,6 +59,8 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cr // catalogView is a branch's meshcatalogclient.Catalog as the runner reads it. type catalogView struct{ c *meshcatalogclient.Catalog } +func (v catalogView) Ready() <-chan struct{} { return v.c.Ready() } + func (v catalogView) Hear(ctx context.Context, data []byte) error { return v.c.Hear(ctx, data) } func (v catalogView) View() (pipeline.Half, bool) { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index e8a98929..92ee45de 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -172,3 +172,45 @@ func TestCompositionIsAskedOfTheAssignerAndItsEndIsTaken(t *testing.T) { t.Fatalf("decoded end = %+v, %v", ev, err) } } + +// A runner restarted after it wrote a pipeline and before its asks reached the queues asks them again when the branch's +// view holds its half: each step placed and still running, of a running pipeline of that branch, under the id it was +// placed under; an ended step, an ended pipeline and another branch's pipeline are asked of no one (design 47 G §88). +func TestResumeAsksTheStepsPlacedAndNotEnded(t *testing.T) { + reg, entries := placeFixture() + placed := func(branch string, source int64) seat.Pipeline { + p := seat.Pipeline{Pipeline: Key(branch, source, "app"), Branch: branch, Source: source, Module: "app", Commit: "c1", Fingerprint: "src2:aa", + Repository: "o/app", Outcome: seat.PipelineOutcomeRunning} + p.Steps, _ = Place(p, reg, entries, time.Unix(1, 0), "the-runner") + return p + } + main4, main5, beta6 := placed("main", 4), placed("main", 5), placed("beta", 6) + main5.Steps[1].Outcome = seat.StepResultOutcomePassed // the build of main.5 ended + ended := placed("main", 3) + ended.Outcome = seat.PipelineOutcomePassed + store := memStore{} + for _, p := range []seat.Pipeline{main4, main5, beta6, ended} { + store[p.Pipeline] = struct { + p seat.Pipeline + rev uint64 + }{p, 1} + } + asks := &stepAsks{} + s := &Starter{Steps: asks, Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }} + n, err := s.Resume(store, "main", Half{Entries: entries}) + // main.4 has the build and three running steps of the fixture; main.5 one fewer, its build ended. + if err != nil || n != 7 || len(asks.asks) != 7 { + t.Fatalf("resume asked %d (%d put), %v; want 7", n, len(asks.asks), err) + } + for _, id := range asks.ids { + found := false + for _, p := range []seat.Pipeline{main4, main5} { + for _, st := range p.Steps { + found = found || (st.Ask == id && st.Outcome == seat.StepResultOutcomeRunning) + } + } + if !found { + t.Errorf("ask %s is of no running step of the branch's running pipelines", id) + } + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go b/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go index 0b1eed89..fbe903d5 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go @@ -81,6 +81,9 @@ type BranchView interface { View() (Half, bool) } +// readyView is a BranchView that says when it holds its first half, as meshcatalogclient.Catalog does. +type readyView interface{ Ready() <-chan struct{} } + // Half is the catalog half a branch view holds, as the runner reads it: its revision and its entries by module. type Half struct { Revision int64 @@ -95,7 +98,10 @@ type Views struct { // Open makes and starts the view of a branch; its error is said in the journal and the branch asked again at the // next registration heard. Open func(branch string) (BranchView, error) - Logf func(string, ...any) + // OnReady is told once for a branch, when its view holds a first half: the steps placed in the pipelines of that branch + // and not yet asked are asked then (§88's restart pickup). + OnReady func(branch string) + Logf func(string, ...any) mu sync.Mutex branches map[string]BranchView @@ -126,6 +132,13 @@ func (v *Views) ensure(branch string) { return } v.branches[branch] = bv + if r, ok := bv.(readyView); ok && v.OnReady != nil { + on := v.OnReady + go func() { + <-r.Ready() + on(branch) + }() + } } // Branches are the watched branches the views hold, sorted. diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/registrations_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/registrations_test.go index 0f150d36..7ff4f096 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/registrations_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/registrations_test.go @@ -83,3 +83,33 @@ type fakeView struct { func (f fakeView) Hear(context.Context, []byte) error { f.heard[f.branch]++; return nil } func (f fakeView) View() (Half, bool) { return Half{Revision: 7}, true } + +type readyFake struct { + fakeView + ch chan struct{} +} + +func (r readyFake) Ready() <-chan struct{} { return r.ch } + +// A branch's view that holds its first half tells Views once, so the runner asks the steps placed on that branch (§88). +func TestViewsTellOnceWhenABranchHoldsItsFirstHalf(t *testing.T) { + ch := make(chan struct{}) + told := make(chan string, 4) + v := &Views{OnReady: func(b string) { told <- b }, Open: func(b string) (BranchView, error) { + return readyFake{fakeView{branch: b, heard: map[string]int{}}, ch}, nil + }} + v.Registered(catalog.Registration{Module: "n", Branches: []string{"main"}}) + v.Registered(catalog.Registration{Module: "m", Branches: []string{"main"}}) + select { + case b := <-told: + t.Fatalf("told %s before the view held a half", b) + default: + } + close(ch) + if b := <-told; b != "main" { + t.Errorf("told %q, want main", b) + } + if len(told) != 0 { + t.Error("a branch's readiness is told once") + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 2ff3a55b..3bfc6e56 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -111,6 +111,64 @@ func (s *Starter) Start(b Begin) (bool, error) { return true, nil } +// Resume asks again, of the build seats and the assigner, each step of branch's running pipelines that is placed and still +// running (design 47 G §88's restart pickup): a pipeline written whose asks never reached their queues (the runner stopped +// between its write and its asks, or a publish failed) is asked at the next start. Each ask carries the id its step was +// placed under, so one the seat holds already is kept once. half is the branch's catalog half, read for the steps' artifacts +// and seats as Place reads it. It says how many steps it asked. +func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) { + if s.Steps == nil || s.Registration == nil { + return 0, nil + } + pkeys, err := store.Keys("*.*.*") + if err != nil { + return 0, fmt.Errorf("the pipelines are not listed: %w", err) + } + now := time.Now().UTC() + if s.Now != nil { + now = s.Now().UTC() + } + asked := 0 + var first error + for _, key := range pkeys { + p, _, err := store.Get(key) + if err != nil { + if first == nil { + first = fmt.Errorf("the pipeline %s is not read: %w", key, err) + } + continue + } + if p.Branch != branch || p.Outcome != seat.PipelineOutcomeRunning { + continue + } + reg, ok := s.Registration(p.Module) + if !ok { + continue + } + running := map[string]bool{} + for _, st := range p.Steps { + if st.Outcome == seat.StepResultOutcomeRunning && st.Ask != "" { + running[st.Ask] = true + } + } + _, placed := Place(p, reg, half.Entries, now, s.Name) + var due []Asked + for _, a := range placed { + if running[a.ID] { + due = append(due, a) + } + } + if len(due) == 0 { + continue + } + if err := askAll(s.Steps, key, due); err != nil && first == nil { + first = err + } + asked += len(due) + } + return asked, first +} + func (s *Starter) logf(format string, args ...any) { if s.Logf != nil { s.Logf(format, args...)