From 3ad2ac455a795ca8012d91ff984b8c3b4983e7b1 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 02:37:03 +0100 Subject: [PATCH] R3: the runner says no pinned or freed; a pipeline's pins are the indexer's MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runner's pins of a pipeline's catalog revision (pins.go, PinRunning at its start, the pin at a start, a rerun and an ask again, the free at an end) go; the indexer pins on the take-in's write and on pipeline-started, and frees on pipeline-ended (its half of §749, written by the indexer's developer). Builds: design 47 H §749, §763, §764 Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../cmd/mesh-delivery-pipeline/main.go | 7 -- .../internal/pipeline/pins.go | 67 ------------ .../internal/pipeline/pins_test.go | 103 ++---------------- .../internal/pipeline/start.go | 1 - .../internal/pipeline/start_test.go | 2 +- .../internal/pipeline/steps.go | 3 - .../internal/pipeline/verbs.go | 3 - 7 files changed, 9 insertions(+), 177 deletions(-) delete mode 100644 modules/mesh-delivery-pipeline/internal/pipeline/pins.go diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 6d125f97..3b784d2c 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -130,13 +130,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } hearer := newHearer(ctx, js, cred, pipelines, kept, say, logger.Printf) hearer.Asker = busAsker{conn: conn, cred: cred} - // A running pipeline keeps the catalog graph revision it runs on (A-449): said again at each start, as the gate states its - // pins, for pins said before the indexer's consumer existed. - if n, err := hearer.PinRunning(); err != nil { - logger.Printf("the running pipelines' pins are not said: %v", err) - } else { - logger.Printf("pinned the catalog graph revision of %d running pipelines", n) - } hearer.Ager = newAger(ctx, js, cred, pipelines, kept, aged, logger.Printf) // The stores' fill is said before a store is full, and a full one refuses a start (§366). fills := &pipeline.Fills{Read: storeFills(ctx, js, cred, time.Now), Logf: logger.Printf, diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/pins.go b/modules/mesh-delivery-pipeline/internal/pipeline/pins.go deleted file mode 100644 index 918c4e37..00000000 --- a/modules/mesh-delivery-pipeline/internal/pipeline/pins.go +++ /dev/null @@ -1,67 +0,0 @@ -package pipeline - -import ( - "strconv" - - delivery "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" - seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" -) - -// PinRecord is the record id a pipeline pins its catalog graph revision under: `pipeline.` (design 47 G §153, §579, -// A-449), as a delivery record pins its revisions under its own id, so the indexer keeps the revision while any pipeline -// runs on it. -func PinRecord(key string) string { return "pipeline." + key } - -// pinned says whether a pipeline pins a revision: a branch's pipeline that names its catalog graph revision. A run on -// request reads its candidate's revision, which the indexer keeps while the candidate is open, never a branch's. -func pinned(p seat.Pipeline) bool { return p.Run == "" && p.Branch != "" && p.Revision > 0 } - -// sayPin says the seat mesh-delivery's pinned or freed for p's catalog graph revision, in the form the gate says them and -// the indexer applies them (a set of record ids per revision, so a repeat changes nothing). -func sayPin(say func(subject string, body any) error, subject string, p seat.Pipeline) error { - if say == nil || !pinned(p) { - return nil - } - return say(subject, delivery.Pins{Record: PinRecord(p.Pipeline), Milestone: "pipeline " + strconv.FormatInt(p.Source, 10), - Catalog: []delivery.PinsCatalogItem{{Branch: p.Branch, Revision: p.Revision}}}) -} - -// Pin says pinned for the catalog graph revision of a running pipeline (A-449). -func (s *Starter) pin(p seat.Pipeline) { - if err := sayPin(s.Say, delivery.SubjectPinned, p); err != nil { - s.logf("%s does not pin its catalog graph revision %d, so the indexer may remove it: %v", p.Pipeline, p.Revision, err) - } -} - -// pin and free say it for a pipeline the hearer reopens or ends (A-449). -func (h *Hearer) pin(p seat.Pipeline) { - if err := sayPin(h.Say, delivery.SubjectPinned, p); err != nil { - h.logf("%s does not pin its catalog graph revision %d, so the indexer may remove it: %v", p.Pipeline, p.Revision, err) - } -} - -func (h *Hearer) free(p seat.Pipeline) { - if err := sayPin(h.Say, delivery.SubjectFreed, p); err != nil { - h.logf("%s does not free its catalog graph revision %d, which the indexer keeps while it is pinned: %v", p.Pipeline, p.Revision, err) - } -} - -// PinRunning says pinned again for every running pipeline at the runner's start, as the gate states its pins at each -// start: pins said before the indexer's consumer existed, or older than the stream keeps, are heard again, and the set -// form absorbs the repeats. It says how many it said. -func (h *Hearer) PinRunning() (int, error) { - pkeys, err := h.Store.Keys("*.*.*") - if err != nil { - return 0, err - } - n := 0 - for _, key := range pkeys { - p, _, err := h.Store.Get(key) - if err != nil || p.Outcome != seat.PipelineOutcomeRunning || !pinned(p) { - continue - } - h.pin(p) - n++ - } - return n, nil -} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go index a179ebd5..cab7a41f 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go @@ -1,7 +1,6 @@ package pipeline import ( - "errors" "testing" catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" @@ -9,78 +8,33 @@ import ( seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" ) -type saidPin struct { - subject string - pins delivery.Pins -} - -// A-449 (design 47 G §153, §579): a pipeline pins its catalog graph revision as a delivery record does, under the record id -// pipeline., when it starts, again at each start of the runner while it runs, and frees it when it ends, by any -// outcome (superseded included); a run on request, which reads its candidate's revision, pins none. -func TestAPipelinePinsItsCatalogRevisionWhileItRuns(t *testing.T) { - var said []saidPin +// R3/AC1 (design 47 H §749, §763): a pipeline's pins are the indexer's. The runner says no pinned and no freed: not at a +// pipeline's start, its end, its supersession or a rerun. +func TestTheRunnerSaysNoPin(t *testing.T) { + var pins []string say := func(subject string, body any) error { - if p, ok := body.(delivery.Pins); ok { - said = append(said, saidPin{subject, p}) + if _, ok := body.(delivery.Pins); ok || subject == delivery.SubjectPinned || subject == delivery.SubjectFreed { + pins = append(pins, subject) } return nil } - want := func(i int, subject, record string, revision int64) { - t.Helper() - if i >= len(said) { - t.Fatalf("said %d pins, want one more (%s %s %d)", len(said), subject, record, revision) - } - p := said[i] - if p.subject != subject || p.pins.Record != record || len(p.pins.Catalog) != 1 || p.pins.Catalog[0].Branch != "main" || p.pins.Catalog[0].Revision != revision { - t.Errorf("pin %d = %+v, want %s of %s at main %d", i, p, subject, record, revision) - } - } - // At its start. s := &Starter{Pipelines: created{}, Say: say} if ok, err := s.Start(Begin{Branch: "main", Source: 7, Module: "a", Commit: "c1", Revision: 12}); !ok || err != nil { t.Fatalf("start: %v %v", ok, err) } - want(0, delivery.SubjectPinned, "pipeline.main.7.a", 12) - // A start that names no revision pins nothing; one written already says nothing more. - said = nil - if _, err := s.Start(Begin{Branch: "main", Source: 8, Module: "a", Commit: "c1"}); err != nil || len(said) != 0 { - t.Errorf("a start with no revision said %v (%v)", said, err) - } - if ok, _ := s.Start(Begin{Branch: "main", Source: 7, Module: "a", Commit: "c1", Revision: 12}); ok || len(said) != 0 { - t.Errorf("a start written already said %v", said) - } - // At the runner's start, for the running pipelines alone; a run on request pins none. h, store, _, _, _, _ := fixture() h.Say = say p := store[pkey] p.p.Revision = 12 store[pkey] = p - store["main.4.old"] = struct { - p seat.Pipeline - rev uint64 - }{seat.Pipeline{Pipeline: "main.4.old", Branch: "main", Revision: 11, Outcome: seat.PipelineOutcomePassed}, 1} - store["main.r1.app"] = struct { - p seat.Pipeline - rev uint64 - }{seat.Pipeline{Pipeline: "main.r1.app", Branch: "main", Revision: 13, Run: "r1", Outcome: seat.PipelineOutcomeRunning}, 1} - said = nil - if n, err := h.PinRunning(); n != 1 || err != nil { - t.Fatalf("PinRunning said %d (%v), want the running pipeline's alone", n, err) - } - want(0, delivery.SubjectPinned, "pipeline."+pkey, 12) - // Its end frees it, and so does a supersede. - said = nil if err := h.said(store[pkey].p, false, true); err != nil { t.Fatal(err) } - want(0, delivery.SubjectFreed, "pipeline."+pkey, 12) - said = nil if n, err := h.Supersede(seat.Pipeline{Pipeline: "main.6.app", Branch: "main", Source: 6, Module: "app", Commit: "c2", Fingerprint: "src2:bb", Outcome: seat.PipelineOutcomeRunning}, nil); n != 1 || err != nil { t.Fatalf("supersede %d (%v)", n, err) } - want(0, delivery.SubjectFreed, "pipeline."+pkey, 12) - if len(said) != 1 { - t.Errorf("a supersede said %d pins, want the one freed (through its end, once)", len(said)) + if len(pins) != 0 { + t.Errorf("the runner said %v, want no pinned and no freed", pins) } } @@ -180,44 +134,3 @@ func TestPlacingAWaitingPipelineAsksNoRunningStepAgain(t *testing.T) { t.Errorf("an advance placed %d (%v) and asked %d steps again, want nothing", n, err, len(asks.asks)-before) } } - -// A-449: the pipeline-ended's widens read the pipeline's revision while it is still pinned; the pin is freed after the end is -// said, also when saying it fails. -func TestThePinIsFreedAfterTheEndAndItsWidensAreRead(t *testing.T) { - h, store, _, _, _, _ := fixture() - p := store[pkey] - p.p.Revision = 12 - store[pkey] = p - freed, readWhileFreed, bodies := false, false, 0 - h.Say = func(subject string, body any) error { - switch body.(type) { - case delivery.Pins: - freed = freed || subject == delivery.SubjectFreed - default: - bodies++ - } - return nil - } - h.Widens = func(seat.Pipeline) []seat.Widening { - readWhileFreed = freed // the revision is gone once the freed is said - return nil - } - if err := h.said(store[pkey].p, false, true); err != nil { - t.Fatal(err) - } - if readWhileFreed || !freed || bodies == 0 { - t.Errorf("widens read after the freed %v, freed %v, %d other events: want the widens read first and the pin freed after the end", readWhileFreed, freed, bodies) - } - // An end that cannot be said still frees the pin. - freed = false - h.Say = func(subject string, body any) error { - if _, ok := body.(delivery.Pins); ok { - freed = freed || subject == delivery.SubjectFreed - return nil - } - return errors.New("bus away") - } - if err := h.said(store[pkey].p, false, true); err == nil || !freed { - t.Errorf("an end not said: err %v, freed %v, want the error and the pin freed", err, freed) - } -} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index b6ce3fae..b9c2448c 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -248,7 +248,6 @@ func (s *Starter) Start(b Begin) (bool, error) { } else if err != nil { return false, fmt.Errorf("the pipeline %s is not written: %w", key, err) } - s.pin(p) // the catalog graph revision it runs on is kept while it runs (A-449) ev := seat.PipelineStarted{V: 1, At: &now, Pipeline: key, Branch: b.Branch, Source: b.Source, Module: b.Module, Commit: b.Commit, Fingerprint: b.Fp, Group: b.Group, Revision: b.Revision, Published: b.Published} commitOrGroup := b.Commit diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go index 8dbbdfde..b6b9d42e 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -31,7 +31,7 @@ func TestStarts(t *testing.T) { var superseding []string s := &Starter{Pipelines: store, Say: func(subject string, body any) error { ev, started := body.(seat.PipelineStarted) - if !started { // a pipeline's pin of its catalog graph revision (A-449) is no pipeline-started + if !started { // only pipeline-started is counted here return nil } said = append(said, subject+" "+ev.Pipeline) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index bc0033f8..817ad8f7 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -374,9 +374,6 @@ func (h *Hearer) said(p seat.Pipeline, asked, ended bool) error { } } if ended { - // The end is said first, with its widens, which read the revision the pipeline pinned; only then is the pin freed (A-449), - // and also when saying the end fails. - defer h.free(p) if err := h.say(seat.EventPipelineEnded, p, nil); err != nil { return fmt.Errorf("pipeline %s ended but %s is not said: %w", p.Pipeline, seat.EventPipelineEnded, err) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go index 5ca58405..607b1d75 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -287,7 +287,6 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error if err != nil { return seat.RerunAnswer{}, err } - v.Hearer.pin(p) // a pipeline that runs again keeps its revision again (A-449) if ask != nil { v.Starter.carry(in.Pipeline, []Asked{*ask}) if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil { @@ -386,7 +385,6 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult) if err != nil { return seat.RerunAnswer{}, err } - h.pin(p) switch { case join || read: v.Starter.joinRun(run, reader) @@ -592,6 +590,5 @@ func (v Verbs) AskAgain(caller string, in seat.AskAgainInput) (seat.AskAgainAnsw if err != nil { return seat.AskAgainAnswer{}, err } - v.Hearer.pin(p) return seat.AskAgainAnswer{Result: p.Steps[i]}, nil }