diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 9f655c13..e1650118 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -125,6 +125,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } hearer := newHearer(ctx, js, cred, pipelines, kept, say, logger.Printf) hearer.Asker = busAsker{conn: conn, cred: cred} + // A running pipeline keeps the catalog graph revision it runs on (A-449): said again at each start, as the gate states its + // pins, for pins said before the indexer's consumer existed. + if n, err := hearer.PinRunning(); err != nil { + logger.Printf("the running pipelines' pins are not said: %v", err) + } else { + logger.Printf("pinned the catalog graph revision of %d running pipelines", n) + } hearer.Ager = newAger(ctx, js, cred, pipelines, kept, aged, logger.Printf) // The stores' fill is said before a store is full, and a full one refuses a start (§366). fills := &pipeline.Fills{Read: storeFills(ctx, js, cred, time.Now), Logf: logger.Printf, diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go index 4a2ed358..ccb3ed09 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go @@ -49,13 +49,30 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cr // is picked up as readiness is: its pipelines started with no steps are placed then (A-327, A-336). var wired atomic.Pointer[meshcatalogclient.Catalog] var held atomic.Bool + var heldRevision atomic.Int64 // the newest revision the view was ready with (A-448) c, _, err := meshcatalogclient.WireCatalogProcess(reader, call, branch, func(string, json.RawMessage, bool, time.Time) { c := wired.Load() if c == nil || onReady == nil { return } - if _, ok := c.View(); ok && !held.Swap(true) { - go onReady(branch) + // Ready on the first half held and again on each newer revision: a pipeline that waits for the revision its take-in + // made (A-448, §200) is placed when the view reaches it. + if h, ok := c.View(); ok { + first := !held.Swap(true) + advanced := false + for rev := h.Revision.Revision; ; { + old := heldRevision.Load() + if rev <= old { + break + } + if heldRevision.CompareAndSwap(old, rev) { + advanced = true + break + } + } + if first || advanced { + go onReady(branch) + } } }) if err != nil { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/pins.go b/modules/mesh-delivery-pipeline/internal/pipeline/pins.go new file mode 100644 index 00000000..918c4e37 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/pins.go @@ -0,0 +1,67 @@ +package pipeline + +import ( + "strconv" + + delivery "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" +) + +// PinRecord is the record id a pipeline pins its catalog graph revision under: `pipeline.` (design 47 G §153, §579, +// A-449), as a delivery record pins its revisions under its own id, so the indexer keeps the revision while any pipeline +// runs on it. +func PinRecord(key string) string { return "pipeline." + key } + +// pinned says whether a pipeline pins a revision: a branch's pipeline that names its catalog graph revision. A run on +// request reads its candidate's revision, which the indexer keeps while the candidate is open, never a branch's. +func pinned(p seat.Pipeline) bool { return p.Run == "" && p.Branch != "" && p.Revision > 0 } + +// sayPin says the seat mesh-delivery's pinned or freed for p's catalog graph revision, in the form the gate says them and +// the indexer applies them (a set of record ids per revision, so a repeat changes nothing). +func sayPin(say func(subject string, body any) error, subject string, p seat.Pipeline) error { + if say == nil || !pinned(p) { + return nil + } + return say(subject, delivery.Pins{Record: PinRecord(p.Pipeline), Milestone: "pipeline " + strconv.FormatInt(p.Source, 10), + Catalog: []delivery.PinsCatalogItem{{Branch: p.Branch, Revision: p.Revision}}}) +} + +// Pin says pinned for the catalog graph revision of a running pipeline (A-449). +func (s *Starter) pin(p seat.Pipeline) { + if err := sayPin(s.Say, delivery.SubjectPinned, p); err != nil { + s.logf("%s does not pin its catalog graph revision %d, so the indexer may remove it: %v", p.Pipeline, p.Revision, err) + } +} + +// pin and free say it for a pipeline the hearer reopens or ends (A-449). +func (h *Hearer) pin(p seat.Pipeline) { + if err := sayPin(h.Say, delivery.SubjectPinned, p); err != nil { + h.logf("%s does not pin its catalog graph revision %d, so the indexer may remove it: %v", p.Pipeline, p.Revision, err) + } +} + +func (h *Hearer) free(p seat.Pipeline) { + if err := sayPin(h.Say, delivery.SubjectFreed, p); err != nil { + h.logf("%s does not free its catalog graph revision %d, which the indexer keeps while it is pinned: %v", p.Pipeline, p.Revision, err) + } +} + +// PinRunning says pinned again for every running pipeline at the runner's start, as the gate states its pins at each +// start: pins said before the indexer's consumer existed, or older than the stream keeps, are heard again, and the set +// form absorbs the repeats. It says how many it said. +func (h *Hearer) PinRunning() (int, error) { + pkeys, err := h.Store.Keys("*.*.*") + if err != nil { + return 0, err + } + n := 0 + for _, key := range pkeys { + p, _, err := h.Store.Get(key) + if err != nil || p.Outcome != seat.PipelineOutcomeRunning || !pinned(p) { + continue + } + h.pin(p) + n++ + } + return n, nil +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go new file mode 100644 index 00000000..24448aec --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go @@ -0,0 +1,120 @@ +package pipeline + +import ( + "testing" + + catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + delivery "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" +) + +type saidPin struct { + subject string + pins delivery.Pins +} + +// A-449 (design 47 G §153, §579): a pipeline pins its catalog graph revision as a delivery record does, under the record id +// pipeline., when it starts, again at each start of the runner while it runs, and frees it when it ends, by any +// outcome (superseded included); a run on request, which reads its candidate's revision, pins none. +func TestAPipelinePinsItsCatalogRevisionWhileItRuns(t *testing.T) { + var said []saidPin + say := func(subject string, body any) error { + if p, ok := body.(delivery.Pins); ok { + said = append(said, saidPin{subject, p}) + } + return nil + } + want := func(i int, subject, record string, revision int64) { + t.Helper() + if i >= len(said) { + t.Fatalf("said %d pins, want one more (%s %s %d)", len(said), subject, record, revision) + } + p := said[i] + if p.subject != subject || p.pins.Record != record || len(p.pins.Catalog) != 1 || p.pins.Catalog[0].Branch != "main" || p.pins.Catalog[0].Revision != revision { + t.Errorf("pin %d = %+v, want %s of %s at main %d", i, p, subject, record, revision) + } + } + // At its start. + s := &Starter{Pipelines: created{}, Say: say} + if ok, err := s.Start(Begin{Branch: "main", Source: 7, Module: "a", Commit: "c1", Revision: 12}); !ok || err != nil { + t.Fatalf("start: %v %v", ok, err) + } + want(0, delivery.SubjectPinned, "pipeline.main.7.a", 12) + // A start that names no revision pins nothing; one written already says nothing more. + said = nil + if _, err := s.Start(Begin{Branch: "main", Source: 8, Module: "a", Commit: "c1"}); err != nil || len(said) != 0 { + t.Errorf("a start with no revision said %v (%v)", said, err) + } + if ok, _ := s.Start(Begin{Branch: "main", Source: 7, Module: "a", Commit: "c1", Revision: 12}); ok || len(said) != 0 { + t.Errorf("a start written already said %v", said) + } + // At the runner's start, for the running pipelines alone; a run on request pins none. + h, store, _, _, _, _ := fixture() + h.Say = say + p := store[pkey] + p.p.Revision = 12 + store[pkey] = p + store["main.4.old"] = struct { + p seat.Pipeline + rev uint64 + }{seat.Pipeline{Pipeline: "main.4.old", Branch: "main", Revision: 11, Outcome: seat.PipelineOutcomePassed}, 1} + store["main.r1.app"] = struct { + p seat.Pipeline + rev uint64 + }{seat.Pipeline{Pipeline: "main.r1.app", Branch: "main", Revision: 13, Run: "r1", Outcome: seat.PipelineOutcomeRunning}, 1} + said = nil + if n, err := h.PinRunning(); n != 1 || err != nil { + t.Fatalf("PinRunning said %d (%v), want the running pipeline's alone", n, err) + } + want(0, delivery.SubjectPinned, "pipeline."+pkey, 12) + // Its end frees it, and so does a supersede. + said = nil + if err := h.said(store[pkey].p, false, true); err != nil { + t.Fatal(err) + } + want(0, delivery.SubjectFreed, "pipeline."+pkey, 12) + said = nil + if n, err := h.Supersede(seat.Pipeline{Pipeline: "main.6.app", Branch: "main", Source: 6, Module: "app", Commit: "c2", Fingerprint: "src2:bb", Outcome: seat.PipelineOutcomeRunning}, nil, nil); n != 1 || err != nil { + t.Fatalf("supersede %d (%v)", n, err) + } + want(0, delivery.SubjectFreed, "pipeline."+pkey, 12) +} + +// A-448 (§200, §476): a pipeline of a take-in starts at the revision the take-in made, never at the one the runner's own view +// holds; while the view holds an older one it starts with no step and places none, and the view reaching the revision places +// them. An event that names no revision starts at the view's, as before. +func TestATakeInStartsItsPipelinesAtItsOwnRevisionAndWaitsForTheView(t *testing.T) { + reg, entries := placeFixture() + store := memCreated{memStore{}} + asks := &stepAsks{} + s := &Starter{Pipelines: store, Steps: asks, Name: "the-runner", Say: func(string, any) error { return nil }, + Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" }} + viewRevision := int64(356) + view := func(string) (Half, bool) { return Half{Revision: viewRevision, Entries: entries}, true } + taken := catalog.TakeInTaken{Key: "m.1", Branch: "main", Source: 947, Revision: 357, + Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}} + if n, err := s.TakenIn(taken, view); n != 1 || err != nil { + t.Fatalf("the take-in started %d (%v)", n, err) + } + p, _, _ := store.Get("main.947.app") + if p.Revision != 357 || len(p.Steps) != 0 || len(asks.asks) != 0 { + t.Fatalf("a take-in of revision 357 with the view at 356 started %+v with %d asks, want revision 357 and no step", p, len(asks.asks)) + } + if n, err := s.Waiting(store.memStore, "main", view); n != 0 || err != nil || len(asks.asks) != 0 { + t.Errorf("the view at 356 placed %d (%v) of the pipeline waiting for 357", n, err) + } + viewRevision = 357 + n, err := s.Waiting(store.memStore, "main", view) + p, _, _ = store.Get("main.947.app") + if n != 1 || err != nil || len(p.Steps) == 0 || len(asks.asks) == 0 || p.Revision != 357 { + t.Errorf("the view reaching 357 placed %d (%v): %d steps, %d asks at revision %d", n, err, len(p.Steps), len(asks.asks), p.Revision) + } + // An event of before A-448 names no revision: the view's. + taken = catalog.TakeInTaken{Key: "m.2", Branch: "main", Source: 948, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c2", Fingerprint: "src2:bb"}}} + if n, err := s.TakenIn(taken, view); n != 1 || err != nil { + t.Fatalf("the old take-in started %d (%v)", n, err) + } + if p, _, _ := store.Get("main.948.app"); p.Revision != 357 || len(p.Steps) == 0 { + t.Errorf("a take-in naming no revision started %+v, want the view's revision 357 with its steps", p) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 775552a9..4d7f67c0 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -129,6 +129,7 @@ func (s *Starter) Start(b Begin) (bool, error) { } else if err != nil { return false, fmt.Errorf("the pipeline %s is not written: %w", key, err) } + s.pin(p) // the catalog graph revision it runs on is kept while it runs (A-449) ev := seat.PipelineStarted{V: 1, At: &now, Pipeline: key, Branch: b.Branch, Source: b.Source, Module: b.Module, Commit: b.Commit, Fingerprint: b.Fp, Group: b.Group, Revision: b.Revision, Published: b.Published} commitOrGroup := b.Commit @@ -276,6 +277,10 @@ func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipe s.logf("the catalog half of %s is not held yet: %s waits on it for its steps, placed once it is held", p.Branch, key) continue } + if p.Revision > 0 && half.Revision < p.Revision { + s.logf("the catalog half of %s holds revision %d: %s waits for revision %d, which its take-in made, to place its steps (§200)", p.Branch, half.Revision, key, p.Revision) + continue + } var asks []Asked p.Steps, asks = Place(p, r, half.Entries, now, s.Name) if _, err := store.Update(key, p, rev); err != nil { @@ -431,11 +436,22 @@ func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half // A take-in heard before its branch's half is held starts its pipelines with no steps, never placed from empty entries // (§200, A-336); Waiting places them once the half is held. half, held := view(ev.Branch) + // The pipelines start at the revision the take-in made, never at the one the runner's own view holds (A-448): while the + // view holds an older one they start with no step, which Waiting places once the view reaches it (§200). An event with + // no revision (said before A-448) starts at the view's, as it did, and says so. + revision := ev.Revision + if revision <= 0 { + revision = half.Revision + s.logf("the take-in %s of %s names no catalog graph revision: its pipelines start at the revision the runner's view holds, %d", ev.Key, ev.Branch, revision) + } else if held && half.Revision < revision { + s.logf("the take-in %s of %s made revision %d and the runner's view holds %d: its pipelines start with no step, placed once the view reaches it (§200)", ev.Key, ev.Branch, revision, half.Revision) + held = false + } started := 0 var first error for _, e := range ev.Entries { ok, err := s.Start(Begin{Branch: ev.Branch, Source: ev.Source, Module: e.Module, Commit: e.Commit, Fp: e.Fingerprint, - Group: ev.Group, Repository: e.Repository, Revision: half.Revision, Entries: half.Entries, unheld: !held}) + Group: ev.Group, Repository: e.Repository, Revision: revision, Entries: half.Entries, unheld: !held}) if err != nil && first == nil { first = err } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go index ac247826..b8a999e1 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -8,6 +8,7 @@ import ( builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + delivery "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" ) @@ -29,7 +30,10 @@ func TestStarts(t *testing.T) { var said []string var superseding []string s := &Starter{Pipelines: store, Say: func(subject string, body any) error { - ev := body.(seat.PipelineStarted) + ev, started := body.(seat.PipelineStarted) + if !started { // a pipeline's pin of its catalog graph revision (A-449) is no pipeline-started + return nil + } said = append(said, subject+" "+ev.Pipeline) if ev.Published == nil || ev.Published.Artifact != "mesh-sdk:go" || ev.Published.Version != "0.1.16" { t.Errorf("pipeline-started of %s says the publish %+v (A-202)", ev.Pipeline, ev.Published) @@ -118,7 +122,12 @@ func (failing) Create(string, []byte) error { return errors.New("bus away") } func TestTakenInStarts(t *testing.T) { store := created{} var said []string - s := &Starter{Pipelines: store, Say: func(subject string, body any) error { said = append(said, subject); return nil }} + s := &Starter{Pipelines: store, Say: func(subject string, body any) error { + if _, pins := body.(delivery.Pins); !pins { + said = append(said, subject) + } + return nil + }} view := func(string) (Half, bool) { return Half{Revision: 12}, true } ev := catalog.TakeInTaken{Kind: "taken", Branch: "main", Source: 7, Group: "g1", Entries: []catalog.TakeInEntry{ {Module: "a", Commit: "c1", Fingerprint: "src2:aa", Repository: "o/a"}, {Module: "b", Commit: "c2", Fingerprint: "src2:bb"}}} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index e8518f84..43d3781d 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -225,6 +225,7 @@ func (h *Hearer) said(p seat.Pipeline, asked, ended bool) error { } } if ended { + h.free(p) // the revision it ran on is no longer kept for it (A-449) if err := h.say(seat.EventPipelineEnded, p, nil); err != nil { return fmt.Errorf("pipeline %s ended but %s is not said: %w", p.Pipeline, seat.EventPipelineEnded, err) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go index a729b552..dc4d472f 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go @@ -127,6 +127,7 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop } else if err != nil { return false, fmt.Errorf("pipeline %s cannot be written: %w", key, err) } + h.free(p) // a superseded pipeline ends too (A-449) var first error for _, i := range changed { if err := h.say(seat.EventStepEnded, p, &p.Steps[i]); err != nil && first == nil { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go index a0118b68..6966b6f6 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -286,6 +286,7 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error if err != nil { return seat.RerunAnswer{}, err } + v.Hearer.pin(p) // a pipeline that runs again keeps its revision again (A-449) if ask != nil { v.Starter.carry(in.Pipeline, []Asked{*ask}) if err := askAll(v.Starter.Steps, v.Starter.Arm, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil { @@ -423,5 +424,6 @@ func (v Verbs) AskAgain(caller string, in seat.AskAgainInput) (seat.AskAgainAnsw if err != nil { return seat.AskAgainAnswer{}, err } + v.Hearer.pin(p) return seat.AskAgainAnswer{Result: p.Steps[i]}, nil }