pipeline runner: a take-in's pipelines start at its own catalog revision, wait for the view, and pin the revision while they run (A-448, A-449)

Builds: design 47 G (G-22 lead answers 2) A-448, A-449; §153, §200, §476, §579
This commit is contained in:
2026-11-16 13:38:57 +01:00
parent d0efbee65b
commit bbcad0845a
9 changed files with 245 additions and 5 deletions
@@ -125,6 +125,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
hearer := newHearer(ctx, js, cred, pipelines, kept, say, logger.Printf)
hearer.Asker = busAsker{conn: conn, cred: cred}
// A running pipeline keeps the catalog graph revision it runs on (A-449): said again at each start, as the gate states its
// pins, for pins said before the indexer's consumer existed.
if n, err := hearer.PinRunning(); err != nil {
logger.Printf("the running pipelines' pins are not said: %v", err)
} else {
logger.Printf("pinned the catalog graph revision of %d running pipelines", n)
}
hearer.Ager = newAger(ctx, js, cred, pipelines, kept, aged, logger.Printf)
// The stores' fill is said before a store is full, and a full one refuses a start (§366).
fills := &pipeline.Fills{Read: storeFills(ctx, js, cred, time.Now), Logf: logger.Printf,
@@ -49,13 +49,30 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cr
// is picked up as readiness is: its pipelines started with no steps are placed then (A-327, A-336).
var wired atomic.Pointer[meshcatalogclient.Catalog]
var held atomic.Bool
var heldRevision atomic.Int64 // the newest revision the view was ready with (A-448)
c, _, err := meshcatalogclient.WireCatalogProcess(reader, call, branch, func(string, json.RawMessage, bool, time.Time) {
c := wired.Load()
if c == nil || onReady == nil {
return
}
if _, ok := c.View(); ok && !held.Swap(true) {
go onReady(branch)
// Ready on the first half held and again on each newer revision: a pipeline that waits for the revision its take-in
// made (A-448, §200) is placed when the view reaches it.
if h, ok := c.View(); ok {
first := !held.Swap(true)
advanced := false
for rev := h.Revision.Revision; ; {
old := heldRevision.Load()
if rev <= old {
break
}
if heldRevision.CompareAndSwap(old, rev) {
advanced = true
break
}
}
if first || advanced {
go onReady(branch)
}
}
})
if err != nil {
@@ -0,0 +1,67 @@
package pipeline
import (
"strconv"
delivery "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// PinRecord is the record id a pipeline pins its catalog graph revision under: `pipeline.<key>` (design 47 G §153, §579,
// A-449), as a delivery record pins its revisions under its own id, so the indexer keeps the revision while any pipeline
// runs on it.
func PinRecord(key string) string { return "pipeline." + key }
// pinned says whether a pipeline pins a revision: a branch's pipeline that names its catalog graph revision. A run on
// request reads its candidate's revision, which the indexer keeps while the candidate is open, never a branch's.
func pinned(p seat.Pipeline) bool { return p.Run == "" && p.Branch != "" && p.Revision > 0 }
// sayPin says the seat mesh-delivery's pinned or freed for p's catalog graph revision, in the form the gate says them and
// the indexer applies them (a set of record ids per revision, so a repeat changes nothing).
func sayPin(say func(subject string, body any) error, subject string, p seat.Pipeline) error {
if say == nil || !pinned(p) {
return nil
}
return say(subject, delivery.Pins{Record: PinRecord(p.Pipeline), Milestone: "pipeline " + strconv.FormatInt(p.Source, 10),
Catalog: []delivery.PinsCatalogItem{{Branch: p.Branch, Revision: p.Revision}}})
}
// Pin says pinned for the catalog graph revision of a running pipeline (A-449).
func (s *Starter) pin(p seat.Pipeline) {
if err := sayPin(s.Say, delivery.SubjectPinned, p); err != nil {
s.logf("%s does not pin its catalog graph revision %d, so the indexer may remove it: %v", p.Pipeline, p.Revision, err)
}
}
// pin and free say it for a pipeline the hearer reopens or ends (A-449).
func (h *Hearer) pin(p seat.Pipeline) {
if err := sayPin(h.Say, delivery.SubjectPinned, p); err != nil {
h.logf("%s does not pin its catalog graph revision %d, so the indexer may remove it: %v", p.Pipeline, p.Revision, err)
}
}
func (h *Hearer) free(p seat.Pipeline) {
if err := sayPin(h.Say, delivery.SubjectFreed, p); err != nil {
h.logf("%s does not free its catalog graph revision %d, which the indexer keeps while it is pinned: %v", p.Pipeline, p.Revision, err)
}
}
// PinRunning says pinned again for every running pipeline at the runner's start, as the gate states its pins at each
// start: pins said before the indexer's consumer existed, or older than the stream keeps, are heard again, and the set
// form absorbs the repeats. It says how many it said.
func (h *Hearer) PinRunning() (int, error) {
pkeys, err := h.Store.Keys("*.*.*")
if err != nil {
return 0, err
}
n := 0
for _, key := range pkeys {
p, _, err := h.Store.Get(key)
if err != nil || p.Outcome != seat.PipelineOutcomeRunning || !pinned(p) {
continue
}
h.pin(p)
n++
}
return n, nil
}
@@ -0,0 +1,120 @@
package pipeline
import (
"testing"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
delivery "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
type saidPin struct {
subject string
pins delivery.Pins
}
// A-449 (design 47 G §153, §579): a pipeline pins its catalog graph revision as a delivery record does, under the record id
// pipeline.<key>, when it starts, again at each start of the runner while it runs, and frees it when it ends, by any
// outcome (superseded included); a run on request, which reads its candidate's revision, pins none.
func TestAPipelinePinsItsCatalogRevisionWhileItRuns(t *testing.T) {
var said []saidPin
say := func(subject string, body any) error {
if p, ok := body.(delivery.Pins); ok {
said = append(said, saidPin{subject, p})
}
return nil
}
want := func(i int, subject, record string, revision int64) {
t.Helper()
if i >= len(said) {
t.Fatalf("said %d pins, want one more (%s %s %d)", len(said), subject, record, revision)
}
p := said[i]
if p.subject != subject || p.pins.Record != record || len(p.pins.Catalog) != 1 || p.pins.Catalog[0].Branch != "main" || p.pins.Catalog[0].Revision != revision {
t.Errorf("pin %d = %+v, want %s of %s at main %d", i, p, subject, record, revision)
}
}
// At its start.
s := &Starter{Pipelines: created{}, Say: say}
if ok, err := s.Start(Begin{Branch: "main", Source: 7, Module: "a", Commit: "c1", Revision: 12}); !ok || err != nil {
t.Fatalf("start: %v %v", ok, err)
}
want(0, delivery.SubjectPinned, "pipeline.main.7.a", 12)
// A start that names no revision pins nothing; one written already says nothing more.
said = nil
if _, err := s.Start(Begin{Branch: "main", Source: 8, Module: "a", Commit: "c1"}); err != nil || len(said) != 0 {
t.Errorf("a start with no revision said %v (%v)", said, err)
}
if ok, _ := s.Start(Begin{Branch: "main", Source: 7, Module: "a", Commit: "c1", Revision: 12}); ok || len(said) != 0 {
t.Errorf("a start written already said %v", said)
}
// At the runner's start, for the running pipelines alone; a run on request pins none.
h, store, _, _, _, _ := fixture()
h.Say = say
p := store[pkey]
p.p.Revision = 12
store[pkey] = p
store["main.4.old"] = struct {
p seat.Pipeline
rev uint64
}{seat.Pipeline{Pipeline: "main.4.old", Branch: "main", Revision: 11, Outcome: seat.PipelineOutcomePassed}, 1}
store["main.r1.app"] = struct {
p seat.Pipeline
rev uint64
}{seat.Pipeline{Pipeline: "main.r1.app", Branch: "main", Revision: 13, Run: "r1", Outcome: seat.PipelineOutcomeRunning}, 1}
said = nil
if n, err := h.PinRunning(); n != 1 || err != nil {
t.Fatalf("PinRunning said %d (%v), want the running pipeline's alone", n, err)
}
want(0, delivery.SubjectPinned, "pipeline."+pkey, 12)
// Its end frees it, and so does a supersede.
said = nil
if err := h.said(store[pkey].p, false, true); err != nil {
t.Fatal(err)
}
want(0, delivery.SubjectFreed, "pipeline."+pkey, 12)
said = nil
if n, err := h.Supersede(seat.Pipeline{Pipeline: "main.6.app", Branch: "main", Source: 6, Module: "app", Commit: "c2", Fingerprint: "src2:bb", Outcome: seat.PipelineOutcomeRunning}, nil, nil); n != 1 || err != nil {
t.Fatalf("supersede %d (%v)", n, err)
}
want(0, delivery.SubjectFreed, "pipeline."+pkey, 12)
}
// A-448 (§200, §476): a pipeline of a take-in starts at the revision the take-in made, never at the one the runner's own view
// holds; while the view holds an older one it starts with no step and places none, and the view reaching the revision places
// them. An event that names no revision starts at the view's, as before.
func TestATakeInStartsItsPipelinesAtItsOwnRevisionAndWaitsForTheView(t *testing.T) {
reg, entries := placeFixture()
store := memCreated{memStore{}}
asks := &stepAsks{}
s := &Starter{Pipelines: store, Steps: asks, Name: "the-runner", Say: func(string, any) error { return nil },
Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" }}
viewRevision := int64(356)
view := func(string) (Half, bool) { return Half{Revision: viewRevision, Entries: entries}, true }
taken := catalog.TakeInTaken{Key: "m.1", Branch: "main", Source: 947, Revision: 357,
Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}
if n, err := s.TakenIn(taken, view); n != 1 || err != nil {
t.Fatalf("the take-in started %d (%v)", n, err)
}
p, _, _ := store.Get("main.947.app")
if p.Revision != 357 || len(p.Steps) != 0 || len(asks.asks) != 0 {
t.Fatalf("a take-in of revision 357 with the view at 356 started %+v with %d asks, want revision 357 and no step", p, len(asks.asks))
}
if n, err := s.Waiting(store.memStore, "main", view); n != 0 || err != nil || len(asks.asks) != 0 {
t.Errorf("the view at 356 placed %d (%v) of the pipeline waiting for 357", n, err)
}
viewRevision = 357
n, err := s.Waiting(store.memStore, "main", view)
p, _, _ = store.Get("main.947.app")
if n != 1 || err != nil || len(p.Steps) == 0 || len(asks.asks) == 0 || p.Revision != 357 {
t.Errorf("the view reaching 357 placed %d (%v): %d steps, %d asks at revision %d", n, err, len(p.Steps), len(asks.asks), p.Revision)
}
// An event of before A-448 names no revision: the view's.
taken = catalog.TakeInTaken{Key: "m.2", Branch: "main", Source: 948, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c2", Fingerprint: "src2:bb"}}}
if n, err := s.TakenIn(taken, view); n != 1 || err != nil {
t.Fatalf("the old take-in started %d (%v)", n, err)
}
if p, _, _ := store.Get("main.948.app"); p.Revision != 357 || len(p.Steps) == 0 {
t.Errorf("a take-in naming no revision started %+v, want the view's revision 357 with its steps", p)
}
}
@@ -129,6 +129,7 @@ func (s *Starter) Start(b Begin) (bool, error) {
} else if err != nil {
return false, fmt.Errorf("the pipeline %s is not written: %w", key, err)
}
s.pin(p) // the catalog graph revision it runs on is kept while it runs (A-449)
ev := seat.PipelineStarted{V: 1, At: &now, Pipeline: key, Branch: b.Branch, Source: b.Source, Module: b.Module,
Commit: b.Commit, Fingerprint: b.Fp, Group: b.Group, Revision: b.Revision, Published: b.Published}
commitOrGroup := b.Commit
@@ -276,6 +277,10 @@ func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipe
s.logf("the catalog half of %s is not held yet: %s waits on it for its steps, placed once it is held", p.Branch, key)
continue
}
if p.Revision > 0 && half.Revision < p.Revision {
s.logf("the catalog half of %s holds revision %d: %s waits for revision %d, which its take-in made, to place its steps (§200)", p.Branch, half.Revision, key, p.Revision)
continue
}
var asks []Asked
p.Steps, asks = Place(p, r, half.Entries, now, s.Name)
if _, err := store.Update(key, p, rev); err != nil {
@@ -431,11 +436,22 @@ func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half
// A take-in heard before its branch's half is held starts its pipelines with no steps, never placed from empty entries
// (§200, A-336); Waiting places them once the half is held.
half, held := view(ev.Branch)
// The pipelines start at the revision the take-in made, never at the one the runner's own view holds (A-448): while the
// view holds an older one they start with no step, which Waiting places once the view reaches it (§200). An event with
// no revision (said before A-448) starts at the view's, as it did, and says so.
revision := ev.Revision
if revision <= 0 {
revision = half.Revision
s.logf("the take-in %s of %s names no catalog graph revision: its pipelines start at the revision the runner's view holds, %d", ev.Key, ev.Branch, revision)
} else if held && half.Revision < revision {
s.logf("the take-in %s of %s made revision %d and the runner's view holds %d: its pipelines start with no step, placed once the view reaches it (§200)", ev.Key, ev.Branch, revision, half.Revision)
held = false
}
started := 0
var first error
for _, e := range ev.Entries {
ok, err := s.Start(Begin{Branch: ev.Branch, Source: ev.Source, Module: e.Module, Commit: e.Commit, Fp: e.Fingerprint,
Group: ev.Group, Repository: e.Repository, Revision: half.Revision, Entries: half.Entries, unheld: !held})
Group: ev.Group, Repository: e.Repository, Revision: revision, Entries: half.Entries, unheld: !held})
if err != nil && first == nil {
first = err
}
@@ -8,6 +8,7 @@ import (
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
delivery "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
@@ -29,7 +30,10 @@ func TestStarts(t *testing.T) {
var said []string
var superseding []string
s := &Starter{Pipelines: store, Say: func(subject string, body any) error {
ev := body.(seat.PipelineStarted)
ev, started := body.(seat.PipelineStarted)
if !started { // a pipeline's pin of its catalog graph revision (A-449) is no pipeline-started
return nil
}
said = append(said, subject+" "+ev.Pipeline)
if ev.Published == nil || ev.Published.Artifact != "mesh-sdk:go" || ev.Published.Version != "0.1.16" {
t.Errorf("pipeline-started of %s says the publish %+v (A-202)", ev.Pipeline, ev.Published)
@@ -118,7 +122,12 @@ func (failing) Create(string, []byte) error { return errors.New("bus away") }
func TestTakenInStarts(t *testing.T) {
store := created{}
var said []string
s := &Starter{Pipelines: store, Say: func(subject string, body any) error { said = append(said, subject); return nil }}
s := &Starter{Pipelines: store, Say: func(subject string, body any) error {
if _, pins := body.(delivery.Pins); !pins {
said = append(said, subject)
}
return nil
}}
view := func(string) (Half, bool) { return Half{Revision: 12}, true }
ev := catalog.TakeInTaken{Kind: "taken", Branch: "main", Source: 7, Group: "g1", Entries: []catalog.TakeInEntry{
{Module: "a", Commit: "c1", Fingerprint: "src2:aa", Repository: "o/a"}, {Module: "b", Commit: "c2", Fingerprint: "src2:bb"}}}
@@ -225,6 +225,7 @@ func (h *Hearer) said(p seat.Pipeline, asked, ended bool) error {
}
}
if ended {
h.free(p) // the revision it ran on is no longer kept for it (A-449)
if err := h.say(seat.EventPipelineEnded, p, nil); err != nil {
return fmt.Errorf("pipeline %s ended but %s is not said: %w", p.Pipeline, seat.EventPipelineEnded, err)
}
@@ -127,6 +127,7 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop
} else if err != nil {
return false, fmt.Errorf("pipeline %s cannot be written: %w", key, err)
}
h.free(p) // a superseded pipeline ends too (A-449)
var first error
for _, i := range changed {
if err := h.say(seat.EventStepEnded, p, &p.Steps[i]); err != nil && first == nil {
@@ -286,6 +286,7 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
if err != nil {
return seat.RerunAnswer{}, err
}
v.Hearer.pin(p) // a pipeline that runs again keeps its revision again (A-449)
if ask != nil {
v.Starter.carry(in.Pipeline, []Asked{*ask})
if err := askAll(v.Starter.Steps, v.Starter.Arm, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil {
@@ -423,5 +424,6 @@ func (v Verbs) AskAgain(caller string, in seat.AskAgainInput) (seat.AskAgainAnsw
if err != nil {
return seat.AskAgainAnswer{}, err
}
v.Hearer.pin(p)
return seat.AskAgainAnswer{Result: p.Steps[i]}, nil
}