diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index a6c3c08d..a599cf88 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -1170,7 +1170,8 @@ func (ix *Indexer) takeInWholeBy(ctx context.Context, entries []PendingEntry, ow ix.forget(claimed...) branch, src, took := ix.takenWhat(ctx, claimed, applied) ix.logf("take-in: %d merges taken in as one revision by %s%s, oldest %s: source revision %d of %s, %d entries", len(claimed), own, byWords(by), claimed[0].Key, src, branch, len(took)) - revision := ix.NewestRevision(ctx, branch) // the revision this take-in just kept (A-448) + revision := ix.NewestRevision(ctx, branch) // the revision this take-in just kept (A-448) + ix.PinPipelines(ctx, branch, revision, src, took) // kept for its pipelines before any later take-in can remove it (A-449a) for _, t := range takenWaits(claimed) { t.By, t.Why = who, why t.Branch, t.Source, t.Entries, t.Revision = branch, src, took, revision diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index b051c2d4..6f8b9c74 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -7,6 +7,7 @@ import ( "fmt" "reflect" "slices" + "strconv" "strings" "testing" "time" @@ -1000,6 +1001,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) !strings.Contains(r.Answer, TakeInPendingTool) || len(b.keys()) != 1 { t.Fatalf("a drop nothing covers: %+v, pending %v", r, b.keys()) } + b.ix.Pins = newMem() b.ix.Revisions = newMemObjects(0) // A-448: the taken event names the catalog graph revision the take-in made b.said = nil if r := b.ix.TakeInPendingByHand(ctx, takein.PendingKey(3), "lead@shanks", "the named indexer can never run"); !r.Stored { @@ -1016,6 +1018,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) t.Errorf("hand take-in said branch %q source %d entries %+v: want main, a source revision and avahi at its commit", got.Branch, got.Source, got.Entries) } else if newest, any, _ := newestTrunk(ctx, b.ix.Revisions); !any || got.Revision != newest.Revision { t.Errorf("hand take-in said revision %d, want the revision it made, %d (A-448)", got.Revision, newest.Revision) + } else if v, _, found, _ := b.ix.Pins.Get(ctx, PinKey(got.Branch, got.Revision)); !found || !strings.Contains(string(v), "pipeline."+got.Branch+"."+strconv.FormatInt(got.Source, 10)+".avahi") { + t.Errorf("the revision the take-in made is pinned %q (found %v), want the record pipeline.%s.%d.avahi (A-449a)", v, found, got.Branch, got.Source) } else if _, err := takein.EncodeTakeIn(got); err != nil { t.Errorf("the taken event is refused by its own contract: %v", err) } diff --git a/modules/mesh-delivery-indexer/internal/index/pins.go b/modules/mesh-delivery-indexer/internal/index/pins.go index cb49a310..bd3ce994 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins.go +++ b/modules/mesh-delivery-indexer/internal/index/pins.go @@ -14,6 +14,7 @@ import ( mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/process" @@ -573,3 +574,24 @@ func ConsumerTools(st Store) []stdio.Tool { Input: map[string]any{"type": "object", "properties": map[string]any{}}, Run: func(map[string]any) (any, error) { return ReadConsumers(context.Background(), st) }}} } + +// PinPipelines pins the catalog graph revision a take-in made for each pipeline the take-in starts, under the record +// `pipeline.` (`..` of each entry, the runner's key), in the indexer's own pins state as a +// pinned of the seat mesh-delivery is applied (design 47 G §579, A-449a): so no later take-in removes the revision before +// the runner hears of it. The runner's pinned at the pipeline's start says the same record (a set), and its freed at the +// pipeline's end removes it. +func (ix *Indexer) PinPipelines(ctx context.Context, branch string, revision, source int64, entries []meshcatalog.TakeInEntry) { + if ix.Pins == nil || revision <= 0 || len(entries) == 0 { + return + } + if err := ix.PinsMade(ctx); err != nil { + ix.logf("pins: the revision %d of %s is not pinned for its pipelines: %v", revision, branch, err) + return + } + for _, e := range entries { + record := "pipeline." + runner.PipelinesByBranchRevisionModuleKey(keys.EscapeName(branch), strconv.FormatInt(source, 10), e.Module) + if err := ix.pin(ctx, PinKey(branch, revision), record); err != nil { + ix.logf("pins: the revision %d of %s is not pinned for %s: %v", revision, branch, record, err) + } + } +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index e1650118..ef8ad6cf 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -95,12 +95,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // A branch's view holding its first half asks again the steps placed in its running pipelines that were never asked // (§88); the starter is made after the views, so it is reached through a pointer set once it is. var resumer atomic.Pointer[func(string)] + var advancer atomic.Pointer[func(string)] var placer atomic.Pointer[func(catalog.CatalogRevisionEvent) error] var registrar atomic.Pointer[func(catalog.Registration)] rv, err := startViews(ctx, js, conn, cred, hub, calls.State("findings"), calls.Call, func(branch string) { if f := resumer.Load(); f != nil { (*f)(branch) } + }, func(branch string) { + if f := advancer.Load(); f != nil { + (*f)(branch) + } }, func(_ context.Context, ev catalog.CatalogRevisionEvent) error { if f := placer.Load(); f != nil { return (*f)(ev) @@ -170,6 +175,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro logger.Printf("the steps placed on %s and not yet asked: %d asked again", branch, n) } resumer.Store(&resume) + // The view reaching a newer revision places the steps of the pipelines that waited for it, and asks nothing again (A-448, §200). + advance := func(branch string) { + if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 { + logger.Printf("the pipelines of %s that waited for a newer revision of its half: %d placed (%v)", branch, n, err) + } + } + advancer.Store(&advance) // A registration-changed places the steps of its module's pipelines started before it, from it (A-327). register := func(reg catalog.Registration) { if n, err := starter.Registered(pipelineStore{ctx: ctx, kv: pipelines}, reg, rv.Catalog.View); err != nil || n > 0 { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go index ccb3ed09..dac58bb4 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go @@ -31,7 +31,7 @@ type runnerViews struct { // startViews wires the runner's views on hub and asks each first view; register every other handler on hub before or // after, then start hub once. func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cred process.Account, hub *process.EventHub, findingsState process.State, - call meshcatalogclient.Call, onReady func(branch string), candidate func(ctx context.Context, ev catalog.CatalogRevisionEvent) error, + call meshcatalogclient.Call, onReady, onAdvance func(branch string), candidate func(ctx context.Context, ev catalog.CatalogRevisionEvent) error, registered func(reg catalog.Registration), logf func(string, ...any)) (*runnerViews, error) { findings, err := findingWriter(ctx, cred, conn, js, findingsState, logf) if err != nil { @@ -48,30 +48,21 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cr // A branch whose first view was none yet is ready with no half; the first half held later, from a catalog-revision, // is picked up as readiness is: its pipelines started with no steps are placed then (A-327, A-336). var wired atomic.Pointer[meshcatalogclient.Catalog] - var held atomic.Bool - var heldRevision atomic.Int64 // the newest revision the view was ready with (A-448) + var ready pipeline.Readiness c, _, err := meshcatalogclient.WireCatalogProcess(reader, call, branch, func(string, json.RawMessage, bool, time.Time) { c := wired.Load() if c == nil || onReady == nil { return } - // Ready on the first half held and again on each newer revision: a pipeline that waits for the revision its take-in - // made (A-448, §200) is placed when the view reaches it. + // Ready on the first half held (the steps placed are asked again, §88); a newer revision only places the steps of + // the pipelines that waited for it (A-448, §200). if h, ok := c.View(); ok { - first := !held.Swap(true) - advanced := false - for rev := h.Revision.Revision; ; { - old := heldRevision.Load() - if rev <= old { - break - } - if heldRevision.CompareAndSwap(old, rev) { - advanced = true - break - } - } - if first || advanced { + first, advanced := ready.Observe(h.Revision.Revision) + switch { + case first: go onReady(branch) + case advanced && onAdvance != nil: + go onAdvance(branch) } } }) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go index 24448aec..b6a330b8 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go @@ -78,6 +78,9 @@ func TestAPipelinePinsItsCatalogRevisionWhileItRuns(t *testing.T) { t.Fatalf("supersede %d (%v)", n, err) } want(0, delivery.SubjectFreed, "pipeline."+pkey, 12) + if len(said) != 1 { + t.Errorf("a supersede said %d pins, want the one freed (through its end, once)", len(said)) + } } // A-448 (§200, §476): a pipeline of a take-in starts at the revision the take-in made, never at the one the runner's own view @@ -118,3 +121,61 @@ func TestATakeInStartsItsPipelinesAtItsOwnRevisionAndWaitsForTheView(t *testing. t.Errorf("a take-in naming no revision started %+v, want the view's revision 357 with its steps", p) } } + +// A-448: the view is read before the starts write; a view that reaches the revision meanwhile places the held-back pipelines at +// once, not at an advance that already came. +func TestAViewThatReachesTheRevisionDuringTheStartsPlacesTheHeldBackPipelines(t *testing.T) { + reg, entries := placeFixture() + store := memCreated{memStore{}} + asks := &stepAsks{} + s := &Starter{Pipelines: store, Steps: asks, Name: "the-runner", Say: func(string, any) error { return nil }, + Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" }} + s.Store = store.memStore + reads := 0 + view := func(string) (Half, bool) { + reads++ + if reads > 1 { // the first read is before the starts wrote; the view reaches 357 after it + return Half{Revision: 357, Entries: entries}, true + } + return Half{Revision: 356, Entries: entries}, true + } + taken := catalog.TakeInTaken{Key: "m.1", Branch: "main", Source: 947, Revision: 357, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}} + if n, err := s.TakenIn(taken, view); n != 1 || err != nil { + t.Fatalf("started %d (%v)", n, err) + } + if p, _, _ := store.Get("main.947.app"); len(p.Steps) == 0 || len(asks.asks) == 0 { + t.Errorf("the pipeline has %d steps and %d asks after the view reached 357 during the start, want them placed", len(p.Steps), len(asks.asks)) + } +} + +// A-448 (§200): the first half a view holds is Ready, which asks again the steps placed (§88); a newer revision is an advance, +// which only places the steps of pipelines that waited for it and asks nothing again. +func TestReadinessSaysTheFirstHoldAndEachNewerRevisionApart(t *testing.T) { + var r Readiness + for i, c := range []struct { + rev int64 + first, advanced bool + }{{356, true, false}, {356, false, false}, {355, false, false}, {357, false, true}, {357, false, false}, {360, false, true}} { + if first, advanced := r.Observe(c.rev); first != c.first || advanced != c.advanced { + t.Errorf("observation %d of revision %d: first %v advanced %v, want %v %v", i, c.rev, first, advanced, c.first, c.advanced) + } + } +} + +// A-448: placing the steps of a pipeline that waited asks nothing again of the running steps of the others. +func TestPlacingAWaitingPipelineAsksNoRunningStepAgain(t *testing.T) { + reg, entries := placeFixture() + store := memCreated{memStore{}} + asks := &stepAsks{} + s := &Starter{Pipelines: store, Steps: asks, Name: "the-runner", Say: func(string, any) error { return nil }, + Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" }} + view := func(string) (Half, bool) { return Half{Revision: 357, Entries: entries}, true } + if n, err := s.TakenIn(catalog.TakeInTaken{Key: "m.1", Branch: "main", Source: 1, Revision: 357, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}, view); n != 1 || err != nil { + t.Fatalf("started %d (%v)", n, err) + } + before := len(asks.asks) + late := func(string) (Half, bool) { return Half{Revision: 358, Entries: entries}, true } + if n, err := s.Waiting(store.memStore, "main", late); n != 0 || err != nil || len(asks.asks) != before { + t.Errorf("an advance placed %d (%v) and asked %d steps again, want nothing", n, err, len(asks.asks)-before) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go b/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go index 91073caf..a2a2db7b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go @@ -187,3 +187,28 @@ func (v *Views) HearRevision(ctx context.Context, data []byte) error { } return first } + +// Readiness tells, from the revisions a branch's view is seen holding, when the view first holds a half and when it reaches +// a newer revision: the first hold asks again the steps already placed (§88's restart pickup), a newer revision only places +// the steps of pipelines that waited for it (A-448, §200). It is safe for concurrent use. +type Readiness struct { + mu sync.Mutex + held bool + rev int64 +} + +// Observe notes that the view holds revision rev: first is true once, at the first hold; advanced is true when rev is newer +// than any seen before and it is not the first. +func (r *Readiness) Observe(rev int64) (first, advanced bool) { + r.mu.Lock() + defer r.mu.Unlock() + if !r.held { + r.held, r.rev = true, rev + return true, false + } + if rev > r.rev { + r.rev = rev + return false, true + } + return false, false +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 4d7f67c0..b95dfc29 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -459,6 +459,15 @@ func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half started++ } } + // The view was read before the starts wrote: when pipelines were held back for it and it has reached the revision since, + // they are placed now, not left for an advance that already came (A-448, §200). + if !held && started > 0 && revision > 0 && s.Store != nil { + if again, ok := view(ev.Branch); ok && again.Revision >= revision { + if _, err := s.Waiting(s.Store, ev.Branch, view); err != nil && first == nil { + first = err + } + } + } return started, first } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go index dc4d472f..a729b552 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go @@ -127,7 +127,6 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop } else if err != nil { return false, fmt.Errorf("pipeline %s cannot be written: %w", key, err) } - h.free(p) // a superseded pipeline ends too (A-449) var first error for _, i := range changed { if err := h.say(seat.EventStepEnded, p, &p.Steps[i]); err != nil && first == nil {