pipeline runner and indexer: review fixes (advance only places waiting steps, view re-read after the starts, one freed) and the indexer pins a take-in's revision for its pipelines (A-448, A-449a)

Builds: design 47 G (G-22 lead answers 2) A-448, A-449, A-449a; §153, §200, §579
This commit is contained in:
2026-11-16 13:46:17 +01:00
parent bbcad0845a
commit 5632924c4f
9 changed files with 144 additions and 20 deletions
@@ -1170,7 +1170,8 @@ func (ix *Indexer) takeInWholeBy(ctx context.Context, entries []PendingEntry, ow
ix.forget(claimed...)
branch, src, took := ix.takenWhat(ctx, claimed, applied)
ix.logf("take-in: %d merges taken in as one revision by %s%s, oldest %s: source revision %d of %s, %d entries", len(claimed), own, byWords(by), claimed[0].Key, src, branch, len(took))
revision := ix.NewestRevision(ctx, branch) // the revision this take-in just kept (A-448)
revision := ix.NewestRevision(ctx, branch) // the revision this take-in just kept (A-448)
ix.PinPipelines(ctx, branch, revision, src, took) // kept for its pipelines before any later take-in can remove it (A-449a)
for _, t := range takenWaits(claimed) {
t.By, t.Why = who, why
t.Branch, t.Source, t.Entries, t.Revision = branch, src, took, revision
@@ -7,6 +7,7 @@ import (
"fmt"
"reflect"
"slices"
"strconv"
"strings"
"testing"
"time"
@@ -1000,6 +1001,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
!strings.Contains(r.Answer, TakeInPendingTool) || len(b.keys()) != 1 {
t.Fatalf("a drop nothing covers: %+v, pending %v", r, b.keys())
}
b.ix.Pins = newMem()
b.ix.Revisions = newMemObjects(0) // A-448: the taken event names the catalog graph revision the take-in made
b.said = nil
if r := b.ix.TakeInPendingByHand(ctx, takein.PendingKey(3), "lead@shanks", "the named indexer can never run"); !r.Stored {
@@ -1016,6 +1018,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Errorf("hand take-in said branch %q source %d entries %+v: want main, a source revision and avahi at its commit", got.Branch, got.Source, got.Entries)
} else if newest, any, _ := newestTrunk(ctx, b.ix.Revisions); !any || got.Revision != newest.Revision {
t.Errorf("hand take-in said revision %d, want the revision it made, %d (A-448)", got.Revision, newest.Revision)
} else if v, _, found, _ := b.ix.Pins.Get(ctx, PinKey(got.Branch, got.Revision)); !found || !strings.Contains(string(v), "pipeline."+got.Branch+"."+strconv.FormatInt(got.Source, 10)+".avahi") {
t.Errorf("the revision the take-in made is pinned %q (found %v), want the record pipeline.%s.%d.avahi (A-449a)", v, found, got.Branch, got.Source)
} else if _, err := takein.EncodeTakeIn(got); err != nil {
t.Errorf("the taken event is refused by its own contract: %v", err)
}
@@ -14,6 +14,7 @@ import (
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
@@ -573,3 +574,24 @@ func ConsumerTools(st Store) []stdio.Tool {
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) { return ReadConsumers(context.Background(), st) }}}
}
// PinPipelines pins the catalog graph revision a take-in made for each pipeline the take-in starts, under the record
// `pipeline.<key>` (`<branch>.<source>.<module>` of each entry, the runner's key), in the indexer's own pins state as a
// pinned of the seat mesh-delivery is applied (design 47 G §579, A-449a): so no later take-in removes the revision before
// the runner hears of it. The runner's pinned at the pipeline's start says the same record (a set), and its freed at the
// pipeline's end removes it.
func (ix *Indexer) PinPipelines(ctx context.Context, branch string, revision, source int64, entries []meshcatalog.TakeInEntry) {
if ix.Pins == nil || revision <= 0 || len(entries) == 0 {
return
}
if err := ix.PinsMade(ctx); err != nil {
ix.logf("pins: the revision %d of %s is not pinned for its pipelines: %v", revision, branch, err)
return
}
for _, e := range entries {
record := "pipeline." + runner.PipelinesByBranchRevisionModuleKey(keys.EscapeName(branch), strconv.FormatInt(source, 10), e.Module)
if err := ix.pin(ctx, PinKey(branch, revision), record); err != nil {
ix.logf("pins: the revision %d of %s is not pinned for %s: %v", revision, branch, record, err)
}
}
}
@@ -95,12 +95,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// A branch's view holding its first half asks again the steps placed in its running pipelines that were never asked
// (§88); the starter is made after the views, so it is reached through a pointer set once it is.
var resumer atomic.Pointer[func(string)]
var advancer atomic.Pointer[func(string)]
var placer atomic.Pointer[func(catalog.CatalogRevisionEvent) error]
var registrar atomic.Pointer[func(catalog.Registration)]
rv, err := startViews(ctx, js, conn, cred, hub, calls.State("findings"), calls.Call, func(branch string) {
if f := resumer.Load(); f != nil {
(*f)(branch)
}
}, func(branch string) {
if f := advancer.Load(); f != nil {
(*f)(branch)
}
}, func(_ context.Context, ev catalog.CatalogRevisionEvent) error {
if f := placer.Load(); f != nil {
return (*f)(ev)
@@ -170,6 +175,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
logger.Printf("the steps placed on %s and not yet asked: %d asked again", branch, n)
}
resumer.Store(&resume)
// The view reaching a newer revision places the steps of the pipelines that waited for it, and asks nothing again (A-448, §200).
advance := func(branch string) {
if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 {
logger.Printf("the pipelines of %s that waited for a newer revision of its half: %d placed (%v)", branch, n, err)
}
}
advancer.Store(&advance)
// A registration-changed places the steps of its module's pipelines started before it, from it (A-327).
register := func(reg catalog.Registration) {
if n, err := starter.Registered(pipelineStore{ctx: ctx, kv: pipelines}, reg, rv.Catalog.View); err != nil || n > 0 {
@@ -31,7 +31,7 @@ type runnerViews struct {
// startViews wires the runner's views on hub and asks each first view; register every other handler on hub before or
// after, then start hub once.
func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cred process.Account, hub *process.EventHub, findingsState process.State,
call meshcatalogclient.Call, onReady func(branch string), candidate func(ctx context.Context, ev catalog.CatalogRevisionEvent) error,
call meshcatalogclient.Call, onReady, onAdvance func(branch string), candidate func(ctx context.Context, ev catalog.CatalogRevisionEvent) error,
registered func(reg catalog.Registration), logf func(string, ...any)) (*runnerViews, error) {
findings, err := findingWriter(ctx, cred, conn, js, findingsState, logf)
if err != nil {
@@ -48,30 +48,21 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cr
// A branch whose first view was none yet is ready with no half; the first half held later, from a catalog-revision,
// is picked up as readiness is: its pipelines started with no steps are placed then (A-327, A-336).
var wired atomic.Pointer[meshcatalogclient.Catalog]
var held atomic.Bool
var heldRevision atomic.Int64 // the newest revision the view was ready with (A-448)
var ready pipeline.Readiness
c, _, err := meshcatalogclient.WireCatalogProcess(reader, call, branch, func(string, json.RawMessage, bool, time.Time) {
c := wired.Load()
if c == nil || onReady == nil {
return
}
// Ready on the first half held and again on each newer revision: a pipeline that waits for the revision its take-in
// made (A-448, §200) is placed when the view reaches it.
// Ready on the first half held (the steps placed are asked again, §88); a newer revision only places the steps of
// the pipelines that waited for it (A-448, §200).
if h, ok := c.View(); ok {
first := !held.Swap(true)
advanced := false
for rev := h.Revision.Revision; ; {
old := heldRevision.Load()
if rev <= old {
break
}
if heldRevision.CompareAndSwap(old, rev) {
advanced = true
break
}
}
if first || advanced {
first, advanced := ready.Observe(h.Revision.Revision)
switch {
case first:
go onReady(branch)
case advanced && onAdvance != nil:
go onAdvance(branch)
}
}
})
@@ -78,6 +78,9 @@ func TestAPipelinePinsItsCatalogRevisionWhileItRuns(t *testing.T) {
t.Fatalf("supersede %d (%v)", n, err)
}
want(0, delivery.SubjectFreed, "pipeline."+pkey, 12)
if len(said) != 1 {
t.Errorf("a supersede said %d pins, want the one freed (through its end, once)", len(said))
}
}
// A-448 (§200, §476): a pipeline of a take-in starts at the revision the take-in made, never at the one the runner's own view
@@ -118,3 +121,61 @@ func TestATakeInStartsItsPipelinesAtItsOwnRevisionAndWaitsForTheView(t *testing.
t.Errorf("a take-in naming no revision started %+v, want the view's revision 357 with its steps", p)
}
}
// A-448: the view is read before the starts write; a view that reaches the revision meanwhile places the held-back pipelines at
// once, not at an advance that already came.
func TestAViewThatReachesTheRevisionDuringTheStartsPlacesTheHeldBackPipelines(t *testing.T) {
reg, entries := placeFixture()
store := memCreated{memStore{}}
asks := &stepAsks{}
s := &Starter{Pipelines: store, Steps: asks, Name: "the-runner", Say: func(string, any) error { return nil },
Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" }}
s.Store = store.memStore
reads := 0
view := func(string) (Half, bool) {
reads++
if reads > 1 { // the first read is before the starts wrote; the view reaches 357 after it
return Half{Revision: 357, Entries: entries}, true
}
return Half{Revision: 356, Entries: entries}, true
}
taken := catalog.TakeInTaken{Key: "m.1", Branch: "main", Source: 947, Revision: 357, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}
if n, err := s.TakenIn(taken, view); n != 1 || err != nil {
t.Fatalf("started %d (%v)", n, err)
}
if p, _, _ := store.Get("main.947.app"); len(p.Steps) == 0 || len(asks.asks) == 0 {
t.Errorf("the pipeline has %d steps and %d asks after the view reached 357 during the start, want them placed", len(p.Steps), len(asks.asks))
}
}
// A-448 (§200): the first half a view holds is Ready, which asks again the steps placed (§88); a newer revision is an advance,
// which only places the steps of pipelines that waited for it and asks nothing again.
func TestReadinessSaysTheFirstHoldAndEachNewerRevisionApart(t *testing.T) {
var r Readiness
for i, c := range []struct {
rev int64
first, advanced bool
}{{356, true, false}, {356, false, false}, {355, false, false}, {357, false, true}, {357, false, false}, {360, false, true}} {
if first, advanced := r.Observe(c.rev); first != c.first || advanced != c.advanced {
t.Errorf("observation %d of revision %d: first %v advanced %v, want %v %v", i, c.rev, first, advanced, c.first, c.advanced)
}
}
}
// A-448: placing the steps of a pipeline that waited asks nothing again of the running steps of the others.
func TestPlacingAWaitingPipelineAsksNoRunningStepAgain(t *testing.T) {
reg, entries := placeFixture()
store := memCreated{memStore{}}
asks := &stepAsks{}
s := &Starter{Pipelines: store, Steps: asks, Name: "the-runner", Say: func(string, any) error { return nil },
Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" }}
view := func(string) (Half, bool) { return Half{Revision: 357, Entries: entries}, true }
if n, err := s.TakenIn(catalog.TakeInTaken{Key: "m.1", Branch: "main", Source: 1, Revision: 357, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}, view); n != 1 || err != nil {
t.Fatalf("started %d (%v)", n, err)
}
before := len(asks.asks)
late := func(string) (Half, bool) { return Half{Revision: 358, Entries: entries}, true }
if n, err := s.Waiting(store.memStore, "main", late); n != 0 || err != nil || len(asks.asks) != before {
t.Errorf("an advance placed %d (%v) and asked %d steps again, want nothing", n, err, len(asks.asks)-before)
}
}
@@ -187,3 +187,28 @@ func (v *Views) HearRevision(ctx context.Context, data []byte) error {
}
return first
}
// Readiness tells, from the revisions a branch's view is seen holding, when the view first holds a half and when it reaches
// a newer revision: the first hold asks again the steps already placed (§88's restart pickup), a newer revision only places
// the steps of pipelines that waited for it (A-448, §200). It is safe for concurrent use.
type Readiness struct {
mu sync.Mutex
held bool
rev int64
}
// Observe notes that the view holds revision rev: first is true once, at the first hold; advanced is true when rev is newer
// than any seen before and it is not the first.
func (r *Readiness) Observe(rev int64) (first, advanced bool) {
r.mu.Lock()
defer r.mu.Unlock()
if !r.held {
r.held, r.rev = true, rev
return true, false
}
if rev > r.rev {
r.rev = rev
return false, true
}
return false, false
}
@@ -459,6 +459,15 @@ func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half
started++
}
}
// The view was read before the starts wrote: when pipelines were held back for it and it has reached the revision since,
// they are placed now, not left for an advance that already came (A-448, §200).
if !held && started > 0 && revision > 0 && s.Store != nil {
if again, ok := view(ev.Branch); ok && again.Revision >= revision {
if _, err := s.Waiting(s.Store, ev.Branch, view); err != nil && first == nil {
first = err
}
}
}
return started, first
}
@@ -127,7 +127,6 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop
} else if err != nil {
return false, fmt.Errorf("pipeline %s cannot be written: %w", key, err)
}
h.free(p) // a superseded pipeline ends too (A-449)
var first error
for _, i := range changed {
if err := h.say(seat.EventStepEnded, p, &p.Steps[i]); err != nil && first == nil {