pipeline runner and indexer: review fixes (advance only places waiting steps, view re-read after the starts, one freed) and the indexer pins a take-in's revision for its pipelines (A-448, A-449a)
Builds: design 47 G (G-22 lead answers 2) A-448, A-449, A-449a; §153, §200, §579
This commit is contained in:
@@ -1170,7 +1170,8 @@ func (ix *Indexer) takeInWholeBy(ctx context.Context, entries []PendingEntry, ow
|
||||
ix.forget(claimed...)
|
||||
branch, src, took := ix.takenWhat(ctx, claimed, applied)
|
||||
ix.logf("take-in: %d merges taken in as one revision by %s%s, oldest %s: source revision %d of %s, %d entries", len(claimed), own, byWords(by), claimed[0].Key, src, branch, len(took))
|
||||
revision := ix.NewestRevision(ctx, branch) // the revision this take-in just kept (A-448)
|
||||
revision := ix.NewestRevision(ctx, branch) // the revision this take-in just kept (A-448)
|
||||
ix.PinPipelines(ctx, branch, revision, src, took) // kept for its pipelines before any later take-in can remove it (A-449a)
|
||||
for _, t := range takenWaits(claimed) {
|
||||
t.By, t.Why = who, why
|
||||
t.Branch, t.Source, t.Entries, t.Revision = branch, src, took, revision
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -1000,6 +1001,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
!strings.Contains(r.Answer, TakeInPendingTool) || len(b.keys()) != 1 {
|
||||
t.Fatalf("a drop nothing covers: %+v, pending %v", r, b.keys())
|
||||
}
|
||||
b.ix.Pins = newMem()
|
||||
b.ix.Revisions = newMemObjects(0) // A-448: the taken event names the catalog graph revision the take-in made
|
||||
b.said = nil
|
||||
if r := b.ix.TakeInPendingByHand(ctx, takein.PendingKey(3), "lead@shanks", "the named indexer can never run"); !r.Stored {
|
||||
@@ -1016,6 +1018,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
t.Errorf("hand take-in said branch %q source %d entries %+v: want main, a source revision and avahi at its commit", got.Branch, got.Source, got.Entries)
|
||||
} else if newest, any, _ := newestTrunk(ctx, b.ix.Revisions); !any || got.Revision != newest.Revision {
|
||||
t.Errorf("hand take-in said revision %d, want the revision it made, %d (A-448)", got.Revision, newest.Revision)
|
||||
} else if v, _, found, _ := b.ix.Pins.Get(ctx, PinKey(got.Branch, got.Revision)); !found || !strings.Contains(string(v), "pipeline."+got.Branch+"."+strconv.FormatInt(got.Source, 10)+".avahi") {
|
||||
t.Errorf("the revision the take-in made is pinned %q (found %v), want the record pipeline.%s.%d.avahi (A-449a)", v, found, got.Branch, got.Source)
|
||||
} else if _, err := takein.EncodeTakeIn(got); err != nil {
|
||||
t.Errorf("the taken event is refused by its own contract: %v", err)
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
@@ -573,3 +574,24 @@ func ConsumerTools(st Store) []stdio.Tool {
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) { return ReadConsumers(context.Background(), st) }}}
|
||||
}
|
||||
|
||||
// PinPipelines pins the catalog graph revision a take-in made for each pipeline the take-in starts, under the record
|
||||
// `pipeline.<key>` (`<branch>.<source>.<module>` of each entry, the runner's key), in the indexer's own pins state as a
|
||||
// pinned of the seat mesh-delivery is applied (design 47 G §579, A-449a): so no later take-in removes the revision before
|
||||
// the runner hears of it. The runner's pinned at the pipeline's start says the same record (a set), and its freed at the
|
||||
// pipeline's end removes it.
|
||||
func (ix *Indexer) PinPipelines(ctx context.Context, branch string, revision, source int64, entries []meshcatalog.TakeInEntry) {
|
||||
if ix.Pins == nil || revision <= 0 || len(entries) == 0 {
|
||||
return
|
||||
}
|
||||
if err := ix.PinsMade(ctx); err != nil {
|
||||
ix.logf("pins: the revision %d of %s is not pinned for its pipelines: %v", revision, branch, err)
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
record := "pipeline." + runner.PipelinesByBranchRevisionModuleKey(keys.EscapeName(branch), strconv.FormatInt(source, 10), e.Module)
|
||||
if err := ix.pin(ctx, PinKey(branch, revision), record); err != nil {
|
||||
ix.logf("pins: the revision %d of %s is not pinned for %s: %v", revision, branch, record, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -95,12 +95,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// A branch's view holding its first half asks again the steps placed in its running pipelines that were never asked
|
||||
// (§88); the starter is made after the views, so it is reached through a pointer set once it is.
|
||||
var resumer atomic.Pointer[func(string)]
|
||||
var advancer atomic.Pointer[func(string)]
|
||||
var placer atomic.Pointer[func(catalog.CatalogRevisionEvent) error]
|
||||
var registrar atomic.Pointer[func(catalog.Registration)]
|
||||
rv, err := startViews(ctx, js, conn, cred, hub, calls.State("findings"), calls.Call, func(branch string) {
|
||||
if f := resumer.Load(); f != nil {
|
||||
(*f)(branch)
|
||||
}
|
||||
}, func(branch string) {
|
||||
if f := advancer.Load(); f != nil {
|
||||
(*f)(branch)
|
||||
}
|
||||
}, func(_ context.Context, ev catalog.CatalogRevisionEvent) error {
|
||||
if f := placer.Load(); f != nil {
|
||||
return (*f)(ev)
|
||||
@@ -170,6 +175,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
logger.Printf("the steps placed on %s and not yet asked: %d asked again", branch, n)
|
||||
}
|
||||
resumer.Store(&resume)
|
||||
// The view reaching a newer revision places the steps of the pipelines that waited for it, and asks nothing again (A-448, §200).
|
||||
advance := func(branch string) {
|
||||
if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 {
|
||||
logger.Printf("the pipelines of %s that waited for a newer revision of its half: %d placed (%v)", branch, n, err)
|
||||
}
|
||||
}
|
||||
advancer.Store(&advance)
|
||||
// A registration-changed places the steps of its module's pipelines started before it, from it (A-327).
|
||||
register := func(reg catalog.Registration) {
|
||||
if n, err := starter.Registered(pipelineStore{ctx: ctx, kv: pipelines}, reg, rv.Catalog.View); err != nil || n > 0 {
|
||||
|
||||
@@ -31,7 +31,7 @@ type runnerViews struct {
|
||||
// startViews wires the runner's views on hub and asks each first view; register every other handler on hub before or
|
||||
// after, then start hub once.
|
||||
func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cred process.Account, hub *process.EventHub, findingsState process.State,
|
||||
call meshcatalogclient.Call, onReady func(branch string), candidate func(ctx context.Context, ev catalog.CatalogRevisionEvent) error,
|
||||
call meshcatalogclient.Call, onReady, onAdvance func(branch string), candidate func(ctx context.Context, ev catalog.CatalogRevisionEvent) error,
|
||||
registered func(reg catalog.Registration), logf func(string, ...any)) (*runnerViews, error) {
|
||||
findings, err := findingWriter(ctx, cred, conn, js, findingsState, logf)
|
||||
if err != nil {
|
||||
@@ -48,30 +48,21 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cr
|
||||
// A branch whose first view was none yet is ready with no half; the first half held later, from a catalog-revision,
|
||||
// is picked up as readiness is: its pipelines started with no steps are placed then (A-327, A-336).
|
||||
var wired atomic.Pointer[meshcatalogclient.Catalog]
|
||||
var held atomic.Bool
|
||||
var heldRevision atomic.Int64 // the newest revision the view was ready with (A-448)
|
||||
var ready pipeline.Readiness
|
||||
c, _, err := meshcatalogclient.WireCatalogProcess(reader, call, branch, func(string, json.RawMessage, bool, time.Time) {
|
||||
c := wired.Load()
|
||||
if c == nil || onReady == nil {
|
||||
return
|
||||
}
|
||||
// Ready on the first half held and again on each newer revision: a pipeline that waits for the revision its take-in
|
||||
// made (A-448, §200) is placed when the view reaches it.
|
||||
// Ready on the first half held (the steps placed are asked again, §88); a newer revision only places the steps of
|
||||
// the pipelines that waited for it (A-448, §200).
|
||||
if h, ok := c.View(); ok {
|
||||
first := !held.Swap(true)
|
||||
advanced := false
|
||||
for rev := h.Revision.Revision; ; {
|
||||
old := heldRevision.Load()
|
||||
if rev <= old {
|
||||
break
|
||||
}
|
||||
if heldRevision.CompareAndSwap(old, rev) {
|
||||
advanced = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if first || advanced {
|
||||
first, advanced := ready.Observe(h.Revision.Revision)
|
||||
switch {
|
||||
case first:
|
||||
go onReady(branch)
|
||||
case advanced && onAdvance != nil:
|
||||
go onAdvance(branch)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
@@ -78,6 +78,9 @@ func TestAPipelinePinsItsCatalogRevisionWhileItRuns(t *testing.T) {
|
||||
t.Fatalf("supersede %d (%v)", n, err)
|
||||
}
|
||||
want(0, delivery.SubjectFreed, "pipeline."+pkey, 12)
|
||||
if len(said) != 1 {
|
||||
t.Errorf("a supersede said %d pins, want the one freed (through its end, once)", len(said))
|
||||
}
|
||||
}
|
||||
|
||||
// A-448 (§200, §476): a pipeline of a take-in starts at the revision the take-in made, never at the one the runner's own view
|
||||
@@ -118,3 +121,61 @@ func TestATakeInStartsItsPipelinesAtItsOwnRevisionAndWaitsForTheView(t *testing.
|
||||
t.Errorf("a take-in naming no revision started %+v, want the view's revision 357 with its steps", p)
|
||||
}
|
||||
}
|
||||
|
||||
// A-448: the view is read before the starts write; a view that reaches the revision meanwhile places the held-back pipelines at
|
||||
// once, not at an advance that already came.
|
||||
func TestAViewThatReachesTheRevisionDuringTheStartsPlacesTheHeldBackPipelines(t *testing.T) {
|
||||
reg, entries := placeFixture()
|
||||
store := memCreated{memStore{}}
|
||||
asks := &stepAsks{}
|
||||
s := &Starter{Pipelines: store, Steps: asks, Name: "the-runner", Say: func(string, any) error { return nil },
|
||||
Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" }}
|
||||
s.Store = store.memStore
|
||||
reads := 0
|
||||
view := func(string) (Half, bool) {
|
||||
reads++
|
||||
if reads > 1 { // the first read is before the starts wrote; the view reaches 357 after it
|
||||
return Half{Revision: 357, Entries: entries}, true
|
||||
}
|
||||
return Half{Revision: 356, Entries: entries}, true
|
||||
}
|
||||
taken := catalog.TakeInTaken{Key: "m.1", Branch: "main", Source: 947, Revision: 357, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}
|
||||
if n, err := s.TakenIn(taken, view); n != 1 || err != nil {
|
||||
t.Fatalf("started %d (%v)", n, err)
|
||||
}
|
||||
if p, _, _ := store.Get("main.947.app"); len(p.Steps) == 0 || len(asks.asks) == 0 {
|
||||
t.Errorf("the pipeline has %d steps and %d asks after the view reached 357 during the start, want them placed", len(p.Steps), len(asks.asks))
|
||||
}
|
||||
}
|
||||
|
||||
// A-448 (§200): the first half a view holds is Ready, which asks again the steps placed (§88); a newer revision is an advance,
|
||||
// which only places the steps of pipelines that waited for it and asks nothing again.
|
||||
func TestReadinessSaysTheFirstHoldAndEachNewerRevisionApart(t *testing.T) {
|
||||
var r Readiness
|
||||
for i, c := range []struct {
|
||||
rev int64
|
||||
first, advanced bool
|
||||
}{{356, true, false}, {356, false, false}, {355, false, false}, {357, false, true}, {357, false, false}, {360, false, true}} {
|
||||
if first, advanced := r.Observe(c.rev); first != c.first || advanced != c.advanced {
|
||||
t.Errorf("observation %d of revision %d: first %v advanced %v, want %v %v", i, c.rev, first, advanced, c.first, c.advanced)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A-448: placing the steps of a pipeline that waited asks nothing again of the running steps of the others.
|
||||
func TestPlacingAWaitingPipelineAsksNoRunningStepAgain(t *testing.T) {
|
||||
reg, entries := placeFixture()
|
||||
store := memCreated{memStore{}}
|
||||
asks := &stepAsks{}
|
||||
s := &Starter{Pipelines: store, Steps: asks, Name: "the-runner", Say: func(string, any) error { return nil },
|
||||
Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" }}
|
||||
view := func(string) (Half, bool) { return Half{Revision: 357, Entries: entries}, true }
|
||||
if n, err := s.TakenIn(catalog.TakeInTaken{Key: "m.1", Branch: "main", Source: 1, Revision: 357, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}, view); n != 1 || err != nil {
|
||||
t.Fatalf("started %d (%v)", n, err)
|
||||
}
|
||||
before := len(asks.asks)
|
||||
late := func(string) (Half, bool) { return Half{Revision: 358, Entries: entries}, true }
|
||||
if n, err := s.Waiting(store.memStore, "main", late); n != 0 || err != nil || len(asks.asks) != before {
|
||||
t.Errorf("an advance placed %d (%v) and asked %d steps again, want nothing", n, err, len(asks.asks)-before)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -187,3 +187,28 @@ func (v *Views) HearRevision(ctx context.Context, data []byte) error {
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
// Readiness tells, from the revisions a branch's view is seen holding, when the view first holds a half and when it reaches
|
||||
// a newer revision: the first hold asks again the steps already placed (§88's restart pickup), a newer revision only places
|
||||
// the steps of pipelines that waited for it (A-448, §200). It is safe for concurrent use.
|
||||
type Readiness struct {
|
||||
mu sync.Mutex
|
||||
held bool
|
||||
rev int64
|
||||
}
|
||||
|
||||
// Observe notes that the view holds revision rev: first is true once, at the first hold; advanced is true when rev is newer
|
||||
// than any seen before and it is not the first.
|
||||
func (r *Readiness) Observe(rev int64) (first, advanced bool) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if !r.held {
|
||||
r.held, r.rev = true, rev
|
||||
return true, false
|
||||
}
|
||||
if rev > r.rev {
|
||||
r.rev = rev
|
||||
return false, true
|
||||
}
|
||||
return false, false
|
||||
}
|
||||
|
||||
@@ -459,6 +459,15 @@ func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half
|
||||
started++
|
||||
}
|
||||
}
|
||||
// The view was read before the starts wrote: when pipelines were held back for it and it has reached the revision since,
|
||||
// they are placed now, not left for an advance that already came (A-448, §200).
|
||||
if !held && started > 0 && revision > 0 && s.Store != nil {
|
||||
if again, ok := view(ev.Branch); ok && again.Revision >= revision {
|
||||
if _, err := s.Waiting(s.Store, ev.Branch, view); err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
}
|
||||
}
|
||||
return started, first
|
||||
}
|
||||
|
||||
|
||||
@@ -127,7 +127,6 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop
|
||||
} else if err != nil {
|
||||
return false, fmt.Errorf("pipeline %s cannot be written: %w", key, err)
|
||||
}
|
||||
h.free(p) // a superseded pipeline ends too (A-449)
|
||||
var first error
|
||||
for _, i := range changed {
|
||||
if err := h.say(seat.EventStepEnded, p, &p.Steps[i]); err != nil && first == nil {
|
||||
|
||||
Reference in New Issue
Block a user