Catch-up says take-in.taken; the runner asks a behind view again and starts the held modules when it reaches the revision

The indexer's catch-up (source.go takeInHeads, via TakeInBehind and CatchUp) says take-in.taken with branch, source, revision and an entry per module it wrote. The runner's TakenIn, with its view behind the take-in's revision, asks the view again and holds the start of the unstarted modules until the view reaches it (StartHeld, run at the advance).

Builds: design 47 G A-490k, A-463, A-464, §493

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
This commit is contained in:
2026-11-19 23:10:10 +01:00
parent 0ea4b60dbd
commit 8206c74420
8 changed files with 196 additions and 5 deletions
@@ -123,7 +123,7 @@ func (ix *Indexer) TakeInBehind(ctx context.Context) []string {
if len(behind) == 0 {
return nil
}
unread := ix.takeInHeads(ctx, behind)
unread, written := ix.takeInHeadsWritten(ctx, behind)
var done []string
for _, repository := range slices.Sorted(maps.Keys(behind)) {
if slices.Contains(unread, repository) {
@@ -137,6 +137,8 @@ func (ix *Indexer) TakeInBehind(ctx context.Context) []string {
if len(done) > 0 {
if err := ix.KeepView(ctx); err != nil {
ix.logf("merges: the view could not be kept after %v were taken in: %v", done, err)
} else {
ix.sayCaughtUp(ctx, written, behind, unread)
}
}
return done
@@ -39,6 +39,17 @@ func TestAMergeTheIndexerNeverTookInIsTakenInAndSaidWhileItIsBehind(t *testing.T
if v, _, _, _ := b.ix.Taken.Get(ctx, key); string(v) == string(data) {
t.Errorf("taken.%s still names %s", key, old)
}
// A-490k: the catch-up says take-in.taken with the branch, source, revision and the module it wrote, so the runner starts
// its pipeline; a take-in.taken said with no entries starts nothing.
var taken []takein.TakeIn
for _, s := range b.said {
if s.Kind == takein.TakeInTaken && len(s.Entries) > 0 {
taken = append(taken, s)
}
}
if len(taken) != 1 || taken[0].Branch == "" || taken[0].Source <= 0 || len(taken[0].Entries) != 1 || taken[0].Entries[0].Module != "avahi" || taken[0].Entries[0].Commit != head {
t.Errorf("the catch-up said %+v, want one take-in.taken naming the branch, a source revision and the entry avahi at %s", b.said, head)
}
// A head the taken merge already holds (the source not yet moved to a merge just taken in) is left.
b.ix.source = sourceNow{rev: 4, heads: map[string]string{catalogueRepo: old}}
if done := b.ix.TakeInBehind(ctx); len(done) != 0 {
@@ -18,6 +18,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
)
@@ -198,7 +199,12 @@ func (ix *Indexer) kickCatchUp(ctx context.Context, heads map[string]string) err
// takeInHeads applies the forge bridge's merged message of each repository's head, as its merge, and answers the
// repositories whose message was not read or not applied.
func (ix *Indexer) takeInHeads(ctx context.Context, heads map[string]string) []string {
var unread []string
unread, _ := ix.takeInHeadsWritten(ctx, heads)
return unread
}
// takeInHeadsWritten is takeInHeads, and answers also the module keys the heads' messages wrote.
func (ix *Indexer) takeInHeadsWritten(ctx context.Context, heads map[string]string) (unread, written []string) {
for _, repository := range slices.Sorted(maps.Keys(heads)) {
head := heads[repository]
if head == "" {
@@ -212,14 +218,47 @@ func (ix *Indexer) takeInHeads(ctx context.Context, heads map[string]string) []s
f, err = ix.wholeAt(ctx, repository, head)
}
if err == nil {
_, err = ix.ApplyMessage(ctx, f)
var a Applied
a, err = ix.ApplyMessage(ctx, f)
written = append(written, a.Written...)
}
if err != nil {
unread = append(unread, repository)
ix.logf("catalog graph: the trunk of %s at %.12s is not taken in: %v", repository, head, err)
}
}
return unread
return unread, written
}
// sayCaughtUp says take-in.taken for the modules a catch-up wrote, as every other source of a take-in does (design 47 G
// A-490k, §493): the trunk's branch, the source revision it makes, the catalog graph revision kept for it (pinned for its
// pipelines) and one entry per module written, so the runner starts their pipelines. The merges it names are the heads taken
// in. A catch-up that wrote no module says nothing. Called once the revision is kept.
func (ix *Indexer) sayCaughtUp(ctx context.Context, written []string, heads map[string]string, unread []string) {
if len(written) == 0 {
return
}
var merges []takein.Merge
var claimed []PendingEntry
for _, repository := range slices.Sorted(maps.Keys(heads)) {
if heads[repository] == "" || slices.Contains(unread, repository) {
continue
}
merges = append(merges, takein.Merge{Repository: repository, Merge: heads[repository]})
claimed = append(claimed, PendingEntry{Forge: forgemsg.Forge{Repository: repository, Merge: heads[repository]}})
}
if len(merges) == 0 {
return
}
branch, src, took := ix.takenWhat(ctx, claimed, Applied{Written: written})
if len(took) == 0 {
return
}
revision := ix.PinPipelines(ctx, branch, src, took)
t := takein.TakeIn{Kind: takein.TakeInTaken, Key: "m.catchup." + fmt.Sprintf("%.12s", merges[0].Merge), What: takein.TakeInRevision,
Merges: merges, Since: ix.now(), Branch: branch, Source: src, Entries: took, Revision: revision}
ix.logf("take-in: the catch-up took in %d merges as source revision %d of %s, %d entries", len(merges), src, branch, len(took))
ix.sayTakeIn(ctx, t)
}
// CatchUp is the indexer's catch-up tool's work (indexer_take_in with no module and no repository, the catch-up its
@@ -243,7 +282,7 @@ func (ix *Indexer) CatchUp(ctx context.Context) TakenIn {
out.Answer = "a catch-up runs already; its end keeps the revision"
return out
}
unread := ix.takeInHeads(ctx, heads)
unread, written := ix.takeInHeadsWritten(ctx, heads)
ix.catchingUp.Store(false)
if ix.behindWaiting.Swap(false) {
defer ix.kickBehind(ctx)
@@ -252,6 +291,7 @@ func (ix *Indexer) CatchUp(ctx context.Context) TakenIn {
out.Answer = "the repositories were taken in but the revision could not be kept: " + err.Error()
return out
}
ix.sayCaughtUp(ctx, written, heads, unread)
out.Stored = len(unread) == 0
out.Unread = unread
out.Answer = fmt.Sprintf("the heads the gate's source names for %d repositories taken in", len(heads)-len(unread))
@@ -158,6 +158,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, js: js, cred: cred}, Name: cred.Module,
Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains),
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing}
starter.Again = func(branch string, revision int64) {
if err := rv.Catalog.Again(ctx, branch, revision); err != nil {
logger.Printf("the view of %s is not asked again for revision %d: %v", branch, revision, err)
}
}
placeHalf := func(ev catalog.CatalogRevisionEvent, half pipeline.Half) error {
n, err := starter.PlaceRun(ev, half, rv.Catalog.View)
logger.Printf("the candidate %s's revision %d placed %d pipelines of its run", ev.Group, ev.Revision, n)
@@ -192,6 +197,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 {
logger.Printf("the pipelines of %s that waited for a newer revision of its half: %d placed (%v)", branch, n, err)
}
// The start a take-in held while the view was behind its revision runs now it has reached it (A-490k).
if half, ok := rv.Catalog.View(branch); ok {
if n, err := starter.StartHeld(branch, half); err != nil || n > 0 {
logger.Printf("the modules of %s with no pipeline that a take-in held for a newer revision of its half: %d started (%v)", branch, n, err)
}
}
}
advancer.Store(&advance)
// A registration-changed places the steps of its module's pipelines started before it, from it (A-327).
@@ -10,6 +10,7 @@ import (
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// Status is a step's status for one artifact of a module, read from the module's registration and never from a
@@ -173,6 +174,26 @@ func (v *Views) View(branch string) (Half, bool) {
return bv.View()
}
// Again asks the view of branch again for revision, which a take-in named and the view does not hold (design 47 G A-490k): it
// hears a trunk catalog-revision event of that revision, as the seat would have said it, and the view asks its holder once
// more on its own goroutine. A branch with no view, or a view that holds the revision, asks nothing.
func (v *Views) Again(ctx context.Context, branch string, revision int64) error {
v.mu.Lock()
bv := v.branches[branch]
v.mu.Unlock()
if bv == nil || revision <= 0 {
return nil
}
if h, ok := bv.View(); ok && h.Revision >= revision {
return nil
}
data, err := json.Marshal(catalog.CatalogRevisionEvent{V: 1, Kind: "trunk", Branch: keys.EscapeName(branch), Revision: revision})
if err != nil {
return err
}
return bv.Hear(ctx, data)
}
// HearRevision hands one catalog-revision event to every branch's view.
func (v *Views) HearRevision(ctx context.Context, data []byte) error {
v.mu.Lock()
@@ -72,6 +72,14 @@ func TestPolicyAndBuildSeat(t *testing.T) {
if opened != 4 {
t.Errorf("a branch whose view was refused is opened again at the next registration: %d opens", opened)
}
// A-490k: a view asked again for a revision it does not hold hears it as the seat's event; one that holds it asks nothing.
if err := v.Again(context.Background(), "main", 9); err != nil || heard["main"] != 1 {
t.Errorf("a view at 7 asked again for 9 heard %d events (%v), want 1", heard["main"], err)
}
if err := v.Again(context.Background(), "main", 7); err != nil || heard["main"] != 1 {
t.Errorf("a view at 7 asked again for 7 heard %d events (%v), want none more", heard["main"], err)
}
heard["main"] = 0
if err := v.HearRevision(context.Background(), []byte(`{}`)); err != nil || heard["main"] != 1 || heard["feature/x"] != 1 {
t.Errorf("a revision heard reaches every branch's view: %v %v", err, heard)
}
@@ -8,6 +8,7 @@ import (
"sort"
"strconv"
"strings"
"sync"
"time"
runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
@@ -98,8 +99,17 @@ type Starter struct {
Runs RunSettings
RunsFill func() (seat.Fill, error)
EndReaders func(run string, readers []seat.TestRunReader, r seat.TestRun) error
// Again asks the view of a branch's catalog half again, for a revision a take-in named that the view does not hold yet
// (design 47 G A-490k): the answer comes as the view's own events, with no timer here; nil asks nothing.
Again func(branch string, revision int64)
heldMu sync.Mutex
held map[string]heldStart
}
// heldStart is the start of a branch's unstarted modules that a take-in made while the view was behind its revision.
type heldStart struct{ source, revision int64 }
// holderless ends at once, as it is placed, a pipeline with a build step whose seat the seat mesh-module-assigner's view
// seat-holders names no holder for (design 47 G §32, §535, §647 as A-466y and A-466z amend them; A-470g, A-470h): judged only
// once that view's first view is read, never from the picture before it. Each such step ends `could not run` naming the seat,
@@ -581,6 +591,12 @@ func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half
} else if held && half.Revision < revision {
s.logf("the take-in %s of %s made revision %d and the runner's view holds %d: its pipelines start with no step, placed once the view reaches it (§200)", ev.Key, ev.Branch, revision, half.Revision)
held = false
// The modules the take-in's batch moved and no pipeline carries start once the view reaches the revision (A-490k), and
// the view is asked again now: it may have been answered by a holder that was behind.
s.holdStart(ev.Branch, ev.Source, revision)
if s.Again != nil {
s.Again(ev.Branch, revision)
}
}
started := 0
var first error
@@ -612,6 +628,16 @@ func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half
}
}
}
// The held start runs here when the view reached the revision while the take-in was being read (A-490k).
if !held && revision > 0 {
if again, ok := view(ev.Branch); ok && again.Revision >= revision {
n, err := s.StartHeld(ev.Branch, again)
started += n
if err != nil && first == nil {
first = err
}
}
}
return started, first
}
@@ -622,6 +648,37 @@ func commitOrGroup(p seat.Pipeline) string {
return p.Commit
}
func (s *Starter) holdStart(branch string, source, revision int64) {
s.heldMu.Lock()
defer s.heldMu.Unlock()
if s.held == nil {
s.held = map[string]heldStart{}
}
if h, ok := s.held[branch]; ok && h.revision >= revision {
return
}
s.held[branch] = heldStart{source: source, revision: revision}
}
// StartHeld runs the start a take-in held while the view of branch was behind its revision, once half holds that revision
// (A-490k): the modules of half with no pipeline at their fingerprint start at the take-in's source revision. It says how
// many it started; with nothing held, or the view still behind, none.
func (s *Starter) StartHeld(branch string, half Half) (int, error) {
s.heldMu.Lock()
h, ok := s.held[branch]
if !ok || half.Revision < h.revision {
s.heldMu.Unlock()
return 0, nil
}
delete(s.held, branch)
s.heldMu.Unlock()
n, err := s.StartUnstarted(branch, h.source, half)
if err != nil {
s.holdStart(branch, h.source, h.revision) // held again: the next advance runs it
}
return n, err
}
// Unstarted names the modules of half whose fingerprint has no pipeline on branch: no pipeline of any outcome, a published
// one included, carries that module at that fingerprint. A run on request is no pipeline of the branch's head. Sorted.
func (s *Starter) Unstarted(branch string, half Half) ([]string, error) {
@@ -314,3 +314,44 @@ func TestTakenInStartsEveryModuleWithNoPipelineAtItsFingerprint(t *testing.T) {
t.Errorf("a half with no source revision started %d (%v)", n, err)
}
}
// A-490k: a take-in whose revision the runner's view has not reached asks the view again, holds the start of the modules its batch
// moved, and runs it once the view reaches the revision, with no timer; a take-in heard with the view behind starts none of them first.
func TestATakeInBehindTheViewAsksTheViewAgainAndStartsTheHeldModulesWhenItReaches(t *testing.T) {
mem := memStore{}
s := &Starter{Pipelines: memCreated{mem}, Store: mem, Say: func(string, any) error { return nil }}
var asked []int64
s.Again = func(branch string, revision int64) {
if branch != "main" {
t.Errorf("the view of %s is asked again, want main", branch)
}
asked = append(asked, revision)
}
entries := map[string]catalog.CatalogEntry{"moved": {Commit: "c1", Fp: "src2:aa"}, "batched": {Commit: "c2", Fp: "src2:bb"}}
half := Half{Revision: 11, Source: 8, Entries: entries}
view := func(string) (Half, bool) { return half, true }
ev := catalog.TakeInTaken{Kind: "taken", Branch: "main", Source: 9, Revision: 12,
Entries: []catalog.TakeInEntry{{Module: "moved", Commit: "c1", Fingerprint: "src2:aa"}}}
if n, err := s.TakenIn(ev, view); err != nil || n != 1 {
t.Fatalf("the take-in started %d (%v), want its one entry", n, err)
}
if len(asked) != 1 || asked[0] != 12 {
t.Errorf("the view was asked again for revisions %v, want [12]", asked)
}
if _, _, err := mem.Get("main.9.batched"); err == nil {
t.Fatal("main.9.batched started while the view is behind the take-in's revision")
}
if n, err := s.StartHeld("main", half); n != 0 || err != nil {
t.Errorf("the view still at 11 started %d held (%v), want none", n, err)
}
half = Half{Revision: 12, Source: 9, Entries: entries}
if n, err := s.StartHeld("main", half); err != nil || n != 1 {
t.Fatalf("the view at 12 started %d held (%v), want batched", n, err)
}
if _, _, err := mem.Get("main.9.batched"); err != nil {
t.Errorf("main.9.batched is not started once the view reached 12: %v", err)
}
if n, err := s.StartHeld("main", half); n != 0 || err != nil {
t.Errorf("the held start ran twice: %d (%v)", n, err)
}
}