Catch-up says take-in.taken; the runner asks a behind view again and starts the held modules when it reaches the revision
The indexer's catch-up (source.go takeInHeads, via TakeInBehind and CatchUp) says take-in.taken with branch, source, revision and an entry per module it wrote. The runner's TakenIn, with its view behind the take-in's revision, asks the view again and holds the start of the unstarted modules until the view reaches it (StartHeld, run at the advance). Builds: design 47 G A-490k, A-463, A-464, §493 Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
This commit is contained in:
@@ -123,7 +123,7 @@ func (ix *Indexer) TakeInBehind(ctx context.Context) []string {
|
||||
if len(behind) == 0 {
|
||||
return nil
|
||||
}
|
||||
unread := ix.takeInHeads(ctx, behind)
|
||||
unread, written := ix.takeInHeadsWritten(ctx, behind)
|
||||
var done []string
|
||||
for _, repository := range slices.Sorted(maps.Keys(behind)) {
|
||||
if slices.Contains(unread, repository) {
|
||||
@@ -137,6 +137,8 @@ func (ix *Indexer) TakeInBehind(ctx context.Context) []string {
|
||||
if len(done) > 0 {
|
||||
if err := ix.KeepView(ctx); err != nil {
|
||||
ix.logf("merges: the view could not be kept after %v were taken in: %v", done, err)
|
||||
} else {
|
||||
ix.sayCaughtUp(ctx, written, behind, unread)
|
||||
}
|
||||
}
|
||||
return done
|
||||
|
||||
@@ -39,6 +39,17 @@ func TestAMergeTheIndexerNeverTookInIsTakenInAndSaidWhileItIsBehind(t *testing.T
|
||||
if v, _, _, _ := b.ix.Taken.Get(ctx, key); string(v) == string(data) {
|
||||
t.Errorf("taken.%s still names %s", key, old)
|
||||
}
|
||||
// A-490k: the catch-up says take-in.taken with the branch, source, revision and the module it wrote, so the runner starts
|
||||
// its pipeline; a take-in.taken said with no entries starts nothing.
|
||||
var taken []takein.TakeIn
|
||||
for _, s := range b.said {
|
||||
if s.Kind == takein.TakeInTaken && len(s.Entries) > 0 {
|
||||
taken = append(taken, s)
|
||||
}
|
||||
}
|
||||
if len(taken) != 1 || taken[0].Branch == "" || taken[0].Source <= 0 || len(taken[0].Entries) != 1 || taken[0].Entries[0].Module != "avahi" || taken[0].Entries[0].Commit != head {
|
||||
t.Errorf("the catch-up said %+v, want one take-in.taken naming the branch, a source revision and the entry avahi at %s", b.said, head)
|
||||
}
|
||||
// A head the taken merge already holds (the source not yet moved to a merge just taken in) is left.
|
||||
b.ix.source = sourceNow{rev: 4, heads: map[string]string{catalogueRepo: old}}
|
||||
if done := b.ix.TakeInBehind(ctx); len(done) != 0 {
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
)
|
||||
|
||||
@@ -198,7 +199,12 @@ func (ix *Indexer) kickCatchUp(ctx context.Context, heads map[string]string) err
|
||||
// takeInHeads applies the forge bridge's merged message of each repository's head, as its merge, and answers the
|
||||
// repositories whose message was not read or not applied.
|
||||
func (ix *Indexer) takeInHeads(ctx context.Context, heads map[string]string) []string {
|
||||
var unread []string
|
||||
unread, _ := ix.takeInHeadsWritten(ctx, heads)
|
||||
return unread
|
||||
}
|
||||
|
||||
// takeInHeadsWritten is takeInHeads, and answers also the module keys the heads' messages wrote.
|
||||
func (ix *Indexer) takeInHeadsWritten(ctx context.Context, heads map[string]string) (unread, written []string) {
|
||||
for _, repository := range slices.Sorted(maps.Keys(heads)) {
|
||||
head := heads[repository]
|
||||
if head == "" {
|
||||
@@ -212,14 +218,47 @@ func (ix *Indexer) takeInHeads(ctx context.Context, heads map[string]string) []s
|
||||
f, err = ix.wholeAt(ctx, repository, head)
|
||||
}
|
||||
if err == nil {
|
||||
_, err = ix.ApplyMessage(ctx, f)
|
||||
var a Applied
|
||||
a, err = ix.ApplyMessage(ctx, f)
|
||||
written = append(written, a.Written...)
|
||||
}
|
||||
if err != nil {
|
||||
unread = append(unread, repository)
|
||||
ix.logf("catalog graph: the trunk of %s at %.12s is not taken in: %v", repository, head, err)
|
||||
}
|
||||
}
|
||||
return unread
|
||||
return unread, written
|
||||
}
|
||||
|
||||
// sayCaughtUp says take-in.taken for the modules a catch-up wrote, as every other source of a take-in does (design 47 G
|
||||
// A-490k, §493): the trunk's branch, the source revision it makes, the catalog graph revision kept for it (pinned for its
|
||||
// pipelines) and one entry per module written, so the runner starts their pipelines. The merges it names are the heads taken
|
||||
// in. A catch-up that wrote no module says nothing. Called once the revision is kept.
|
||||
func (ix *Indexer) sayCaughtUp(ctx context.Context, written []string, heads map[string]string, unread []string) {
|
||||
if len(written) == 0 {
|
||||
return
|
||||
}
|
||||
var merges []takein.Merge
|
||||
var claimed []PendingEntry
|
||||
for _, repository := range slices.Sorted(maps.Keys(heads)) {
|
||||
if heads[repository] == "" || slices.Contains(unread, repository) {
|
||||
continue
|
||||
}
|
||||
merges = append(merges, takein.Merge{Repository: repository, Merge: heads[repository]})
|
||||
claimed = append(claimed, PendingEntry{Forge: forgemsg.Forge{Repository: repository, Merge: heads[repository]}})
|
||||
}
|
||||
if len(merges) == 0 {
|
||||
return
|
||||
}
|
||||
branch, src, took := ix.takenWhat(ctx, claimed, Applied{Written: written})
|
||||
if len(took) == 0 {
|
||||
return
|
||||
}
|
||||
revision := ix.PinPipelines(ctx, branch, src, took)
|
||||
t := takein.TakeIn{Kind: takein.TakeInTaken, Key: "m.catchup." + fmt.Sprintf("%.12s", merges[0].Merge), What: takein.TakeInRevision,
|
||||
Merges: merges, Since: ix.now(), Branch: branch, Source: src, Entries: took, Revision: revision}
|
||||
ix.logf("take-in: the catch-up took in %d merges as source revision %d of %s, %d entries", len(merges), src, branch, len(took))
|
||||
ix.sayTakeIn(ctx, t)
|
||||
}
|
||||
|
||||
// CatchUp is the indexer's catch-up tool's work (indexer_take_in with no module and no repository, the catch-up its
|
||||
@@ -243,7 +282,7 @@ func (ix *Indexer) CatchUp(ctx context.Context) TakenIn {
|
||||
out.Answer = "a catch-up runs already; its end keeps the revision"
|
||||
return out
|
||||
}
|
||||
unread := ix.takeInHeads(ctx, heads)
|
||||
unread, written := ix.takeInHeadsWritten(ctx, heads)
|
||||
ix.catchingUp.Store(false)
|
||||
if ix.behindWaiting.Swap(false) {
|
||||
defer ix.kickBehind(ctx)
|
||||
@@ -252,6 +291,7 @@ func (ix *Indexer) CatchUp(ctx context.Context) TakenIn {
|
||||
out.Answer = "the repositories were taken in but the revision could not be kept: " + err.Error()
|
||||
return out
|
||||
}
|
||||
ix.sayCaughtUp(ctx, written, heads, unread)
|
||||
out.Stored = len(unread) == 0
|
||||
out.Unread = unread
|
||||
out.Answer = fmt.Sprintf("the heads the gate's source names for %d repositories taken in", len(heads)-len(unread))
|
||||
|
||||
@@ -158,6 +158,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, js: js, cred: cred}, Name: cred.Module,
|
||||
Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains),
|
||||
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing}
|
||||
starter.Again = func(branch string, revision int64) {
|
||||
if err := rv.Catalog.Again(ctx, branch, revision); err != nil {
|
||||
logger.Printf("the view of %s is not asked again for revision %d: %v", branch, revision, err)
|
||||
}
|
||||
}
|
||||
placeHalf := func(ev catalog.CatalogRevisionEvent, half pipeline.Half) error {
|
||||
n, err := starter.PlaceRun(ev, half, rv.Catalog.View)
|
||||
logger.Printf("the candidate %s's revision %d placed %d pipelines of its run", ev.Group, ev.Revision, n)
|
||||
@@ -192,6 +197,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 {
|
||||
logger.Printf("the pipelines of %s that waited for a newer revision of its half: %d placed (%v)", branch, n, err)
|
||||
}
|
||||
// The start a take-in held while the view was behind its revision runs now it has reached it (A-490k).
|
||||
if half, ok := rv.Catalog.View(branch); ok {
|
||||
if n, err := starter.StartHeld(branch, half); err != nil || n > 0 {
|
||||
logger.Printf("the modules of %s with no pipeline that a take-in held for a newer revision of its half: %d started (%v)", branch, n, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
advancer.Store(&advance)
|
||||
// A registration-changed places the steps of its module's pipelines started before it, from it (A-327).
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// Status is a step's status for one artifact of a module, read from the module's registration and never from a
|
||||
@@ -173,6 +174,26 @@ func (v *Views) View(branch string) (Half, bool) {
|
||||
return bv.View()
|
||||
}
|
||||
|
||||
// Again asks the view of branch again for revision, which a take-in named and the view does not hold (design 47 G A-490k): it
|
||||
// hears a trunk catalog-revision event of that revision, as the seat would have said it, and the view asks its holder once
|
||||
// more on its own goroutine. A branch with no view, or a view that holds the revision, asks nothing.
|
||||
func (v *Views) Again(ctx context.Context, branch string, revision int64) error {
|
||||
v.mu.Lock()
|
||||
bv := v.branches[branch]
|
||||
v.mu.Unlock()
|
||||
if bv == nil || revision <= 0 {
|
||||
return nil
|
||||
}
|
||||
if h, ok := bv.View(); ok && h.Revision >= revision {
|
||||
return nil
|
||||
}
|
||||
data, err := json.Marshal(catalog.CatalogRevisionEvent{V: 1, Kind: "trunk", Branch: keys.EscapeName(branch), Revision: revision})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return bv.Hear(ctx, data)
|
||||
}
|
||||
|
||||
// HearRevision hands one catalog-revision event to every branch's view.
|
||||
func (v *Views) HearRevision(ctx context.Context, data []byte) error {
|
||||
v.mu.Lock()
|
||||
|
||||
@@ -72,6 +72,14 @@ func TestPolicyAndBuildSeat(t *testing.T) {
|
||||
if opened != 4 {
|
||||
t.Errorf("a branch whose view was refused is opened again at the next registration: %d opens", opened)
|
||||
}
|
||||
// A-490k: a view asked again for a revision it does not hold hears it as the seat's event; one that holds it asks nothing.
|
||||
if err := v.Again(context.Background(), "main", 9); err != nil || heard["main"] != 1 {
|
||||
t.Errorf("a view at 7 asked again for 9 heard %d events (%v), want 1", heard["main"], err)
|
||||
}
|
||||
if err := v.Again(context.Background(), "main", 7); err != nil || heard["main"] != 1 {
|
||||
t.Errorf("a view at 7 asked again for 7 heard %d events (%v), want none more", heard["main"], err)
|
||||
}
|
||||
heard["main"] = 0
|
||||
if err := v.HearRevision(context.Background(), []byte(`{}`)); err != nil || heard["main"] != 1 || heard["feature/x"] != 1 {
|
||||
t.Errorf("a revision heard reaches every branch's view: %v %v", err, heard)
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
|
||||
@@ -98,8 +99,17 @@ type Starter struct {
|
||||
Runs RunSettings
|
||||
RunsFill func() (seat.Fill, error)
|
||||
EndReaders func(run string, readers []seat.TestRunReader, r seat.TestRun) error
|
||||
// Again asks the view of a branch's catalog half again, for a revision a take-in named that the view does not hold yet
|
||||
// (design 47 G A-490k): the answer comes as the view's own events, with no timer here; nil asks nothing.
|
||||
Again func(branch string, revision int64)
|
||||
|
||||
heldMu sync.Mutex
|
||||
held map[string]heldStart
|
||||
}
|
||||
|
||||
// heldStart is the start of a branch's unstarted modules that a take-in made while the view was behind its revision.
|
||||
type heldStart struct{ source, revision int64 }
|
||||
|
||||
// holderless ends at once, as it is placed, a pipeline with a build step whose seat the seat mesh-module-assigner's view
|
||||
// seat-holders names no holder for (design 47 G §32, §535, §647 as A-466y and A-466z amend them; A-470g, A-470h): judged only
|
||||
// once that view's first view is read, never from the picture before it. Each such step ends `could not run` naming the seat,
|
||||
@@ -581,6 +591,12 @@ func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half
|
||||
} else if held && half.Revision < revision {
|
||||
s.logf("the take-in %s of %s made revision %d and the runner's view holds %d: its pipelines start with no step, placed once the view reaches it (§200)", ev.Key, ev.Branch, revision, half.Revision)
|
||||
held = false
|
||||
// The modules the take-in's batch moved and no pipeline carries start once the view reaches the revision (A-490k), and
|
||||
// the view is asked again now: it may have been answered by a holder that was behind.
|
||||
s.holdStart(ev.Branch, ev.Source, revision)
|
||||
if s.Again != nil {
|
||||
s.Again(ev.Branch, revision)
|
||||
}
|
||||
}
|
||||
started := 0
|
||||
var first error
|
||||
@@ -612,6 +628,16 @@ func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half
|
||||
}
|
||||
}
|
||||
}
|
||||
// The held start runs here when the view reached the revision while the take-in was being read (A-490k).
|
||||
if !held && revision > 0 {
|
||||
if again, ok := view(ev.Branch); ok && again.Revision >= revision {
|
||||
n, err := s.StartHeld(ev.Branch, again)
|
||||
started += n
|
||||
if err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
}
|
||||
}
|
||||
return started, first
|
||||
}
|
||||
|
||||
@@ -622,6 +648,37 @@ func commitOrGroup(p seat.Pipeline) string {
|
||||
return p.Commit
|
||||
}
|
||||
|
||||
func (s *Starter) holdStart(branch string, source, revision int64) {
|
||||
s.heldMu.Lock()
|
||||
defer s.heldMu.Unlock()
|
||||
if s.held == nil {
|
||||
s.held = map[string]heldStart{}
|
||||
}
|
||||
if h, ok := s.held[branch]; ok && h.revision >= revision {
|
||||
return
|
||||
}
|
||||
s.held[branch] = heldStart{source: source, revision: revision}
|
||||
}
|
||||
|
||||
// StartHeld runs the start a take-in held while the view of branch was behind its revision, once half holds that revision
|
||||
// (A-490k): the modules of half with no pipeline at their fingerprint start at the take-in's source revision. It says how
|
||||
// many it started; with nothing held, or the view still behind, none.
|
||||
func (s *Starter) StartHeld(branch string, half Half) (int, error) {
|
||||
s.heldMu.Lock()
|
||||
h, ok := s.held[branch]
|
||||
if !ok || half.Revision < h.revision {
|
||||
s.heldMu.Unlock()
|
||||
return 0, nil
|
||||
}
|
||||
delete(s.held, branch)
|
||||
s.heldMu.Unlock()
|
||||
n, err := s.StartUnstarted(branch, h.source, half)
|
||||
if err != nil {
|
||||
s.holdStart(branch, h.source, h.revision) // held again: the next advance runs it
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Unstarted names the modules of half whose fingerprint has no pipeline on branch: no pipeline of any outcome, a published
|
||||
// one included, carries that module at that fingerprint. A run on request is no pipeline of the branch's head. Sorted.
|
||||
func (s *Starter) Unstarted(branch string, half Half) ([]string, error) {
|
||||
|
||||
@@ -314,3 +314,44 @@ func TestTakenInStartsEveryModuleWithNoPipelineAtItsFingerprint(t *testing.T) {
|
||||
t.Errorf("a half with no source revision started %d (%v)", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A-490k: a take-in whose revision the runner's view has not reached asks the view again, holds the start of the modules its batch
|
||||
// moved, and runs it once the view reaches the revision, with no timer; a take-in heard with the view behind starts none of them first.
|
||||
func TestATakeInBehindTheViewAsksTheViewAgainAndStartsTheHeldModulesWhenItReaches(t *testing.T) {
|
||||
mem := memStore{}
|
||||
s := &Starter{Pipelines: memCreated{mem}, Store: mem, Say: func(string, any) error { return nil }}
|
||||
var asked []int64
|
||||
s.Again = func(branch string, revision int64) {
|
||||
if branch != "main" {
|
||||
t.Errorf("the view of %s is asked again, want main", branch)
|
||||
}
|
||||
asked = append(asked, revision)
|
||||
}
|
||||
entries := map[string]catalog.CatalogEntry{"moved": {Commit: "c1", Fp: "src2:aa"}, "batched": {Commit: "c2", Fp: "src2:bb"}}
|
||||
half := Half{Revision: 11, Source: 8, Entries: entries}
|
||||
view := func(string) (Half, bool) { return half, true }
|
||||
ev := catalog.TakeInTaken{Kind: "taken", Branch: "main", Source: 9, Revision: 12,
|
||||
Entries: []catalog.TakeInEntry{{Module: "moved", Commit: "c1", Fingerprint: "src2:aa"}}}
|
||||
if n, err := s.TakenIn(ev, view); err != nil || n != 1 {
|
||||
t.Fatalf("the take-in started %d (%v), want its one entry", n, err)
|
||||
}
|
||||
if len(asked) != 1 || asked[0] != 12 {
|
||||
t.Errorf("the view was asked again for revisions %v, want [12]", asked)
|
||||
}
|
||||
if _, _, err := mem.Get("main.9.batched"); err == nil {
|
||||
t.Fatal("main.9.batched started while the view is behind the take-in's revision")
|
||||
}
|
||||
if n, err := s.StartHeld("main", half); n != 0 || err != nil {
|
||||
t.Errorf("the view still at 11 started %d held (%v), want none", n, err)
|
||||
}
|
||||
half = Half{Revision: 12, Source: 9, Entries: entries}
|
||||
if n, err := s.StartHeld("main", half); err != nil || n != 1 {
|
||||
t.Fatalf("the view at 12 started %d held (%v), want batched", n, err)
|
||||
}
|
||||
if _, _, err := mem.Get("main.9.batched"); err != nil {
|
||||
t.Errorf("main.9.batched is not started once the view reached 12: %v", err)
|
||||
}
|
||||
if n, err := s.StartHeld("main", half); n != 0 || err != nil {
|
||||
t.Errorf("the held start ran twice: %d (%v)", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user