indexer: a head the pending holds is not behind; a revision heard during a catch-up is compared after it (A-463a)
Builds: design 47 G-22 A-463a, A-463
This commit is contained in:
@@ -29,7 +29,11 @@ func untakenFindingID(own, key string) string { return own + ".untaken." + key }
|
||||
// kickBehind runs the comparison in the background, one at a time: a source revision that arrives while one runs makes it
|
||||
// run once more, on the latest heads. It runs after the first start's catch-up, which takes every head in itself.
|
||||
func (ix *Indexer) kickBehind(ctx context.Context) {
|
||||
if ix.Taken == nil || ix.Message == nil || ix.Modules == nil || ix.catchingUp.Load() {
|
||||
if ix.Taken == nil || ix.Message == nil || ix.Modules == nil {
|
||||
return
|
||||
}
|
||||
if ix.catchingUp.Load() {
|
||||
ix.behindWaiting.Store(true) // compared again when that catch-up ends (A-463a)
|
||||
return
|
||||
}
|
||||
if ix.behindRunning.Swap(true) {
|
||||
@@ -61,6 +65,7 @@ func (ix *Indexer) TakeInBehind(ctx context.Context) []string {
|
||||
}
|
||||
behind := map[string]string{}
|
||||
taken := map[string]string{}
|
||||
held := ix.pendingHeads(ctx)
|
||||
for _, repository := range slices.Sorted(maps.Keys(heads)) {
|
||||
head := heads[repository]
|
||||
key, err := trunk.RepositoryKey(repository)
|
||||
@@ -85,6 +90,10 @@ func (ix *Indexer) TakeInBehind(ctx context.Context) []string {
|
||||
merge = t.Merge
|
||||
}
|
||||
switch {
|
||||
case held[head]:
|
||||
// A head the pending holds (a member waiting for its group's end or its build) is taken in by that path, whole with
|
||||
// its group: it is not behind (A-463a).
|
||||
ix.clearUntaken(ctx, key, "the head of "+repository+" waits in the indexer's pending")
|
||||
case merge == head:
|
||||
ix.clearUntaken(ctx, key, "the indexer took in the head of "+repository)
|
||||
case merge == "":
|
||||
@@ -160,3 +169,22 @@ func (ix *Indexer) clearUntaken(ctx context.Context, key, why string) {
|
||||
ix.logf("merges: the finding %s for %s is not cleared: %v", UntakenMergeKind, key, err)
|
||||
}
|
||||
}
|
||||
|
||||
// pendingHeads are the merge commits the indexer's pending holds (A-463a).
|
||||
func (ix *Indexer) pendingHeads(ctx context.Context) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
if ix.Pending == nil || !ix.pendingOnBus(ctx) {
|
||||
return out
|
||||
}
|
||||
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
|
||||
if err != nil {
|
||||
ix.logf("merges: the pending state cannot be read, so every head is compared: %v", err)
|
||||
return out
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Forge.Merge != "" {
|
||||
out[e.Forge.Merge] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -2,8 +2,11 @@ package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
)
|
||||
@@ -19,7 +22,9 @@ func TestAMergeTheIndexerNeverTookInIsTakenInAndSaidWhileItIsBehind(t *testing.T
|
||||
data, _ := takein.EncodeTaken(takein.Taken{Repository: trunk.RepositoryName(catalogueRepo), Merge: old})
|
||||
b.ix.Taken.(*memStore).put(key, data)
|
||||
ancestry := map[[2]string]bool{{old, head}: true} // old is in head
|
||||
b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { return commit == in || ancestry[[2]string{commit, in}], nil }
|
||||
b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) {
|
||||
return commit == in || ancestry[[2]string{commit, in}], nil
|
||||
}
|
||||
b.ix.source = sourceNow{rev: 3, heads: map[string]string{catalogueRepo: head}}
|
||||
|
||||
// The head has no merged message yet: nothing is taken in and the finding stays.
|
||||
@@ -50,3 +55,58 @@ func TestAMergeTheIndexerNeverTookInIsTakenInAndSaidWhileItIsBehind(t *testing.T
|
||||
t.Errorf("a source revision naming %s left taken.%s at %s", head, key, old)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G A-463a: a head the pending holds (a group member waiting for its group's end) is not behind: it raises no finding
|
||||
// and is not taken in alone.
|
||||
func TestAHeadThePendingHoldsIsNotBehind(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
b := newTakeInBed(t)
|
||||
b.ix.Taken, b.ix.Findings = newMem(), newFindings()
|
||||
key, _ := trunk.RepositoryKey(catalogueRepo)
|
||||
old, head := sha('7'), sha('8')
|
||||
data, _ := takein.EncodeTaken(takein.Taken{Repository: trunk.RepositoryName(catalogueRepo), Merge: old})
|
||||
b.ix.Taken.(*memStore).put(key, data)
|
||||
b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { return commit == in || commit == old, nil }
|
||||
b.publish(t, b.merge(30, head, "tree-avahi-3", ""))
|
||||
// The head waits in pending as a member of a group not ended.
|
||||
held := b.merge(30, head, "tree-avahi-3", "")
|
||||
raw, _ := forgemsg.EncodeForge(held)
|
||||
pv, _ := json.Marshal(Pending{V: 1, Forge: raw, Group: "feat", Since: time.Now(), LastWriter: "x"})
|
||||
b.pending.put(takein.PendingKey(1), pv)
|
||||
b.ix.source = sourceNow{rev: 3, heads: map[string]string{catalogueRepo: head}}
|
||||
if done := b.ix.TakeInBehind(ctx); len(done) != 0 || len(b.ix.Findings.Held()) != 0 || b.commit(t, "avahi") == head {
|
||||
t.Fatalf("a head the pending holds: taken in %v, findings %+v, avahi at %s: want none taken in and none raised", done, b.ix.Findings.Held(), b.commit(t, "avahi"))
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G A-463a: a source revision heard while a catch-up runs is compared again when that catch-up ends.
|
||||
func TestASourceRevisionHeardDuringACatchUpIsComparedWhenItEnds(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
b := newTakeInBed(t)
|
||||
b.ix.Taken, b.ix.Findings = newMem(), newFindings()
|
||||
key, _ := trunk.RepositoryKey(catalogueRepo)
|
||||
h1, h2 := sha('8'), sha('9')
|
||||
b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) {
|
||||
return commit == in || (commit == h1 && in == h2), nil
|
||||
}
|
||||
b.publish(t, b.merge(30, h1, "tree-avahi-3", ""))
|
||||
b.publish(t, b.merge(31, h2, "tree-avahi-4", ""))
|
||||
b.ix.source = sourceNow{rev: 3, heads: map[string]string{catalogueRepo: h1}}
|
||||
inner := b.ix.Message
|
||||
heard := false
|
||||
b.ix.Message = func(c context.Context, r, m string, n int) ([]byte, error) {
|
||||
if !heard {
|
||||
heard = true // the catch-up of h1 runs: a new source revision is heard now
|
||||
if err := applyS(b.ix, ctx, 4, sourceOf(catalogueRepo, h2)); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
return inner(c, r, m, n)
|
||||
}
|
||||
b.ix.CatchUp(ctx)
|
||||
b.ix.catchUpWG.Wait()
|
||||
v, _, _, _ := b.ix.Taken.Get(ctx, key)
|
||||
if got, _ := takein.DecodeTaken(v); got.Merge != h2 {
|
||||
t.Errorf("taken after the catch-up = %s, want %s: the revision heard during it is compared afterwards\n%s", got.Merge, h2, b.logs)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -206,6 +206,7 @@ type Indexer struct {
|
||||
packageNames map[string]string // the package names read through the forge, by repository, commit, path and artifact (artifacts.go)
|
||||
catchingUp atomic.Bool // the first start's catch-up runs: its merges keep no revision until it ends (CatchUp)
|
||||
behindRunning atomic.Bool // a comparison of the taken merges with the source's heads runs (kickBehind, A-463)
|
||||
behindWaiting atomic.Bool // a source revision came while a catch-up ran (A-463a)
|
||||
behindAgain atomic.Bool // a source revision came while it ran
|
||||
catchUpWG sync.WaitGroup // the first start's catch-up in the background (kickCatchUp), for tests to wait on
|
||||
ownFP string // the fingerprint of the build this program runs, fixed once at the start (FixOwnBuild)
|
||||
|
||||
@@ -178,6 +178,9 @@ func (ix *Indexer) kickCatchUp(ctx context.Context, heads map[string]string) err
|
||||
defer ix.catchUpWG.Done()
|
||||
unread := ix.takeInHeads(ctx, heads)
|
||||
ix.catchingUp.Store(false)
|
||||
if ix.behindWaiting.Swap(false) {
|
||||
defer ix.kickBehind(ctx)
|
||||
}
|
||||
if len(unread) > 0 {
|
||||
ix.logf("catalog graph: the first start's catch-up ended and no first revision is kept: %d repositories were not taken in (%s); taken in again at the next source revision",
|
||||
len(unread), strings.Join(unread, ", "))
|
||||
@@ -242,6 +245,9 @@ func (ix *Indexer) CatchUp(ctx context.Context) TakenIn {
|
||||
}
|
||||
unread := ix.takeInHeads(ctx, heads)
|
||||
ix.catchingUp.Store(false)
|
||||
if ix.behindWaiting.Swap(false) {
|
||||
defer ix.kickBehind(ctx)
|
||||
}
|
||||
if err := ix.KeepView(ctx); err != nil {
|
||||
out.Answer = "the repositories were taken in but the revision could not be kept: " + err.Error()
|
||||
return out
|
||||
|
||||
Reference in New Issue
Block a user