indexer: take in a merge it never took in, and say so while it is behind (A-463)

At its start and on every source revision the indexer compares each repository's taken merge with the head the gate's source names; a repository behind is taken in as its merged message would have been, with the finding merge-untaken raised until it is.

Builds: design 47 G-22 A-463
This commit is contained in:
2026-11-17 01:55:46 +01:00
parent 61843f823d
commit 250a0f1b12
4 changed files with 228 additions and 10 deletions
@@ -0,0 +1,162 @@
package index
import (
"context"
"errors"
"fmt"
"maps"
"slices"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
)
// A merge the indexer never took in (design 47 G A-463, amended by A-464): at its start and on every source revision the
// gate's source names, the indexer compares each repository's `taken` merge with the head that source names. A repository
// whose head is newer than its taken merge (the taken merge is that head's ancestor, or no merge is taken) is taken in as
// its merged message would have been (the catch-up of indexer_take_in, for that repository), said in the log, with the
// finding merge-untaken raised while it stays behind and cleared once it is taken in. A head the taken merge contains (the
// source not yet moved to a merge just taken in) is left, and a head the taken merge neither contains nor is in is said.
// UntakenMergeKind is the finding of a repository whose head the gate's source names but whose merge the indexer never took in.
const UntakenMergeKind = "merge-untaken"
func untakenFindingID(own, key string) string { return own + ".untaken." + key }
// kickBehind runs the comparison in the background, one at a time: a source revision that arrives while one runs makes it
// run once more, on the latest heads. It runs after the first start's catch-up, which takes every head in itself.
func (ix *Indexer) kickBehind(ctx context.Context) {
if ix.Taken == nil || ix.Message == nil || ix.Modules == nil || ix.catchingUp.Load() {
return
}
if ix.behindRunning.Swap(true) {
ix.behindAgain.Store(true)
return
}
ix.catchUpWG.Add(1)
go func() {
defer ix.catchUpWG.Done()
for {
ix.behindAgain.Store(false)
ix.TakeInBehind(ctx)
if !ix.behindAgain.Load() {
ix.behindRunning.Store(false)
if !ix.behindAgain.Load() || ix.behindRunning.Swap(true) {
return
}
}
}
}()
}
// TakeInBehind compares each repository's taken merge with the head the latest source names, takes in each repository
// that is behind and answers the repositories it took in.
func (ix *Indexer) TakeInBehind(ctx context.Context) []string {
heads := ix.sourceHeads()
if ix.Taken == nil || ix.Message == nil || ix.Modules == nil || len(heads) == 0 {
return nil
}
behind := map[string]string{}
taken := map[string]string{}
for _, repository := range slices.Sorted(maps.Keys(heads)) {
head := heads[repository]
key, err := trunk.RepositoryKey(repository)
if head == "" || err != nil {
continue
}
value, _, found, err := ix.Taken.Get(ctx, key)
if errors.Is(err, process.ErrNotYet) {
return nil // the state is not on the bus yet: the next source revision compares again
}
if err != nil {
ix.logf("merges: taken.%s cannot be read, so %s is not compared with its head: %v", key, repository, err)
continue
}
merge := ""
if found {
t, err := takein.DecodeTaken(value)
if err != nil {
ix.logf("merges: taken.%s holds what is not a taken merge, left: %v", key, err)
continue
}
merge = t.Merge
}
switch {
case merge == head:
ix.clearUntaken(ctx, key, "the indexer took in the head of "+repository)
case merge == "":
behind[repository], taken[repository] = head, ""
case ix.Contains == nil:
ix.logf("merges: %s is taken in at %.12s and its head is %.12s, but the indexer reads no ancestry of the forge, so which is newer is not said", repository, merge, head)
default:
inHead, err := ix.Contains(ctx, repository, merge, head)
if err != nil {
ix.logf("merges: whether %.12s is in %.12s of %s cannot be read: %v", merge, head, repository, err)
continue
}
if inHead {
behind[repository], taken[repository] = head, merge
continue
}
ix.clearUntaken(ctx, key, "the head of "+repository+" is in the merge taken in")
if headIn, err := ix.Contains(ctx, repository, head, merge); err == nil && !headIn {
ix.logf("merges: %s is taken in at %.12s and its head is %.12s, and neither holds the other: left", repository, merge, head)
}
}
}
for _, repository := range slices.Sorted(maps.Keys(behind)) {
ix.sayUntaken(ctx, repository, taken[repository], behind[repository])
ix.logf("merges: %s is taken in at %.12s but the gate's source names %.12s: taken in now", repository, taken[repository], behind[repository])
}
if len(behind) == 0 {
return nil
}
unread := ix.takeInHeads(ctx, behind)
var done []string
for _, repository := range slices.Sorted(maps.Keys(behind)) {
if slices.Contains(unread, repository) {
continue
}
done = append(done, repository)
key, _ := trunk.RepositoryKey(repository)
ix.clearUntaken(ctx, key, fmt.Sprintf("%s taken in at %.12s", repository, behind[repository]))
ix.logf("merges: %s taken in at %.12s", repository, behind[repository])
}
if len(done) > 0 {
if err := ix.KeepView(ctx); err != nil {
ix.logf("merges: the view could not be kept after %v were taken in: %v", done, err)
}
}
return done
}
func (ix *Indexer) sayUntaken(ctx context.Context, repository, merge, head string) {
if ix.Findings == nil {
return
}
key, _ := trunk.RepositoryKey(repository)
taken := "no merge"
if merge != "" {
taken = fmt.Sprintf("%.12s", merge)
}
f := ix.observed(mcc.ScopeModule, untakenFindingID(ix.Module, key), UntakenMergeKind,
fmt.Sprintf("the indexer took in %s at %s, and the gate's source names the head %.12s", repository, taken, head),
"A merge the indexer never took in",
"A merge into a watched branch was never heard or applied, so no package or module of the repository follows its trunk. The indexer takes the head in itself and clears this; indexer_take_in with catch-up does it by hand.",
UntakenMergeKind)
if err := ix.Findings.Write(ctx, f); err != nil {
ix.logf("merges: the finding %s for %s is not written: %v", UntakenMergeKind, repository, err)
}
}
func (ix *Indexer) clearUntaken(ctx context.Context, key, why string) {
if ix.Findings == nil {
return
}
if err := ix.clearFinding(ctx, mcc.ScopeModule, untakenFindingID(ix.Module, key), UntakenMergeKind, why); err != nil {
ix.logf("merges: the finding %s for %s is not cleared: %v", UntakenMergeKind, key, err)
}
}
@@ -0,0 +1,52 @@
package index
import (
"context"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
)
// Design 47 G A-463: a merge the indexer never took in is noticed at a source revision and taken in as its merged message
// would have been, with a finding raised while it stays behind and cleared once it is taken in.
func TestAMergeTheIndexerNeverTookInIsTakenInAndSaidWhileItIsBehind(t *testing.T) {
ctx := context.Background()
b := newTakeInBed(t)
b.ix.Taken, b.ix.Findings = newMem(), newFindings()
key, _ := trunk.RepositoryKey(catalogueRepo)
old, head := sha('7'), sha('8')
data, _ := takein.EncodeTaken(takein.Taken{Repository: trunk.RepositoryName(catalogueRepo), Merge: old})
b.ix.Taken.(*memStore).put(key, data)
ancestry := map[[2]string]bool{{old, head}: true} // old is in head
b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { return commit == in || ancestry[[2]string{commit, in}], nil }
b.ix.source = sourceNow{rev: 3, heads: map[string]string{catalogueRepo: head}}
// The head has no merged message yet: nothing is taken in and the finding stays.
if done := b.ix.TakeInBehind(ctx); len(done) != 0 || len(b.ix.Findings.Held()) != 1 || b.ix.Findings.Held()[0].Kind != UntakenMergeKind {
t.Fatalf("a head whose message is not read: taken in %v, findings %+v: want none taken in and %s raised", done, b.ix.Findings.Held(), UntakenMergeKind)
}
// Its message is there: it is taken in, its taken key moves and the finding is cleared.
b.publish(t, b.merge(30, head, "tree-avahi-3", ""))
if done := b.ix.TakeInBehind(ctx); len(done) != 1 || len(b.ix.Findings.Held()) != 0 || b.commit(t, "avahi") != head {
t.Fatalf("a head with its message: taken in %v, findings %+v, avahi at %s: want it taken in at %s and the finding cleared", done, b.ix.Findings.Held(), b.commit(t, "avahi"), head)
}
if v, _, _, _ := b.ix.Taken.Get(ctx, key); string(v) == string(data) {
t.Errorf("taken.%s still names %s", key, old)
}
// A head the taken merge already holds (the source not yet moved to a merge just taken in) is left.
b.ix.source = sourceNow{rev: 4, heads: map[string]string{catalogueRepo: old}}
if done := b.ix.TakeInBehind(ctx); len(done) != 0 {
t.Errorf("a head older than the taken merge: taken in %v, want none", done)
}
// A source revision starts the comparison by itself (no timer): the taken key is put back to the old merge and the source
// names the head again.
b.ix.Taken.(*memStore).put(key, data)
if err := applyS(b.ix, ctx, 5, sourceOf(catalogueRepo, head)); err != nil {
t.Fatal(err)
}
b.ix.catchUpWG.Wait()
if v, _, _, _ := b.ix.Taken.Get(ctx, key); string(v) == string(data) {
t.Errorf("a source revision naming %s left taken.%s at %s", head, key, old)
}
}
@@ -200,16 +200,18 @@ type Indexer struct {
keepMu sync.Mutex // one catalog graph revision kept at a time (keepEntries)
// Pins is the state pins (pins.go); nil keeps none. PinsPending says how many messages the indexer's durable consumer of the
// events stream has not been given: a revision is removed only while none is pending (§579).
Pins Store
PinsPending func(ctx context.Context) (uint64, error)
namesMu sync.Mutex // packageNames
packageNames map[string]string // the package names read through the forge, by repository, commit, path and artifact (artifacts.go)
catchingUp atomic.Bool // the first start's catch-up runs: its merges keep no revision until it ends (CatchUp)
catchUpWG sync.WaitGroup // the first start's catch-up in the background (kickCatchUp), for tests to wait on
ownFP string // the fingerprint of the build this program runs, fixed once at the start (FixOwnBuild)
ownFixed bool // FixOwnBuild asked already: it asks once
ownSince time.Time // when the build was asked and not known, with ownWhy
ownWhy string
Pins Store
PinsPending func(ctx context.Context) (uint64, error)
namesMu sync.Mutex // packageNames
packageNames map[string]string // the package names read through the forge, by repository, commit, path and artifact (artifacts.go)
catchingUp atomic.Bool // the first start's catch-up runs: its merges keep no revision until it ends (CatchUp)
behindRunning atomic.Bool // a comparison of the taken merges with the source's heads runs (kickBehind, A-463)
behindAgain atomic.Bool // a source revision came while it ran
catchUpWG sync.WaitGroup // the first start's catch-up in the background (kickCatchUp), for tests to wait on
ownFP string // the fingerprint of the build this program runs, fixed once at the start (FixOwnBuild)
ownFixed bool // FixOwnBuild asked already: it asks once
ownSince time.Time // when the build was asked and not known, with ownWhy
ownWhy string
// OwnBuildState keeps which build this process knows it runs, or since when and why not (FixOwnBuild); nil keeps none.
OwnBuildState Store
// Heard keeps what the indexer heard and compares later (A-80): the time a call is unanswered since (steps-since, call-since),
@@ -77,6 +77,8 @@ func (ix *Indexer) ApplySource(ctx context.Context, a meshdelivery.SourceAnswer)
if err := ix.kickCatchUp(ctx, s.Heads); err != nil {
return err
}
// A merge the indexer never took in is taken in at each source revision, and at the start (A-463).
ix.kickBehind(ctx)
// A registered module the forge could not read is read again at each new source revision (issue 1945), in the
// background: a forge that does not answer must not hold this revision. One taken in later is stamped when it is written
// (applyManifests reads srcOf).