diff --git a/modules/mesh-delivery-indexer/internal/index/behind.go b/modules/mesh-delivery-indexer/internal/index/behind.go new file mode 100644 index 00000000..81c02951 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/behind.go @@ -0,0 +1,190 @@ +package index + +import ( + "context" + "errors" + "fmt" + "maps" + "slices" + + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + "git.novox.be/novox/mesh-sdk/go/process" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" +) + +// A merge the indexer never took in (design 47 G A-463, amended by A-464): at its start and on every source revision the +// gate's source names, the indexer compares each repository's `taken` merge with the head that source names. A repository +// whose head is newer than its taken merge (the taken merge is that head's ancestor, or no merge is taken) is taken in as +// its merged message would have been (the catch-up of indexer_take_in, for that repository), said in the log, with the +// finding merge-untaken raised while it stays behind and cleared once it is taken in. A head the taken merge contains (the +// source not yet moved to a merge just taken in) is left, and a head the taken merge neither contains nor is in is said. + +// UntakenMergeKind is the finding of a repository whose head the gate's source names but whose merge the indexer never took in. +const UntakenMergeKind = "merge-untaken" + +func untakenFindingID(own, key string) string { return own + ".untaken." + key } + +// kickBehind runs the comparison in the background, one at a time: a source revision that arrives while one runs makes it +// run once more, on the latest heads. It runs after the first start's catch-up, which takes every head in itself. +func (ix *Indexer) kickBehind(ctx context.Context) { + if ix.Taken == nil || ix.Message == nil || ix.Modules == nil { + return + } + if ix.catchingUp.Load() { + ix.behindWaiting.Store(true) // compared again when that catch-up ends (A-463a) + return + } + if ix.behindRunning.Swap(true) { + ix.behindAgain.Store(true) + return + } + ix.catchUpWG.Add(1) + go func() { + defer ix.catchUpWG.Done() + for { + ix.behindAgain.Store(false) + ix.TakeInBehind(ctx) + if !ix.behindAgain.Load() { + ix.behindRunning.Store(false) + if !ix.behindAgain.Load() || ix.behindRunning.Swap(true) { + return + } + } + } + }() +} + +// TakeInBehind compares each repository's taken merge with the head the latest source names, takes in each repository +// that is behind and answers the repositories it took in. +func (ix *Indexer) TakeInBehind(ctx context.Context) []string { + heads := ix.sourceHeads() + if ix.Taken == nil || ix.Message == nil || ix.Modules == nil || len(heads) == 0 { + return nil + } + behind := map[string]string{} + taken := map[string]string{} + held := ix.pendingHeads(ctx) + for _, repository := range slices.Sorted(maps.Keys(heads)) { + head := heads[repository] + key, err := trunk.RepositoryKey(repository) + if head == "" || err != nil { + continue + } + value, _, found, err := ix.Taken.Get(ctx, key) + if errors.Is(err, process.ErrNotYet) { + return nil // the state is not on the bus yet: the next source revision compares again + } + if err != nil { + ix.logf("merges: taken.%s cannot be read, so %s is not compared with its head: %v", key, repository, err) + continue + } + merge := "" + if found { + t, err := takein.DecodeTaken(value) + if err != nil { + ix.logf("merges: taken.%s holds what is not a taken merge, left: %v", key, err) + continue + } + merge = t.Merge + } + switch { + case held[head]: + // A head the pending holds (a member waiting for its group's end or its build) is taken in by that path, whole with + // its group: it is not behind (A-463a). + ix.clearUntaken(ctx, key, "the head of "+repository+" waits in the indexer's pending") + case merge == head: + ix.clearUntaken(ctx, key, "the indexer took in the head of "+repository) + case merge == "": + behind[repository], taken[repository] = head, "" + case ix.Contains == nil: + ix.logf("merges: %s is taken in at %.12s and its head is %.12s, but the indexer reads no ancestry of the forge, so which is newer is not said", repository, merge, head) + default: + inHead, err := ix.Contains(ctx, repository, merge, head) + if err != nil { + ix.logf("merges: whether %.12s is in %.12s of %s cannot be read: %v", merge, head, repository, err) + continue + } + if inHead { + behind[repository], taken[repository] = head, merge + continue + } + ix.clearUntaken(ctx, key, "the head of "+repository+" is in the merge taken in") + if headIn, err := ix.Contains(ctx, repository, head, merge); err == nil && !headIn { + ix.logf("merges: %s is taken in at %.12s and its head is %.12s, and neither holds the other: left", repository, merge, head) + } + } + } + for _, repository := range slices.Sorted(maps.Keys(behind)) { + ix.sayUntaken(ctx, repository, taken[repository], behind[repository]) + ix.logf("merges: %s is taken in at %.12s but the gate's source names %.12s: taken in now", repository, taken[repository], behind[repository]) + } + if len(behind) == 0 { + return nil + } + unread := ix.takeInHeads(ctx, behind) + var done []string + for _, repository := range slices.Sorted(maps.Keys(behind)) { + if slices.Contains(unread, repository) { + continue + } + done = append(done, repository) + key, _ := trunk.RepositoryKey(repository) + ix.clearUntaken(ctx, key, fmt.Sprintf("%s taken in at %.12s", repository, behind[repository])) + ix.logf("merges: %s taken in at %.12s", repository, behind[repository]) + } + if len(done) > 0 { + if err := ix.KeepView(ctx); err != nil { + ix.logf("merges: the view could not be kept after %v were taken in: %v", done, err) + } + } + return done +} + +func (ix *Indexer) sayUntaken(ctx context.Context, repository, merge, head string) { + if ix.Findings == nil { + return + } + key, _ := trunk.RepositoryKey(repository) + taken := "no merge" + if merge != "" { + taken = fmt.Sprintf("%.12s", merge) + } + f := ix.observed(mcc.ScopeModule, untakenFindingID(ix.Module, key), UntakenMergeKind, + fmt.Sprintf("the indexer took in %s at %s, and the gate's source names the head %.12s", repository, taken, head), + "A merge the indexer never took in", + "A merge into a watched branch was never heard or applied, so no package or module of the repository follows its trunk. The indexer takes the head in itself and clears this; indexer_take_in with catch-up does it by hand.", + UntakenMergeKind) + if err := ix.Findings.Write(ctx, f); err != nil { + ix.logf("merges: the finding %s for %s is not written: %v", UntakenMergeKind, repository, err) + } +} + +func (ix *Indexer) clearUntaken(ctx context.Context, key, why string) { + if ix.Findings == nil { + return + } + if err := ix.clearFinding(ctx, mcc.ScopeModule, untakenFindingID(ix.Module, key), UntakenMergeKind, why); err != nil { + ix.logf("merges: the finding %s for %s is not cleared: %v", UntakenMergeKind, key, err) + } +} + +// pendingHeads are the merge commits the indexer's pending holds (A-463a). +func (ix *Indexer) pendingHeads(ctx context.Context) map[string]bool { + out := map[string]bool{} + if ix.Pending == nil || !ix.pendingOnBus(ctx) { + return out + } + entries, err := ReadPending(ctx, ix.Pending, ix.logf) + if err != nil { + ix.logf("merges: the pending state cannot be read, so every head is compared: %v", err) + return out + } + for _, e := range entries { + if e.Forge.Merge != "" { + out[e.Forge.Merge] = true + } + } + return out +} diff --git a/modules/mesh-delivery-indexer/internal/index/behind_test.go b/modules/mesh-delivery-indexer/internal/index/behind_test.go new file mode 100644 index 00000000..770eb47d --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/behind_test.go @@ -0,0 +1,112 @@ +package index + +import ( + "context" + "encoding/json" + "testing" + "time" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" +) + +// Design 47 G A-463: a merge the indexer never took in is noticed at a source revision and taken in as its merged message +// would have been, with a finding raised while it stays behind and cleared once it is taken in. +func TestAMergeTheIndexerNeverTookInIsTakenInAndSaidWhileItIsBehind(t *testing.T) { + ctx := context.Background() + b := newTakeInBed(t) + b.ix.Taken, b.ix.Findings = newMem(), newFindings() + key, _ := trunk.RepositoryKey(catalogueRepo) + old, head := sha('7'), sha('8') + data, _ := takein.EncodeTaken(takein.Taken{Repository: trunk.RepositoryName(catalogueRepo), Merge: old}) + b.ix.Taken.(*memStore).put(key, data) + ancestry := map[[2]string]bool{{old, head}: true} // old is in head + b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { + return commit == in || ancestry[[2]string{commit, in}], nil + } + b.ix.source = sourceNow{rev: 3, heads: map[string]string{catalogueRepo: head}} + + // The head has no merged message yet: nothing is taken in and the finding stays. + if done := b.ix.TakeInBehind(ctx); len(done) != 0 || len(b.ix.Findings.Held()) != 1 || b.ix.Findings.Held()[0].Kind != UntakenMergeKind { + t.Fatalf("a head whose message is not read: taken in %v, findings %+v: want none taken in and %s raised", done, b.ix.Findings.Held(), UntakenMergeKind) + } + // Its message is there: it is taken in, its taken key moves and the finding is cleared. + b.publish(t, b.merge(30, head, "tree-avahi-3", "")) + if done := b.ix.TakeInBehind(ctx); len(done) != 1 || len(b.ix.Findings.Held()) != 0 || b.commit(t, "avahi") != head { + t.Fatalf("a head with its message: taken in %v, findings %+v, avahi at %s: want it taken in at %s and the finding cleared", done, b.ix.Findings.Held(), b.commit(t, "avahi"), head) + } + if v, _, _, _ := b.ix.Taken.Get(ctx, key); string(v) == string(data) { + t.Errorf("taken.%s still names %s", key, old) + } + // A head the taken merge already holds (the source not yet moved to a merge just taken in) is left. + b.ix.source = sourceNow{rev: 4, heads: map[string]string{catalogueRepo: old}} + if done := b.ix.TakeInBehind(ctx); len(done) != 0 { + t.Errorf("a head older than the taken merge: taken in %v, want none", done) + } + // A source revision starts the comparison by itself (no timer): the taken key is put back to the old merge and the source + // names the head again. + b.ix.Taken.(*memStore).put(key, data) + if err := applyS(b.ix, ctx, 5, sourceOf(catalogueRepo, head)); err != nil { + t.Fatal(err) + } + b.ix.catchUpWG.Wait() + if v, _, _, _ := b.ix.Taken.Get(ctx, key); string(v) == string(data) { + t.Errorf("a source revision naming %s left taken.%s at %s", head, key, old) + } +} + +// Design 47 G A-463a: a head the pending holds (a group member waiting for its group's end) is not behind: it raises no finding +// and is not taken in alone. +func TestAHeadThePendingHoldsIsNotBehind(t *testing.T) { + ctx := context.Background() + b := newTakeInBed(t) + b.ix.Taken, b.ix.Findings = newMem(), newFindings() + key, _ := trunk.RepositoryKey(catalogueRepo) + old, head := sha('7'), sha('8') + data, _ := takein.EncodeTaken(takein.Taken{Repository: trunk.RepositoryName(catalogueRepo), Merge: old}) + b.ix.Taken.(*memStore).put(key, data) + b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { return commit == in || commit == old, nil } + b.publish(t, b.merge(30, head, "tree-avahi-3", "")) + // The head waits in pending as a member of a group not ended. + held := b.merge(30, head, "tree-avahi-3", "") + raw, _ := forgemsg.EncodeForge(held) + pv, _ := json.Marshal(Pending{V: 1, Forge: raw, Group: "feat", Since: time.Now(), LastWriter: "x"}) + b.pending.put(takein.PendingKey(1), pv) + b.ix.source = sourceNow{rev: 3, heads: map[string]string{catalogueRepo: head}} + if done := b.ix.TakeInBehind(ctx); len(done) != 0 || len(b.ix.Findings.Held()) != 0 || b.commit(t, "avahi") == head { + t.Fatalf("a head the pending holds: taken in %v, findings %+v, avahi at %s: want none taken in and none raised", done, b.ix.Findings.Held(), b.commit(t, "avahi")) + } +} + +// Design 47 G A-463a: a source revision heard while a catch-up runs is compared again when that catch-up ends. +func TestASourceRevisionHeardDuringACatchUpIsComparedWhenItEnds(t *testing.T) { + ctx := context.Background() + b := newTakeInBed(t) + b.ix.Taken, b.ix.Findings = newMem(), newFindings() + key, _ := trunk.RepositoryKey(catalogueRepo) + h1, h2 := sha('8'), sha('9') + b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { + return commit == in || (commit == h1 && in == h2), nil + } + b.publish(t, b.merge(30, h1, "tree-avahi-3", "")) + b.publish(t, b.merge(31, h2, "tree-avahi-4", "")) + b.ix.source = sourceNow{rev: 3, heads: map[string]string{catalogueRepo: h1}} + inner := b.ix.Message + heard := false + b.ix.Message = func(c context.Context, r, m string, n int) ([]byte, error) { + if !heard { + heard = true // the catch-up of h1 runs: a new source revision is heard now + if err := applyS(b.ix, ctx, 4, sourceOf(catalogueRepo, h2)); err != nil { + t.Error(err) + } + } + return inner(c, r, m, n) + } + b.ix.CatchUp(ctx) + b.ix.catchUpWG.Wait() + v, _, _, _ := b.ix.Taken.Get(ctx, key) + if got, _ := takein.DecodeTaken(v); got.Merge != h2 { + t.Errorf("taken after the catch-up = %s, want %s: the revision heard during it is compared afterwards\n%s", got.Merge, h2, b.logs) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 8a6a521f..eaa2ed47 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -200,16 +200,19 @@ type Indexer struct { keepMu sync.Mutex // one catalog graph revision kept at a time (keepEntries) // Pins is the state pins (pins.go); nil keeps none. PinsPending says how many messages the indexer's durable consumer of the // events stream has not been given: a revision is removed only while none is pending (§579). - Pins Store - PinsPending func(ctx context.Context) (uint64, error) - namesMu sync.Mutex // packageNames - packageNames map[string]string // the package names read through the forge, by repository, commit, path and artifact (artifacts.go) - catchingUp atomic.Bool // the first start's catch-up runs: its merges keep no revision until it ends (CatchUp) - catchUpWG sync.WaitGroup // the first start's catch-up in the background (kickCatchUp), for tests to wait on - ownFP string // the fingerprint of the build this program runs, fixed once at the start (FixOwnBuild) - ownFixed bool // FixOwnBuild asked already: it asks once - ownSince time.Time // when the build was asked and not known, with ownWhy - ownWhy string + Pins Store + PinsPending func(ctx context.Context) (uint64, error) + namesMu sync.Mutex // packageNames + packageNames map[string]string // the package names read through the forge, by repository, commit, path and artifact (artifacts.go) + catchingUp atomic.Bool // the first start's catch-up runs: its merges keep no revision until it ends (CatchUp) + behindRunning atomic.Bool // a comparison of the taken merges with the source's heads runs (kickBehind, A-463) + behindWaiting atomic.Bool // a source revision came while a catch-up ran (A-463a) + behindAgain atomic.Bool // a source revision came while it ran + catchUpWG sync.WaitGroup // the first start's catch-up in the background (kickCatchUp), for tests to wait on + ownFP string // the fingerprint of the build this program runs, fixed once at the start (FixOwnBuild) + ownFixed bool // FixOwnBuild asked already: it asks once + ownSince time.Time // when the build was asked and not known, with ownWhy + ownWhy string // OwnBuildState keeps which build this process knows it runs, or since when and why not (FixOwnBuild); nil keeps none. OwnBuildState Store // Heard keeps what the indexer heard and compares later (A-80): the time a call is unanswered since (steps-since, call-since), diff --git a/modules/mesh-delivery-indexer/internal/index/source.go b/modules/mesh-delivery-indexer/internal/index/source.go index 88bb3bed..70e1d9fb 100644 --- a/modules/mesh-delivery-indexer/internal/index/source.go +++ b/modules/mesh-delivery-indexer/internal/index/source.go @@ -77,6 +77,8 @@ func (ix *Indexer) ApplySource(ctx context.Context, a meshdelivery.SourceAnswer) if err := ix.kickCatchUp(ctx, s.Heads); err != nil { return err } + // A merge the indexer never took in is taken in at each source revision, and at the start (A-463). + ix.kickBehind(ctx) // A registered module the forge could not read is read again at each new source revision (issue 1945), in the // background: a forge that does not answer must not hold this revision. One taken in later is stamped when it is written // (applyManifests reads srcOf). @@ -122,7 +124,7 @@ func (ix *Indexer) ApplySource(ctx context.Context, a meshdelivery.SourceAnswer) } } // A module built against the commit the source named as a repository's head (a package no module publishes, A-365) is - // computed again when that head moves: the gate moves the source only after the take-in. + // computed again when that head moves: the gate moves the source on the forge bridge's merged message (A-464), before the take-in, and at the take-in for a head no merged moved. if movedPackages(held, before, s.Heads) { if err := ix.Recompute(ctx); err != nil { return err @@ -176,6 +178,9 @@ func (ix *Indexer) kickCatchUp(ctx context.Context, heads map[string]string) err defer ix.catchUpWG.Done() unread := ix.takeInHeads(ctx, heads) ix.catchingUp.Store(false) + if ix.behindWaiting.Swap(false) { + defer ix.kickBehind(ctx) + } if len(unread) > 0 { ix.logf("catalog graph: the first start's catch-up ended and no first revision is kept: %d repositories were not taken in (%s); taken in again at the next source revision", len(unread), strings.Join(unread, ", ")) @@ -240,6 +245,9 @@ func (ix *Indexer) CatchUp(ctx context.Context) TakenIn { } unread := ix.takeInHeads(ctx, heads) ix.catchingUp.Store(false) + if ix.behindWaiting.Swap(false) { + defer ix.kickBehind(ctx) + } if err := ix.KeepView(ctx); err != nil { out.Answer = "the repositories were taken in but the revision could not be kept: " + err.Error() return out diff --git a/modules/mesh-delivery-indexer/internal/index/views.go b/modules/mesh-delivery-indexer/internal/index/views.go index 16327b47..3c5814a1 100644 --- a/modules/mesh-delivery-indexer/internal/index/views.go +++ b/modules/mesh-delivery-indexer/internal/index/views.go @@ -31,6 +31,7 @@ var ReadTools = []ViewedTool{ {Tool: "indexer_read_graph", State: graph.State, Words: "the module graph: each module's build and provision edges."}, {Tool: "indexer_read_heard", State: HeardState, Words: "the forge messages the indexer heard and what it did with each."}, {Tool: "indexer_read_taken", State: "taken", Words: "each repository's merge the indexer took in last."}, + {Tool: "indexer_read_takeins", State: TakeInsState, Words: "each take-in asked of the indexer: its ask, whether and when the indexer answered it and its answer (design 47 G A-464a)."}, {Tool: "indexer_read_findings", State: FindingsState, Words: "the indexer's findings, which the condition keeper raises and clears."}, {Tool: "indexer_read_seat_definitions", State: SeatDefinitionsState, Words: "the seat definitions the trunk's and each open pull request's contracts modules hold, resolved, or why one does not parse."}, {Tool: "indexer_read_unjudged", State: UnjudgedState, Words: "the manifests a take-in newly wrote that no judge pass judged yet, by digest, each with its module and merge; indexer_judge judges them."}, @@ -44,7 +45,7 @@ type ViewedTool struct { // ViewedStates are the states the indexer's views family says: the ones its readers read. var ViewedStates = map[string]bool{ ManifestsState: true, VerdictsState: true, PredefinedState: true, ConfigCandidatesState: true, CandidatesState: true, - PackagesState: true, HeardState: true, TakenState: true, FindingsState: true, SeatDefinitionsState: true, UnjudgedState: true, + PackagesState: true, HeardState: true, TakenState: true, TakeInsState: true, FindingsState: true, SeatDefinitionsState: true, UnjudgedState: true, } // ModulesViewsEvent is what the indexer says on its views family of modules when a new revision of the trunk branch diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 367e8d78..cc051196 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -206,7 +206,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build index.EntryIndexTool, index.NamedDropTool, // design 47 G §675 index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.ConsumerTool, index.RunsTool, index.RunDropTool, // design 47 G §667, §8, §579, §64 (A-194) index.ValidateAgainTool) // design 47 G §713 - if !reflect.DeepEqual(m.Tools, want) || len(want) != 51 || !slices.Contains(m.Tools, "modules") { + if !reflect.DeepEqual(m.Tools, want) || len(want) != 52 || !slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section. diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index c9102662..6d387c43 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -43,6 +43,7 @@ "indexer_read_graph", "indexer_read_heard", "indexer_read_taken", + "indexer_read_takeins", "indexer_read_findings", "indexer_read_seat_definitions", "indexer_read_unjudged",