diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index b0b2025..4784980 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -84,6 +84,7 @@ type Indexer struct { toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go) tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go) unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread + moving map[string]delivery.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go) retrying atomic.Bool // a RetryUnread runs in the background (kickRetry) retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 7eeae25..b786d1b 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -22,7 +22,8 @@ import ( // 1851), so a module added by the merge gets a value, and a fingerprint, with no build anywhere. The value of today's // form (Module.FP, None) is no longer written: it was computed from those records (ADR 0453 clause 6). A module the // bucket holds of another repository is left as it is and said: a move between repositories is the operator's (ADR 0420 -// clause 7). Two directories naming one module write the one the bucket holds, or the first by name, and say the other. +// clause 7), and kept waiting: once the holding repository's trunk drops the module, the merge that drops it hands it to the +// waiting repository instead of deleting it (moves.go). Two directories naming one module write the one the bucket holds, or the first by name, and say the other. // A value the module already holds but for its commit is not written again: its commit stays the one its fingerprint first // appeared at (writeOverlay), so a merge writes only the modules it changes. // @@ -36,6 +37,12 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) (*mergeWri // deleted. A message that names only some of a repository's modules (a registered module read on its own, registered.go) // drops none. func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bool) (*mergeWrite, error) { + return ix.applyMoving(ctx, f, drop, nil) +} + +// applyMoving is applyManifests; moves names the modules keys the message may take from the repository that holds them +// (a move handed over, moves.go). Every other module held of another repository is left, said and kept waiting. +func (ix *Indexer) applyMoving(ctx context.Context, f delivery.Forge, drop bool, moves map[string]bool) (*mergeWrite, error) { w := newMergeWrite(f) if drop { if err := ix.lateMerge(ctx, w); err != nil || w.late != "" { @@ -71,8 +78,12 @@ func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bo named[key] = true cur, has := held[key] if has && cur.Repository != "" && !delivery.SameRepository(cur.Repository, f.Repository) { - ix.logf("%s at %s: %q names the module %s, which the indexer holds of %s; left as it is (a move between repositories is the operator's)", f.Repository, f.Merge, dir, name, cur.Repository) - continue + if !moves[key] { + ix.waitToMove(key, f, dir) + ix.logf("%s at %s: %q names the module %s, which the indexer holds of %s; left as it is until %s's trunk drops it, and then moved here (a move between repositories is the merge of the group that moves it, ADR 0420 clause 7)", f.Repository, f.Merge, dir, name, cur.Repository, cur.Repository) + continue + } + ix.logf("%s at %s: the module %s moves here from %s, whose trunk no longer holds it", f.Repository, f.Merge, name, cur.Repository) } if other, dup := dirOf[name]; dup { if !has || cur.Path != pathOf(dir) { @@ -190,11 +201,16 @@ func (ix *Indexer) namedObjects(ctx context.Context, f delivery.Forge) (map[stri // dropRemoved deletes, from the modules the bucket holds of the repository, each one the trunk no longer holds (issue 1900): // held(key, module) says whether the trunk holds it. A module of another repository, or of none known, is never touched. func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[string]delivery.Module, trunkHolds func(key string, m delivery.Module) bool) error { + var handed []string for _, key := range slices.Sorted(maps.Keys(held)) { cur := held[key] if cur.Repository == "" || !delivery.SameRepository(cur.Repository, repository) || trunkHolds(key, cur) { continue } + if ix.movingTo(key, repository) { + handed = append(handed, key) // written in place by the repository it moves to, never deleted first + continue + } err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) { if !found { return nil, false, nil @@ -209,6 +225,11 @@ func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[ } ix.logf("modules.%s deleted: %s holds no module of that name at its trunk (issue 1900)", key, repository) } + for _, key := range handed { + if err := ix.handOver(ctx, key); err != nil { + return err + } + } return nil } diff --git a/modules/mesh-delivery-indexer/internal/index/modules_test.go b/modules/mesh-delivery-indexer/internal/index/modules_test.go index d090f08..b75b5a8 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules_test.go +++ b/modules/mesh-delivery-indexer/internal/index/modules_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "path" + "slices" "strings" "testing" "time" @@ -402,20 +403,64 @@ func TestAMergeWritesOnlyTheModulesItsManifestsName(t *testing.T) { // I1 } } -func TestAModuleHeldOfAnotherRepositoryIsLeftAndSaid(t *testing.T) { // I1: a move between repositories is the operator's (ADR 0420 clause 7) - ix, mods := modulesIndexer(t) - var logs []string - ix.Logf = func(format string, args ...any) { logs = append(logs, format) } - ctx := context.Background() - if err := ix.Apply(ctx, mergedMsg("novox/mesh-catalog", map[string]string{"modules/a": "t"})); err != nil { - t.Fatal(err) - } - before := string(mods.vals["a"]) - if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t2"})); err != nil { - t.Fatal(err) - } - if string(mods.vals["a"]) != before || len(logs) == 0 { - t.Errorf("modules.a = %s (was %s), logs %v: want it left, and said", mods.vals["a"], before, logs) +// deletesSeen is a modules store that remembers each key deleted. +type deletesSeen struct { + *memStore + deleted []string +} + +func (d *deletesSeen) Delete(ctx context.Context, key string, rev uint64) error { + d.deleted = append(d.deleted, key) + return d.memStore.Delete(ctx, key, rev) +} + +// I1: a move between repositories is the merges of the group that moves it (ADR 0420 clause 7; issue 1126, route-proxy +// leaving the controller's repository). A merge naming a module another repository's trunk still holds leaves it, said; the +// holding repository's merge that drops it hands it over in place, never deleting its key; merged the other way round, +// the module is deleted and then written new. +func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) { + const from, to = "novox/mesh-controller", "novox/mesh-networking" + held := mergedMsg(from, map[string]string{"modules/a": "t", "modules/b": "t"}) + named := mergedMsg(to, map[string]string{"modules/a": "t2"}) + dropped := mergedMsg(from, map[string]string{"modules/b": "t"}) + for _, tc := range []struct { + name string + merges []delivery.Forge + repository string // modules.a's repository at the end, "" for none + deletedKeyA bool + }{ + {"named while its trunk holds it: left", []delivery.Forge{held, named}, from, false}, + {"named, then dropped by its trunk: handed over in place", []delivery.Forge{held, named, dropped}, to, false}, + {"dropped by its trunk, then named: deleted, then written new", []delivery.Forge{held, dropped, named}, to, true}, + {"dropped by its trunk with no merge naming it: deleted", []delivery.Forge{held, dropped}, "", true}, + } { + t.Run(tc.name, func(t *testing.T) { + mods := &deletesSeen{memStore: newMem()} + ix := &Indexer{Store: newMem(), Modules: mods, Builds: NewBuilds()} + var logs []string + ix.Logf = func(format string, args ...any) { logs = append(logs, format) } + for _, f := range tc.merges { + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + } + m, ok := moduleOf(t, mods.memStore, "a") + if got := map[bool]string{true: m.Repository}[ok]; got != tc.repository { + t.Errorf("modules.a of %q, want %q (logs %v)", got, tc.repository, logs) + } + if ok && m.Path != "modules/a" { + t.Errorf("modules.a at %q, want modules/a", m.Path) + } + if deleted := slices.Contains(mods.deleted, "a"); deleted != tc.deletedKeyA { + t.Errorf("modules.a deleted %v, want %v", deleted, tc.deletedKeyA) + } + if _, ok := mods.vals["b"]; !ok { + t.Error("modules.b, which its trunk still holds, is gone") + } + if len(logs) == 0 { + t.Error("nothing was said") + } + }) } } diff --git a/modules/mesh-delivery-indexer/internal/index/moves.go b/modules/mesh-delivery-indexer/internal/index/moves.go new file mode 100644 index 0000000..8567d4d --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/moves.go @@ -0,0 +1,69 @@ +package index + +import ( + "context" + "fmt" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// A module moves between repositories by the merges of the delivery group that moves it (novox/hq ADR 0420 clause 7, ADR +// 0501 clause 3; issue 1126: route-proxy leaves the controller's repository, which is archived). The merge of the +// repository it moves to names it while the indexer holds it of another: that merge leaves it, says so, and waits here. +// The merge of the repository that held it, whose trunk no longer holds it, then hands it over: the module is written from +// the waiting merge, in place, so its key never goes missing between the two (dropRemoved). Merged the other way round, +// the first merge deletes the key and the second writes it as a new module. Nothing else moves a module: a merge of a +// repository naming a module whose holding trunk still holds it changes nothing. +// +// The wait is the indexer's memory of a merge it heard, not state of its own: an indexer started between the two merges +// has forgotten it, and the module is deleted by the second; the next merge of the repository it moved to writes it. + +// waitToMove keeps, by modules key, the part of a merged message naming one module directory the indexer holds of another +// repository. A later merge of the same repository replaces it. +func (ix *Indexer) waitToMove(key string, f delivery.Forge, dir string) { + one := delivery.Forge{V: f.V, Kind: f.Kind, Repository: f.Repository, Branch: f.Branch, Number: f.Number, Merge: f.Merge, Root: f.Root, + Trees: map[string]string{dir: f.Trees[dir]}, Manifests: map[string]string{dir: f.Manifests[dir]}} + if l := f.Locks[dir]; len(l) > 0 { + one.Locks = map[string]map[string]string{dir: l} + } + if p := f.Paths[dir]; len(p) > 0 { + one.Paths = map[string]map[string]string{dir: p} + } + if p := f.Packages[dir]; len(p) > 0 { + one.Packages = map[string][]string{dir: p} + } + ix.mu.Lock() + defer ix.mu.Unlock() + if ix.moving == nil { + ix.moving = map[string]delivery.Forge{} + } + ix.moving[key] = one +} + +// movingTo says a merge of another repository than from waits to take the module at key. +func (ix *Indexer) movingTo(key, from string) bool { + ix.mu.Lock() + defer ix.mu.Unlock() + f, ok := ix.moving[key] + return ok && !delivery.SameRepository(f.Repository, from) +} + +// handOver writes the module at key from the merge waiting to take it, and forgets the wait. +func (ix *Indexer) handOver(ctx context.Context, key string) error { + ix.mu.Lock() + f, ok := ix.moving[key] + delete(ix.moving, key) + ix.mu.Unlock() + if !ok { + return nil + } + w, err := ix.applyMoving(ctx, f, false, map[string]bool{key: true}) + if err != nil { + return fmt.Errorf("modules.%s, moving to %s: %w", key, f.Repository, err) + } + // A module moved in is judged as a merge's are (ADR 0578 §1, ADR 0579 §1). + if _, err := ix.JudgeAll(ctx, w.fresh); err != nil { + ix.logf("judge: the manifests are not judged after moving %s to %s: %v", key, f.Repository, err) + } + return nil +}