Let a module move between repositories by the merges that move it (issue 1126, route-proxy)

route-proxy leaves the controller's repository, which is archived, and nothing registered a move since the controller
went: the indexer left a module named by another repository's merge as it was, forever. Now that merge waits, and the
holding repository's merge that drops the module hands it over in place, so its key never goes missing between the two;
merged the other way round, the key is deleted and written new. A trunk that still holds the module keeps it (ADR 0420
clause 7, ADR 0501 clause 3).

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL

modules/mesh-delivery-indexer/internal/index/candidates.go

@@ -84,6 +84,7 @@ type Indexer struct {
 	toolchains   map[string]map[string]delivery.Toolchain // language, then node (toolchains.go)
 	tried        map[string]bool                          // repository, commit and directory a registered module's manifest was tried at (registered.go)
 	unread       map[string]bool                          // registered modules whose last read the forge could not answer, read again by RetryUnread
+	moving       map[string]delivery.Forge                // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
 	retrying     atomic.Bool                              // a RetryUnread runs in the background (kickRetry)
 	retryAgain   atomic.Bool                              // an event asked for one more RetryUnread after the running one
 	retryWG      sync.WaitGroup                           // the background RetryUnread, for tests to wait on

modules/mesh-delivery-indexer/internal/index/modules.go

@@ -22,7 +22,8 @@ import (
 // 1851), so a module added by the merge gets a value, and a fingerprint, with no build anywhere. The value of today's
 // form (Module.FP, None) is no longer written: it was computed from those records (ADR 0453 clause 6). A module the
 // bucket holds of another repository is left as it is and said: a move between repositories is the operator's (ADR 0420
-// clause 7). Two directories naming one module write the one the bucket holds, or the first by name, and say the other.
+// clause 7), and kept waiting: once the holding repository's trunk drops the module, the merge that drops it hands it to the
+// waiting repository instead of deleting it (moves.go). Two directories naming one module write the one the bucket holds, or the first by name, and say the other.
 // A value the module already holds but for its commit is not written again: its commit stays the one its fingerprint first
 // appeared at (writeOverlay), so a merge writes only the modules it changes.
 //
@@ -36,6 +37,12 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) (*mergeWri
 // deleted. A message that names only some of a repository's modules (a registered module read on its own, registered.go)
 // drops none.
 func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bool) (*mergeWrite, error) {
+	return ix.applyMoving(ctx, f, drop, nil)
+}
+
+// applyMoving is applyManifests; moves names the modules keys the message may take from the repository that holds them
+// (a move handed over, moves.go). Every other module held of another repository is left, said and kept waiting.
+func (ix *Indexer) applyMoving(ctx context.Context, f delivery.Forge, drop bool, moves map[string]bool) (*mergeWrite, error) {
 	w := newMergeWrite(f)
 	if drop {
 		if err := ix.lateMerge(ctx, w); err != nil || w.late != "" {
@@ -71,8 +78,12 @@ func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bo
 		named[key] = true
 		cur, has := held[key]
 		if has && cur.Repository != "" && !delivery.SameRepository(cur.Repository, f.Repository) {
-			ix.logf("%s at %s: %q names the module %s, which the indexer holds of %s; left as it is (a move between repositories is the operator's)", f.Repository, f.Merge, dir, name, cur.Repository)
-			continue
+			if !moves[key] {
+				ix.waitToMove(key, f, dir)
+				ix.logf("%s at %s: %q names the module %s, which the indexer holds of %s; left as it is until %s's trunk drops it, and then moved here (a move between repositories is the merge of the group that moves it, ADR 0420 clause 7)", f.Repository, f.Merge, dir, name, cur.Repository, cur.Repository)
+				continue
+			}
+			ix.logf("%s at %s: the module %s moves here from %s, whose trunk no longer holds it", f.Repository, f.Merge, name, cur.Repository)
 		}
 		if other, dup := dirOf[name]; dup {
 			if !has || cur.Path != pathOf(dir) {
@@ -190,11 +201,16 @@ func (ix *Indexer) namedObjects(ctx context.Context, f delivery.Forge) (map[stri
 // dropRemoved deletes, from the modules the bucket holds of the repository, each one the trunk no longer holds (issue 1900):
 // held(key, module) says whether the trunk holds it. A module of another repository, or of none known, is never touched.
 func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[string]delivery.Module, trunkHolds func(key string, m delivery.Module) bool) error {
+	var handed []string
 	for _, key := range slices.Sorted(maps.Keys(held)) {
 		cur := held[key]
 		if cur.Repository == "" || !delivery.SameRepository(cur.Repository, repository) || trunkHolds(key, cur) {
 			continue
 		}
+		if ix.movingTo(key, repository) {
+			handed = append(handed, key) // written in place by the repository it moves to, never deleted first
+			continue
+		}
 		err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
 			if !found {
 				return nil, false, nil
@@ -209,6 +225,11 @@ func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[
 		}
 		ix.logf("modules.%s deleted: %s holds no module of that name at its trunk (issue 1900)", key, repository)
 	}
+	for _, key := range handed {
+		if err := ix.handOver(ctx, key); err != nil {
+			return err
+		}
+	}
 	return nil
 }
 

modules/mesh-delivery-indexer/internal/index/modules_test.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"path"
+	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -402,20 +403,64 @@ func TestAMergeWritesOnlyTheModulesItsManifestsName(t *testing.T) { // I1
 	}
 }
 
-func TestAModuleHeldOfAnotherRepositoryIsLeftAndSaid(t *testing.T) { // I1: a move between repositories is the operator's (ADR 0420 clause 7)
-	ix, mods := modulesIndexer(t)
-	var logs []string
-	ix.Logf = func(format string, args ...any) { logs = append(logs, format) }
-	ctx := context.Background()
-	if err := ix.Apply(ctx, mergedMsg("novox/mesh-catalog", map[string]string{"modules/a": "t"})); err != nil {
-		t.Fatal(err)
-	}
-	before := string(mods.vals["a"])
-	if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t2"})); err != nil {
-		t.Fatal(err)
-	}
-	if string(mods.vals["a"]) != before || len(logs) == 0 {
-		t.Errorf("modules.a = %s (was %s), logs %v: want it left, and said", mods.vals["a"], before, logs)
+// deletesSeen is a modules store that remembers each key deleted.
+type deletesSeen struct {
+	*memStore
+	deleted []string
+}
+
+func (d *deletesSeen) Delete(ctx context.Context, key string, rev uint64) error {
+	d.deleted = append(d.deleted, key)
+	return d.memStore.Delete(ctx, key, rev)
+}
+
+// I1: a move between repositories is the merges of the group that moves it (ADR 0420 clause 7; issue 1126, route-proxy
+// leaving the controller's repository). A merge naming a module another repository's trunk still holds leaves it, said; the
+// holding repository's merge that drops it hands it over in place, never deleting its key; merged the other way round,
+// the module is deleted and then written new.
+func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) {
+	const from, to = "novox/mesh-controller", "novox/mesh-networking"
+	held := mergedMsg(from, map[string]string{"modules/a": "t", "modules/b": "t"})
+	named := mergedMsg(to, map[string]string{"modules/a": "t2"})
+	dropped := mergedMsg(from, map[string]string{"modules/b": "t"})
+	for _, tc := range []struct {
+		name        string
+		merges      []delivery.Forge
+		repository  string // modules.a's repository at the end, "" for none
+		deletedKeyA bool
+	}{
+		{"named while its trunk holds it: left", []delivery.Forge{held, named}, from, false},
+		{"named, then dropped by its trunk: handed over in place", []delivery.Forge{held, named, dropped}, to, false},
+		{"dropped by its trunk, then named: deleted, then written new", []delivery.Forge{held, dropped, named}, to, true},
+		{"dropped by its trunk with no merge naming it: deleted", []delivery.Forge{held, dropped}, "", true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			mods := &deletesSeen{memStore: newMem()}
+			ix := &Indexer{Store: newMem(), Modules: mods, Builds: NewBuilds()}
+			var logs []string
+			ix.Logf = func(format string, args ...any) { logs = append(logs, format) }
+			for _, f := range tc.merges {
+				if err := ix.Apply(context.Background(), f); err != nil {
+					t.Fatal(err)
+				}
+			}
+			m, ok := moduleOf(t, mods.memStore, "a")
+			if got := map[bool]string{true: m.Repository}[ok]; got != tc.repository {
+				t.Errorf("modules.a of %q, want %q (logs %v)", got, tc.repository, logs)
+			}
+			if ok && m.Path != "modules/a" {
+				t.Errorf("modules.a at %q, want modules/a", m.Path)
+			}
+			if deleted := slices.Contains(mods.deleted, "a"); deleted != tc.deletedKeyA {
+				t.Errorf("modules.a deleted %v, want %v", deleted, tc.deletedKeyA)
+			}
+			if _, ok := mods.vals["b"]; !ok {
+				t.Error("modules.b, which its trunk still holds, is gone")
+			}
+			if len(logs) == 0 {
+				t.Error("nothing was said")
+			}
+		})
 	}
 }
 

modules/mesh-delivery-indexer/internal/index/moves.go

@@ -0,0 +1,69 @@
+package index
+
+import (
+	"context"
+	"fmt"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+// A module moves between repositories by the merges of the delivery group that moves it (novox/hq ADR 0420 clause 7, ADR
+// 0501 clause 3; issue 1126: route-proxy leaves the controller's repository, which is archived). The merge of the
+// repository it moves to names it while the indexer holds it of another: that merge leaves it, says so, and waits here.
+// The merge of the repository that held it, whose trunk no longer holds it, then hands it over: the module is written from
+// the waiting merge, in place, so its key never goes missing between the two (dropRemoved). Merged the other way round,
+// the first merge deletes the key and the second writes it as a new module. Nothing else moves a module: a merge of a
+// repository naming a module whose holding trunk still holds it changes nothing.
+//
+// The wait is the indexer's memory of a merge it heard, not state of its own: an indexer started between the two merges
+// has forgotten it, and the module is deleted by the second; the next merge of the repository it moved to writes it.
+
+// waitToMove keeps, by modules key, the part of a merged message naming one module directory the indexer holds of another
+// repository. A later merge of the same repository replaces it.
+func (ix *Indexer) waitToMove(key string, f delivery.Forge, dir string) {
+	one := delivery.Forge{V: f.V, Kind: f.Kind, Repository: f.Repository, Branch: f.Branch, Number: f.Number, Merge: f.Merge, Root: f.Root,
+		Trees: map[string]string{dir: f.Trees[dir]}, Manifests: map[string]string{dir: f.Manifests[dir]}}
+	if l := f.Locks[dir]; len(l) > 0 {
+		one.Locks = map[string]map[string]string{dir: l}
+	}
+	if p := f.Paths[dir]; len(p) > 0 {
+		one.Paths = map[string]map[string]string{dir: p}
+	}
+	if p := f.Packages[dir]; len(p) > 0 {
+		one.Packages = map[string][]string{dir: p}
+	}
+	ix.mu.Lock()
+	defer ix.mu.Unlock()
+	if ix.moving == nil {
+		ix.moving = map[string]delivery.Forge{}
+	}
+	ix.moving[key] = one
+}
+
+// movingTo says a merge of another repository than from waits to take the module at key.
+func (ix *Indexer) movingTo(key, from string) bool {
+	ix.mu.Lock()
+	defer ix.mu.Unlock()
+	f, ok := ix.moving[key]
+	return ok && !delivery.SameRepository(f.Repository, from)
+}
+
+// handOver writes the module at key from the merge waiting to take it, and forgets the wait.
+func (ix *Indexer) handOver(ctx context.Context, key string) error {
+	ix.mu.Lock()
+	f, ok := ix.moving[key]
+	delete(ix.moving, key)
+	ix.mu.Unlock()
+	if !ok {
+		return nil
+	}
+	w, err := ix.applyMoving(ctx, f, false, map[string]bool{key: true})
+	if err != nil {
+		return fmt.Errorf("modules.%s, moving to %s: %w", key, f.Repository, err)
+	}
+	// A module moved in is judged as a merge's are (ADR 0578 §1, ADR 0579 §1).
+	if _, err := ix.JudgeAll(ctx, w.fresh); err != nil {
+		ix.logf("judge: the manifests are not judged after moving %s to %s: %v", key, f.Repository, err)
+	}
+	return nil
+}