Fetch a repository and its siblings through a bare mirror in the cache, updated under a lock, so a check transfers only new objects (issue 890, ADR 0536 s2)

2026-10-23 14:06:42 +02:00
parent 547f4ee27d
commit 24cbc8961e
6 changed files with 198 additions and 3 deletions
@@ -114,7 +114,7 @@ func (m *Module) fetchInto(ctx context.Context, req buildstep.FetchRequest, id s
 		return nil, err
 	}
 	remote := m.remote(repository)
-	if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref, m.FetchOptions...); err != nil {
+	if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref, m.fetchOptions(ctx, repository, commit, ref)...); err != nil {
 		return nil, err
 	}
 	res := FetchResult{Workspace: id, Commit: commit}
@@ -0,0 +1,59 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+	"syscall"
+
+	"git.novox.be/novox/mesh-sdk/go/buildstep"
+)
+
+// mirrorLocks hold one mutex per mirror of this process; the lock file beside the mirror holds off another process.
+var mirrorLocks sync.Map
+
+// mirrorDir is where the bare mirror of a repository lies: one per repository in the module's cache (ADR 0536 §2).
+func (m *Module) mirrorDir(repository string) string {
+	return filepath.Join(m.Cache, "mirror", filepath.FromSlash(repository)+".git")
+}
+
+// fetchOptions are the options a fetch of repository runs with: the module's own, and the mirror of the repository
+// brought to the commit, so git transfers only the objects the mirror lacks. A mirror that cannot be brought to the
+// commit is said and left out: the fetch is then the full one, as it was.
+func (m *Module) fetchOptions(ctx context.Context, repository, commit, ref string) []buildstep.Option {
+	opts := m.FetchOptions
+	if m.Cache == "" || !repositoryOK.MatchString(repository) {
+		return opts
+	}
+	mirror := m.mirrorDir(repository)
+	if err := m.updateMirror(ctx, mirror, m.remote(repository), commit, ref); err != nil {
+		if m.Say != nil {
+			m.Say("the mirror of %s is not used: %v", repository, err)
+		}
+		return opts
+	}
+	return append(append([]buildstep.Option{}, opts...), buildstep.WithMirror(mirror))
+}
+
+// updateMirror brings the mirror to the commit under the mirror's lock: a mutex for the checks of this process, a lock
+// file for another process of the node.
+func (m *Module) updateMirror(ctx context.Context, mirror, remote, commit, ref string) error {
+	mu, _ := mirrorLocks.LoadOrStore(mirror, &sync.Mutex{})
+	mu.(*sync.Mutex).Lock()
+	defer mu.(*sync.Mutex).Unlock()
+	if err := os.MkdirAll(mirror, 0o700); err != nil { // git init --bare takes an empty directory
+		return err
+	}
+	lock, err := os.OpenFile(mirror+".lock", os.O_CREATE|os.O_RDWR, 0o600)
+	if err != nil {
+		return err
+	}
+	defer lock.Close()
+	if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil {
+		return fmt.Errorf("lock %s: %w", mirror, err)
+	}
+	defer syscall.Flock(int(lock.Fd()), syscall.LOCK_UN)
+	return buildstep.UpdateMirror(ctx, m.Run, mirror, remote, commit, ref, m.FetchOptions...)
+}
@@ -0,0 +1,134 @@
+package main
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+func fetchOnce(t *testing.T, m *Module, states *keptStates, args map[string]any) FetchResult {
+	t.Helper()
+	if _, err := fetchTool(t, m)(args); err != nil {
+		t.Fatal(err)
+	}
+	st := states.waitEnd(t)
+	if st.State != delivery.StepPassed {
+		t.Fatalf("the step ended %s: %s", st.State, st.Reason)
+	}
+	var res FetchResult
+	if err := json.Unmarshal(st.Result, &res); err != nil {
+		t.Fatal(err)
+	}
+	return res
+}
+
+// Issue 890/AC4 (ADR 0536 §2): two fetches of one repository on one module bring the same mirror to their commit, and
+// each workspace's alternates file names the mirror's objects.
+func TestTwoFetchesOfARepositoryUseOneMirrorAndReferenceIt(t *testing.T) {
+	f := &fakeStep{gomods: "go.mod\n"}
+	m, states := fetchModule(t, f)
+	mirror := filepath.Join(m.Cache, "mirror", "novox", "mesh-sdk.git")
+	var workspaces []string
+	for i := 0; i < 2; i++ {
+		workspaces = append(workspaces, fetchOnce(t, m, states, fetchArgs).Workspace)
+	}
+	updates := 0
+	for _, l := range f.lines() {
+		if strings.Contains(l, "-C "+mirror+" fetch --no-tags origin "+commitSHA) {
+			updates++
+		}
+		if strings.Contains(l, "mirror") && !strings.Contains(l, mirror) {
+			t.Errorf("a command names another mirror: %s", l)
+		}
+	}
+	if updates != 2 {
+		t.Errorf("the mirror %s was updated %d times, want 2:\n%s", mirror, updates, strings.Join(f.lines(), "\n"))
+	}
+	for _, ws := range workspaces {
+		raw, err := os.ReadFile(filepath.Join(m.Work, ws, "tree", ".git", "objects", "info", "alternates"))
+		if err != nil || strings.TrimSpace(string(raw)) != filepath.Join(mirror, "objects") {
+			t.Errorf("the alternates of %s are %q (%v)", ws, raw, err)
+		}
+	}
+}
+
+// A sibling is placed through its own repository's mirror.
+func TestASiblingIsPlacedThroughItsOwnMirror(t *testing.T) {
+	f := &fakeStep{gomods: "go.mod\n"}
+	m, states := fetchModule(t, f)
+	args := map[string]any{"repository": "novox/mesh-controller", "commit": commitSHA, "beside": []any{
+		map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA}}}
+	ws := fetchOnce(t, m, states, args).Workspace
+	for _, repo := range []string{"mesh-controller", "mesh-sdk"} {
+		mirror := filepath.Join(m.Cache, "mirror", "novox", repo+".git")
+		if !strings.Contains(strings.Join(f.lines(), "\n"), "-C "+mirror+" fetch --no-tags origin "+commitSHA) {
+			t.Errorf("the mirror of %s was not updated", repo)
+		}
+	}
+	raw, _ := os.ReadFile(filepath.Join(m.Work, ws, "beside", "mesh-sdk", ".git", "objects", "info", "alternates"))
+	if strings.TrimSpace(string(raw)) != filepath.Join(m.Cache, "mirror", "novox", "mesh-sdk.git", "objects") {
+		t.Errorf("the sibling's alternates are %q", raw)
+	}
+}
+
+// Two checks at once on one node update one mirror one after the other.
+func TestTwoUpdatesOfOneMirrorNeverRunAtOnce(t *testing.T) {
+	var mu sync.Mutex
+	inside, most := 0, 0
+	f := &fakeStep{gomods: "go.mod\n"}
+	f.rule = func(line string) (r runAnswer, ok bool) {
+		if !strings.Contains(line, "fetch --no-tags origin") || strings.Contains(line, "--depth") {
+			return r, false
+		}
+		mu.Lock()
+		inside++
+		if inside > most {
+			most = inside
+		}
+		mu.Unlock()
+		sleepABit()
+		mu.Lock()
+		inside--
+		mu.Unlock()
+		return r, true
+	}
+	m, states := fetchModule(t, f)
+	var wg sync.WaitGroup
+	for i := 0; i < 2; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			fetchTool(t, m)(fetchArgs)
+		}()
+	}
+	wg.Wait()
+	states.waitEnd(t)
+	states.waitEnd(t)
+	if most != 1 {
+		t.Errorf("%d updates of the mirror ran at once", most)
+	}
+}
+
+// A mirror that cannot be updated does not fail the fetch: the fetch goes on without it.
+func TestAMirrorThatCannotBeUpdatedLetsTheFetchGoOnWithoutIt(t *testing.T) {
+	f := &fakeStep{gomods: "go.mod\n"}
+	f.rule = func(line string) (r runAnswer, ok bool) {
+		if strings.Contains(line, "mirror") && strings.Contains(line, " fetch ") {
+			return runAnswer{Status: 1, Stderr: "fatal: no space left\n"}, true
+		}
+		return r, false
+	}
+	m, states := fetchModule(t, f)
+	ws := fetchOnce(t, m, states, fetchArgs).Workspace
+	if _, err := os.Stat(filepath.Join(m.Work, ws, "tree", ".git", "objects", "info", "alternates")); err == nil {
+		t.Errorf("the workspace borrows from a mirror that is not there")
+	}
+}
+
+func sleepABit() { time.Sleep(30 * time.Millisecond) }
@@ -96,7 +96,7 @@ func (m *Module) placeSiblings(ctx context.Context, root string, siblings []sibl
 		if err := os.MkdirAll(dir, 0o700); err != nil {
 			return nil, err
 		}
-		if err := buildstep.FetchCommit(ctx, m.Run, dir, m.remote(s.repository), s.commit, s.ref, m.FetchOptions...); err != nil {
+		if err := buildstep.FetchCommit(ctx, m.Run, dir, m.remote(s.repository), s.commit, s.ref, m.fetchOptions(ctx, s.repository, s.commit, s.ref)...); err != nil {
 			return nil, fmt.Errorf("beside %s: %w", s.repository, err)
 		}
 		if link := firstSymlink(dir); link != "" {
@@ -3,7 +3,7 @@ module meshbuildgo
 go 1.26.0

 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023120306-780be2fae5a1
 	github.com/distribution/distribution/v3 v3.1.2
 	github.com/nats-io/nats-server/v2 v2.11.17
 	github.com/nats-io/nats.go v1.54.0
@@ -2,6 +2,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYE
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467 h1:i85+fFT0wZiTZ9CRVfMzWg1+Zr7oOLl3UH7hEaKcOIc=
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023120306-780be2fae5a1 h1:MqP6mWWVxM+niUGPxSSh4Q8lduYkkQ7oTAkDeC9nyv4=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023120306-780be2fae5a1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
 github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=