From 24cbc8961ef3fc1d380dfe504686d7b88171fb04 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 23 Oct 2026 14:06:42 +0200 Subject: [PATCH] Fetch a repository and its siblings through a bare mirror in the cache, updated under a lock, so a check transfers only new objects (issue 890, ADR 0536 s2) --- .../mesh-build-go/cmd/mesh-build-go/fetch.go | 2 +- .../mesh-build-go/cmd/mesh-build-go/mirror.go | 59 ++++++++ .../cmd/mesh-build-go/mirror_test.go | 134 ++++++++++++++++++ .../cmd/mesh-build-go/preview.go | 2 +- modules/mesh-build-go/go.mod | 2 +- modules/mesh-build-go/go.sum | 2 + 6 files changed, 198 insertions(+), 3 deletions(-) create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/mirror.go create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/mirror_test.go diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index 435def4d..b8392a58 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -114,7 +114,7 @@ func (m *Module) fetchInto(ctx context.Context, req buildstep.FetchRequest, id s return nil, err } remote := m.remote(repository) - if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref, m.FetchOptions...); err != nil { + if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref, m.fetchOptions(ctx, repository, commit, ref)...); err != nil { return nil, err } res := FetchResult{Workspace: id, Commit: commit} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/mirror.go b/modules/mesh-build-go/cmd/mesh-build-go/mirror.go new file mode 100644 index 00000000..555e58c6 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/mirror.go @@ -0,0 +1,59 @@ +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + "syscall" + + "git.novox.be/novox/mesh-sdk/go/buildstep" +) + +// mirrorLocks hold one mutex per mirror of this process; the lock file beside the mirror holds off another process. +var mirrorLocks sync.Map + +// mirrorDir is where the bare mirror of a repository lies: one per repository in the module's cache (ADR 0536 §2). +func (m *Module) mirrorDir(repository string) string { + return filepath.Join(m.Cache, "mirror", filepath.FromSlash(repository)+".git") +} + +// fetchOptions are the options a fetch of repository runs with: the module's own, and the mirror of the repository +// brought to the commit, so git transfers only the objects the mirror lacks. A mirror that cannot be brought to the +// commit is said and left out: the fetch is then the full one, as it was. +func (m *Module) fetchOptions(ctx context.Context, repository, commit, ref string) []buildstep.Option { + opts := m.FetchOptions + if m.Cache == "" || !repositoryOK.MatchString(repository) { + return opts + } + mirror := m.mirrorDir(repository) + if err := m.updateMirror(ctx, mirror, m.remote(repository), commit, ref); err != nil { + if m.Say != nil { + m.Say("the mirror of %s is not used: %v", repository, err) + } + return opts + } + return append(append([]buildstep.Option{}, opts...), buildstep.WithMirror(mirror)) +} + +// updateMirror brings the mirror to the commit under the mirror's lock: a mutex for the checks of this process, a lock +// file for another process of the node. +func (m *Module) updateMirror(ctx context.Context, mirror, remote, commit, ref string) error { + mu, _ := mirrorLocks.LoadOrStore(mirror, &sync.Mutex{}) + mu.(*sync.Mutex).Lock() + defer mu.(*sync.Mutex).Unlock() + if err := os.MkdirAll(mirror, 0o700); err != nil { // git init --bare takes an empty directory + return err + } + lock, err := os.OpenFile(mirror+".lock", os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return err + } + defer lock.Close() + if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil { + return fmt.Errorf("lock %s: %w", mirror, err) + } + defer syscall.Flock(int(lock.Fd()), syscall.LOCK_UN) + return buildstep.UpdateMirror(ctx, m.Run, mirror, remote, commit, ref, m.FetchOptions...) +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/mirror_test.go b/modules/mesh-build-go/cmd/mesh-build-go/mirror_test.go new file mode 100644 index 00000000..dabf67eb --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/mirror_test.go @@ -0,0 +1,134 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +func fetchOnce(t *testing.T, m *Module, states *keptStates, args map[string]any) FetchResult { + t.Helper() + if _, err := fetchTool(t, m)(args); err != nil { + t.Fatal(err) + } + st := states.waitEnd(t) + if st.State != delivery.StepPassed { + t.Fatalf("the step ended %s: %s", st.State, st.Reason) + } + var res FetchResult + if err := json.Unmarshal(st.Result, &res); err != nil { + t.Fatal(err) + } + return res +} + +// Issue 890/AC4 (ADR 0536 §2): two fetches of one repository on one module bring the same mirror to their commit, and +// each workspace's alternates file names the mirror's objects. +func TestTwoFetchesOfARepositoryUseOneMirrorAndReferenceIt(t *testing.T) { + f := &fakeStep{gomods: "go.mod\n"} + m, states := fetchModule(t, f) + mirror := filepath.Join(m.Cache, "mirror", "novox", "mesh-sdk.git") + var workspaces []string + for i := 0; i < 2; i++ { + workspaces = append(workspaces, fetchOnce(t, m, states, fetchArgs).Workspace) + } + updates := 0 + for _, l := range f.lines() { + if strings.Contains(l, "-C "+mirror+" fetch --no-tags origin "+commitSHA) { + updates++ + } + if strings.Contains(l, "mirror") && !strings.Contains(l, mirror) { + t.Errorf("a command names another mirror: %s", l) + } + } + if updates != 2 { + t.Errorf("the mirror %s was updated %d times, want 2:\n%s", mirror, updates, strings.Join(f.lines(), "\n")) + } + for _, ws := range workspaces { + raw, err := os.ReadFile(filepath.Join(m.Work, ws, "tree", ".git", "objects", "info", "alternates")) + if err != nil || strings.TrimSpace(string(raw)) != filepath.Join(mirror, "objects") { + t.Errorf("the alternates of %s are %q (%v)", ws, raw, err) + } + } +} + +// A sibling is placed through its own repository's mirror. +func TestASiblingIsPlacedThroughItsOwnMirror(t *testing.T) { + f := &fakeStep{gomods: "go.mod\n"} + m, states := fetchModule(t, f) + args := map[string]any{"repository": "novox/mesh-controller", "commit": commitSHA, "beside": []any{ + map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA}}} + ws := fetchOnce(t, m, states, args).Workspace + for _, repo := range []string{"mesh-controller", "mesh-sdk"} { + mirror := filepath.Join(m.Cache, "mirror", "novox", repo+".git") + if !strings.Contains(strings.Join(f.lines(), "\n"), "-C "+mirror+" fetch --no-tags origin "+commitSHA) { + t.Errorf("the mirror of %s was not updated", repo) + } + } + raw, _ := os.ReadFile(filepath.Join(m.Work, ws, "beside", "mesh-sdk", ".git", "objects", "info", "alternates")) + if strings.TrimSpace(string(raw)) != filepath.Join(m.Cache, "mirror", "novox", "mesh-sdk.git", "objects") { + t.Errorf("the sibling's alternates are %q", raw) + } +} + +// Two checks at once on one node update one mirror one after the other. +func TestTwoUpdatesOfOneMirrorNeverRunAtOnce(t *testing.T) { + var mu sync.Mutex + inside, most := 0, 0 + f := &fakeStep{gomods: "go.mod\n"} + f.rule = func(line string) (r runAnswer, ok bool) { + if !strings.Contains(line, "fetch --no-tags origin") || strings.Contains(line, "--depth") { + return r, false + } + mu.Lock() + inside++ + if inside > most { + most = inside + } + mu.Unlock() + sleepABit() + mu.Lock() + inside-- + mu.Unlock() + return r, true + } + m, states := fetchModule(t, f) + var wg sync.WaitGroup + for i := 0; i < 2; i++ { + wg.Add(1) + go func() { + defer wg.Done() + fetchTool(t, m)(fetchArgs) + }() + } + wg.Wait() + states.waitEnd(t) + states.waitEnd(t) + if most != 1 { + t.Errorf("%d updates of the mirror ran at once", most) + } +} + +// A mirror that cannot be updated does not fail the fetch: the fetch goes on without it. +func TestAMirrorThatCannotBeUpdatedLetsTheFetchGoOnWithoutIt(t *testing.T) { + f := &fakeStep{gomods: "go.mod\n"} + f.rule = func(line string) (r runAnswer, ok bool) { + if strings.Contains(line, "mirror") && strings.Contains(line, " fetch ") { + return runAnswer{Status: 1, Stderr: "fatal: no space left\n"}, true + } + return r, false + } + m, states := fetchModule(t, f) + ws := fetchOnce(t, m, states, fetchArgs).Workspace + if _, err := os.Stat(filepath.Join(m.Work, ws, "tree", ".git", "objects", "info", "alternates")); err == nil { + t.Errorf("the workspace borrows from a mirror that is not there") + } +} + +func sleepABit() { time.Sleep(30 * time.Millisecond) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/preview.go b/modules/mesh-build-go/cmd/mesh-build-go/preview.go index 87ae1066..192b8cbc 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/preview.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/preview.go @@ -96,7 +96,7 @@ func (m *Module) placeSiblings(ctx context.Context, root string, siblings []sibl if err := os.MkdirAll(dir, 0o700); err != nil { return nil, err } - if err := buildstep.FetchCommit(ctx, m.Run, dir, m.remote(s.repository), s.commit, s.ref, m.FetchOptions...); err != nil { + if err := buildstep.FetchCommit(ctx, m.Run, dir, m.remote(s.repository), s.commit, s.ref, m.fetchOptions(ctx, s.repository, s.commit, s.ref)...); err != nil { return nil, fmt.Errorf("beside %s: %w", s.repository, err) } if link := firstSymlink(dir); link != "" { diff --git a/modules/mesh-build-go/go.mod b/modules/mesh-build-go/go.mod index 999b31b9..9973c007 100644 --- a/modules/mesh-build-go/go.mod +++ b/modules/mesh-build-go/go.mod @@ -3,7 +3,7 @@ module meshbuildgo go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023120306-780be2fae5a1 github.com/distribution/distribution/v3 v3.1.2 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/modules/mesh-build-go/go.sum b/modules/mesh-build-go/go.sum index ccb75980..e5951bbf 100644 --- a/modules/mesh-build-go/go.sum +++ b/modules/mesh-build-go/go.sum @@ -2,6 +2,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYE git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467 h1:i85+fFT0wZiTZ9CRVfMzWg1+Zr7oOLl3UH7hEaKcOIc= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023120306-780be2fae5a1 h1:MqP6mWWVxM+niUGPxSSh4Q8lduYkkQ7oTAkDeC9nyv4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023120306-780be2fae5a1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI= github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=