From 780be2fae5a19ac3d339df3a1382051939027df3 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 23 Oct 2026 14:03:06 +0200 Subject: [PATCH 1/2] =?UTF-8?q?Fetch=20through=20a=20bare=20mirror=20with?= =?UTF-8?q?=20git=20alternates,=20so=20a=20check=20transfers=20only=20new?= =?UTF-8?q?=20objects=20(issue=20890,=20ADR=200536=20=C2=A72)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/buildstep/fetch.go | 89 +++++++++++++++++++- go/buildstep/merge.go | 3 + go/buildstep/mirror_integration_test.go | 65 +++++++++++++++ go/buildstep/mirror_test.go | 106 ++++++++++++++++++++++++ 4 files changed, 262 insertions(+), 1 deletion(-) create mode 100644 go/buildstep/mirror_integration_test.go create mode 100644 go/buildstep/mirror_test.go diff --git a/go/buildstep/fetch.go b/go/buildstep/fetch.go index df6a1b0..2715405 100644 --- a/go/buildstep/fetch.go +++ b/go/buildstep/fetch.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "net/url" + "os" + "path/filepath" "regexp" "strings" @@ -13,12 +15,37 @@ import ( // Option changes FetchCommit. type Option func(*fetchOptions) -type fetchOptions struct{ allowFile bool } +type fetchOptions struct { + allowFile bool + mirror string +} // AllowFileRemote lets FetchCommit take a file:// remote. It is for a test's local repository; production // code does not pass it, and FetchCommit then refuses anything but ssh://. func AllowFileRemote() Option { return func(o *fetchOptions) { o.allowFile = true } } +// WithMirror makes FetchCommit (and MergeInto) borrow objects from mirror, a bare repository that UpdateMirror keeps +// in the build module's own cache (novox/hq ADR 0536 §2; issue 890): the workspace's .git/objects/info/alternates +// names the mirror's objects directory, so git transfers only the objects the mirror lacks. It is a plain file, never +// a symlink. mirror must be an absolute path of an existing directory; the workspace never writes to it. +func WithMirror(mirror string) Option { return func(o *fetchOptions) { o.mirror = mirror } } + +// useMirror writes the alternates file of the repository in dir, when a mirror is set. +func useMirror(o fetchOptions, dir string) error { + if o.mirror == "" { + return nil + } + info := filepath.Join(dir, ".git", "objects", "info") + if err := os.MkdirAll(info, 0o755); err != nil { + return fmt.Errorf("buildstep: the mirror cannot be used: %w", err) + } + line := filepath.Join(o.mirror, "objects") + "\n" + if err := os.WriteFile(filepath.Join(info, "alternates"), []byte(line), 0o644); err != nil { + return fmt.Errorf("buildstep: the mirror cannot be used: %w", err) + } + return nil +} + var ( commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`) refOK = regexp.MustCompile(`^refs/[A-Za-z0-9._/-]+$`) @@ -51,6 +78,9 @@ func FetchCommit(ctx context.Context, run forgekey.Runner, dir, remote, commit, if _, err := git("init", "-q"); err != nil { return err } + if err := useMirror(o, dir); err != nil { + return err + } if _, err := git("remote", "add", "origin", remote); err != nil { return err } @@ -78,10 +108,67 @@ func FetchCommit(ctx context.Context, run forgekey.Runner, dir, remote, commit, return nil } +// UpdateMirror brings the bare repository mirror to commit: it makes the mirror when it is absent (git init --bare, +// origin set to remote), then fetches the commit, or the ref when only the ref holds it, with the same rules as +// FetchCommit (no prompt, no credential helper, ssh:// or an allowed file:// remote, a 40-hex commit). The mirror +// keeps every commit it was brought to, so later fetches through WithMirror transfer only what is new. Callers +// serialise updates of one mirror. +func UpdateMirror(ctx context.Context, run forgekey.Runner, mirror, remote, commit, ref string, opts ...Option) error { + var o fetchOptions + for _, opt := range opts { + opt(&o) + } + if !filepath.IsAbs(mirror) { + return fmt.Errorf("buildstep: the mirror %q is not an absolute path", mirror) + } + if err := checkFetch(o, mirror, remote, commit, ref); err != nil { + return err + } + env := []string{"GIT_TERMINAL_PROMPT=0"} + git := func(args ...string) error { + r := run(ctx, env, "git", append([]string{"-c", "credential.helper="}, args...)...) + if r.Status != 0 || r.Err != "" { + return fmt.Errorf("git %s: %s", args[0], firstLine(r.Err+" "+r.Stderr)) + } + return nil + } + in := func(args ...string) error { return git(append([]string{"-C", mirror}, args...)...) } + if _, err := os.Stat(filepath.Join(mirror, "HEAD")); err != nil { + if err := git("init", "-q", "--bare", mirror); err != nil { + return err + } + if err := in("remote", "add", "origin", remote); err != nil { + return err + } + } else if err := in("remote", "set-url", "origin", remote); err != nil { + return err + } + if err := in("fetch", "--no-tags", "origin", commit); err != nil { + if ref == "" { + return err + } + if rerr := in("fetch", "--no-tags", "origin", ref); rerr != nil { + return fmt.Errorf("%v; then the ref %s: %w", err, ref, rerr) + } + } + if err := in("cat-file", "-e", commit+"^{commit}"); err != nil { + return fmt.Errorf("the commit %s is not in the mirror: %w", commit, err) + } + return nil +} + func checkFetch(o fetchOptions, dir, remote, commit, ref string) error { if dir == "" { return fmt.Errorf("buildstep: no workspace directory") } + if o.mirror != "" { + if !filepath.IsAbs(o.mirror) { + return fmt.Errorf("buildstep: the mirror %q is not an absolute path", o.mirror) + } + if fi, err := os.Stat(o.mirror); err != nil || !fi.IsDir() { + return fmt.Errorf("buildstep: the mirror %q is not an existing directory", o.mirror) + } + } if !commitOK.MatchString(commit) { return fmt.Errorf("buildstep: %q is not a full commit (40 hex digits)", commit) } diff --git a/go/buildstep/merge.go b/go/buildstep/merge.go index 6bc8df9..deee405 100644 --- a/go/buildstep/merge.go +++ b/go/buildstep/merge.go @@ -44,6 +44,9 @@ func MergeInto(ctx context.Context, run forgekey.Runner, dir, remote, base, comm if err := checkFetch(o, dir, remote, commit, ""); err != nil { return res, err } + if err := useMirror(o, dir); err != nil { + return res, err + } env := []string{"GIT_TERMINAL_PROMPT=0"} git := func(args ...string) (string, error) { full := append([]string{"-c", "credential.helper=", "-C", dir}, args...) diff --git a/go/buildstep/mirror_integration_test.go b/go/buildstep/mirror_integration_test.go new file mode 100644 index 0000000..084b78f --- /dev/null +++ b/go/buildstep/mirror_integration_test.go @@ -0,0 +1,65 @@ +//go:build integration + +package buildstep + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// Issue 890 C1 (AC3): a mirror brought to two commits in turn; a fetch through it checks out each commit, the +// alternates name the mirror, and no symlink is in the workspace. +func TestMirrorServesTwoCommitsWithoutSymlinks(t *testing.T) { + root := t.TempDir() + src := filepath.Join(root, "src.git") + work := filepath.Join(root, "work") + for _, d := range []string{src, work} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + gitIn(t, src, "init", "-q", "--bare") + gitIn(t, work, "init", "-q") + var commits []string + for _, c := range []string{"one", "two"} { + if err := os.WriteFile(filepath.Join(work, "f"), []byte(c), 0o644); err != nil { + t.Fatal(err) + } + gitIn(t, work, "add", "f") + gitIn(t, work, "commit", "-q", "-m", c) + commits = append(commits, gitIn(t, work, "rev-parse", "HEAD")) + gitIn(t, work, "push", "-q", src, "HEAD:refs/heads/main") + } + remote := "file://" + src + mirror := filepath.Join(root, "mirror.git") + for i, commit := range commits { + if err := UpdateMirror(context.Background(), forgekey.ExecRunner, mirror, remote, commit, "", AllowFileRemote()); err != nil { + t.Fatal(err) + } + ws := filepath.Join(root, "ws"+string(rune('0'+i))) + if err := os.MkdirAll(ws, 0o755); err != nil { + t.Fatal(err) + } + if err := FetchCommit(context.Background(), forgekey.ExecRunner, ws, remote, commit, "", AllowFileRemote(), WithMirror(mirror)); err != nil { + t.Fatal(err) + } + if head := gitIn(t, ws, "rev-parse", "HEAD"); head != commit { + t.Errorf("HEAD = %s, want %s", head, commit) + } + b, _ := os.ReadFile(filepath.Join(ws, ".git", "objects", "info", "alternates")) + if !strings.Contains(string(b), filepath.Join(mirror, "objects")) { + t.Errorf("alternates %q", b) + } + _ = filepath.Walk(ws, func(p string, fi os.FileInfo, err error) error { + if err == nil && fi.Mode()&os.ModeSymlink != 0 { + t.Errorf("symlink %s", p) + } + return nil + }) + } +} diff --git a/go/buildstep/mirror_test.go b/go/buildstep/mirror_test.go new file mode 100644 index 0000000..21c82fc --- /dev/null +++ b/go/buildstep/mirror_test.go @@ -0,0 +1,106 @@ +package buildstep + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// Issue 890 C1 (ADR 0536 §2): a fetch through a bare mirror names the mirror in .git/objects/info/alternates, a +// plain file, and never links it. +func TestFetchCommitWithMirrorWritesAlternates(t *testing.T) { + mirror := t.TempDir() + ws := t.TempDir() + f := &fakeGit{} + // the fake's init does not create .git; make the objects directory as git init would. + if err := os.MkdirAll(filepath.Join(ws, ".git", "objects", "info"), 0o755); err != nil { + t.Fatal(err) + } + err := FetchCommit(context.Background(), f.run, ws, "ssh://mesh-forge/novox/mesh-sdk.git", sha, "", WithMirror(mirror)) + if err != nil { + t.Fatal(err) + } + p := filepath.Join(ws, ".git", "objects", "info", "alternates") + b, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + if got, want := strings.TrimSpace(string(b)), filepath.Join(mirror, "objects"); got != want { + t.Errorf("alternates %q, want %q", got, want) + } + if fi, err := os.Lstat(p); err != nil || !fi.Mode().IsRegular() { + t.Errorf("alternates is not a plain file: %v %v", fi, err) + } + // the alternates exist before the fetch runs + if len(f.calls) < 3 || !strings.Contains(f.calls[2], "fetch") { + t.Errorf("calls %v", f.calls) + } +} + +func TestWithMirrorRefusesARelativeOrMissingMirror(t *testing.T) { + for _, m := range []string{"rel/mirror", filepath.Join(t.TempDir(), "absent")} { + f := &fakeGit{} + err := FetchCommit(context.Background(), f.run, t.TempDir(), "ssh://mesh-forge/novox/x.git", sha, "", WithMirror(m)) + if err == nil || len(f.calls) != 0 { + t.Errorf("mirror %q: err %v, calls %v", m, err, f.calls) + } + } +} + +func TestUpdateMirrorInitsBareThenFetches(t *testing.T) { + mirror := filepath.Join(t.TempDir(), "mirror.git") + f := &fakeGit{} + if err := UpdateMirror(context.Background(), f.run, mirror, "ssh://mesh-forge/novox/x.git", sha, ""); err != nil { + t.Fatal(err) + } + pm := "git -c credential.helper= -C " + mirror + " " + want := []string{ + "git -c credential.helper= init -q --bare " + mirror, + pm + "remote add origin ssh://mesh-forge/novox/x.git", + pm + "fetch --no-tags origin " + sha, + pm + "cat-file -e " + sha + "^{commit}", + } + if strings.Join(f.calls, "\n") != strings.Join(want, "\n") { + t.Errorf("commands:\n%s\nwant:\n%s", strings.Join(f.calls, "\n"), strings.Join(want, "\n")) + } +} + +func TestUpdateMirrorKeepsAnExistingMirrorAndFallsBackToTheRef(t *testing.T) { + mirror := t.TempDir() // exists: no init + if err := os.WriteFile(filepath.Join(mirror, "HEAD"), []byte("ref: refs/heads/main\n"), 0o644); err != nil { + t.Fatal(err) + } + f := &fakeGit{} + f.rule = func(args string) (r forgekey.Ran) { + if strings.HasSuffix(args, "origin "+sha) { + r.Status = 1 + r.Stderr = "not our ref" + } + return + } + if err := UpdateMirror(context.Background(), f.run, mirror, "ssh://mesh-forge/novox/x.git", sha, "refs/pull/3/head"); err != nil { + t.Fatal(err) + } + all := strings.Join(f.calls, "\n") + if strings.Contains(all, "init") || strings.Contains(all, "remote add") || !strings.Contains(all, "fetch --no-tags origin refs/pull/3/head") { + t.Errorf("commands:\n%s", all) + } +} + +func TestUpdateMirrorRefusesBeforeRunning(t *testing.T) { + f := &fakeGit{} + for _, c := range []struct{ remote, commit string }{ + {"https://x/y.git", sha}, {"ssh://u@h/o/r.git", sha}, {"ssh://h/o/r.git", "abc"}, + } { + if err := UpdateMirror(context.Background(), f.run, t.TempDir(), c.remote, c.commit, ""); err == nil { + t.Errorf("%v accepted", c) + } + } + if len(f.calls) != 0 { + t.Errorf("ran %v", f.calls) + } +} From 2248b985bfac6ef86f7f77973667443e5c4c849b Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 23 Oct 2026 14:15:26 +0200 Subject: [PATCH 2/2] Raise @novox/mesh-sdk to 0.1.16, so its package build is not refused as a second, different 0.1.15 (issue 1899) --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 04ae640..781fd1a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@novox/mesh-sdk", - "version": "0.1.15", + "version": "0.1.16", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@novox/mesh-sdk", - "version": "0.1.15", + "version": "0.1.16", "devDependencies": { "@types/node": "^22.0.0", "typescript": "^5.6.0" diff --git a/package.json b/package.json index 3190b63..ac094db 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@novox/mesh-sdk", - "version": "0.1.15", + "version": "0.1.16", "description": "The stable spine a Novox Mesh module's own code builds against.", "type": "module", "exports": {