Make the build modules' fetch merge the head onto a trunk commit and place siblings beside the tree, so the checker judges the merge and tests read their siblings (issues 1675, 1676, 1677)
This commit is contained in:
@@ -36,6 +36,8 @@ const sdkPackage = "@novox/mesh-sdk"
|
||||
// WorkspaceAge is how long a workspace is kept.
|
||||
const WorkspaceAge = 24 * time.Hour
|
||||
|
||||
var commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||
|
||||
var repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
||||
|
||||
// FetchResult is what a passed fetch step says.
|
||||
@@ -48,6 +50,15 @@ type FetchResult struct {
|
||||
// Refused are the bundles whose packages were not installed, each with why (a module naming packages beyond the SDK
|
||||
// with no package-lock.json). The fetch passes; a build of such a bundle fails naming it (issue 1459).
|
||||
Refused []Refusal `json:"refused"`
|
||||
// The merge preview (ADR 0484 clause 3), only when the call gave a base, as build_go_fetch answers it: the trunk
|
||||
// commit, the merge commit of the head onto it, the paths the merge changes and the tree of each module directory.
|
||||
// A head that does not merge cleanly is an answer: Merge is empty, Conflicts names the files, and the workspace is
|
||||
// left at the trunk commit with nothing installed.
|
||||
Base string `json:"base,omitempty"`
|
||||
Merge string `json:"merge,omitempty"`
|
||||
Conflicts []string `json:"conflicts,omitempty"`
|
||||
Changed []string `json:"changed,omitempty"`
|
||||
Trees map[string]ModuleTree `json:"trees,omitempty"`
|
||||
}
|
||||
|
||||
// Refusal is a module a fetch did not install the packages of.
|
||||
@@ -68,6 +79,7 @@ func (m *Module) fetchTool() StepTool {
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
|
||||
"commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
|
||||
"ref": map[string]any{"type": "string", "description": "refs/pull/<n>/head when only a pull request's head carries the commit"},
|
||||
"base": map[string]any{"type": "string", "description": "optional trunk commit, 40 hex digits: the head is merged onto it as the forge's merge would, and the answer carries the merge commit, the files in conflict (an answer, the workspace is then left at the trunk commit), the changed paths and the tree of each module directory"},
|
||||
},
|
||||
"required": []string{"repository", "commit"},
|
||||
},
|
||||
@@ -80,6 +92,10 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
||||
repository, _ := args["repository"].(string)
|
||||
commit, _ := args["commit"].(string)
|
||||
ref, _ := args["ref"].(string)
|
||||
base, _ := args["base"].(string)
|
||||
if base != "" && !commitOK.MatchString(base) {
|
||||
return nil, fmt.Errorf("%q is not a trunk commit (40 hex digits)", base)
|
||||
}
|
||||
if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") {
|
||||
return nil, fmt.Errorf("%q is not a repository, <owner>/<name>", repository)
|
||||
}
|
||||
@@ -94,15 +110,23 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
||||
gitRun := func(ctx context.Context, env []string, name string, a ...string) forgekey.Ran {
|
||||
return m.Run(ctx, append(m.baseEnv(), env...), name, a...)
|
||||
}
|
||||
remote := "ssh://" + ForgeAlias + "/" + repository + ".git"
|
||||
if err := buildstep.FetchCommit(ctx, gitRun, tree, remote, commit, ref); err != nil {
|
||||
remote := m.remote(repository)
|
||||
if err := buildstep.FetchCommit(ctx, gitRun, tree, remote, commit, ref, m.FetchOptions...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := FetchResult{Workspace: id, Commit: commit, Modules: []string{}, Installed: []string{}, Refused: []Refusal{}}
|
||||
if base != "" {
|
||||
if err := m.merge(ctx, gitRun, &res, tree, remote, base, commit); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if res.Merge == "" {
|
||||
return res, nil // a conflict: the tree is at the trunk, nothing of the head is installed
|
||||
}
|
||||
}
|
||||
dirs, err := m.typeScriptModules(ctx, tree)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := FetchResult{Workspace: id, Commit: commit, Modules: []string{}, Installed: []string{}, Refused: []Refusal{}}
|
||||
for _, d := range dirs {
|
||||
res.Modules = append(res.Modules, d)
|
||||
did, refused, err := m.installOwn(ctx, root, filepath.Join(tree, filepath.FromSlash(d)), d)
|
||||
|
||||
@@ -48,7 +48,7 @@ func (f *fakeMachine) run(_ context.Context, env []string, name string, args ...
|
||||
}
|
||||
} else if name == "git" {
|
||||
for _, a := range args {
|
||||
if slices.Contains([]string{"init", "remote", "fetch", "checkout", "rev-parse", "ls-files"}, a) {
|
||||
if slices.Contains([]string{"init", "remote", "fetch", "checkout", "rev-parse", "ls-files", "merge", "diff"}, a) {
|
||||
key = "git " + a
|
||||
break
|
||||
}
|
||||
|
||||
@@ -31,9 +31,13 @@ type Module struct {
|
||||
Toolchain *Toolchain
|
||||
// Work is the work directory, outside the home: a workspace is <Work>/<step id>/tree. Run runs the commands of a
|
||||
// step (a fake in a test). Clock is the time the steps are timed by.
|
||||
Work string
|
||||
Run forgekey.Runner
|
||||
Clock func() time.Time
|
||||
Work string
|
||||
Run forgekey.Runner
|
||||
// RemoteBase is where repositories are fetched from, "ssh://mesh-forge/" when empty; FetchOptions are the options of
|
||||
// the SDK's fetch and merge. A test points them at a local repository (buildstep.AllowFileRemote).
|
||||
RemoteBase string
|
||||
FetchOptions []buildstep.Option
|
||||
Clock func() time.Time
|
||||
// Toolchains and Steps are the Put of the states of those names.
|
||||
Toolchains, Steps buildstep.Put
|
||||
// ReadSteps and ReadToolchainsState read those two states back for the read tools (issue 1562).
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
package main
|
||||
|
||||
// The merge preview of a fetch (novox/hq ADR 0484 clause 3; issue 1676), as build_go_fetch answers it. The merge itself
|
||||
// is the SDK's (buildstep.MergeInto), the one every build module's fetch step makes.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// remote is the address a repository is fetched from: the forge over ssh as the account, or RemoteBase when a test set it.
|
||||
func (m *Module) remote(repository string) string {
|
||||
base := m.RemoteBase
|
||||
if base == "" {
|
||||
base = "ssh://" + ForgeAlias + "/"
|
||||
}
|
||||
return base + repository + ".git"
|
||||
}
|
||||
|
||||
// merge makes the merge preview of the head onto base and writes the answer into res.
|
||||
func (m *Module) merge(ctx context.Context, run forgekey.Runner, res *FetchResult, tree, remote, base, commit string) error {
|
||||
mr, err := buildstep.MergeInto(ctx, run, tree, remote, base, commit, m.FetchOptions...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
res.Base = mr.Base
|
||||
res.Conflicts, res.Merge, res.Changed = mr.Conflicts, mr.Merge, mr.Changed
|
||||
if res.Merge == "" {
|
||||
return nil
|
||||
}
|
||||
res.Trees, err = m.moduleTrees(ctx, tree, res.Merge)
|
||||
return err
|
||||
}
|
||||
|
||||
// ModuleTree is what the merge holds of one module directory: its tree id, the sha256 of its module.json and the sha256
|
||||
// of each lock file it has (go.sum, package-lock.json), from which the checker and the indexer take the module's inputs.
|
||||
type ModuleTree struct {
|
||||
Tree string `json:"tree"`
|
||||
Manifest string `json:"manifest"`
|
||||
Locks map[string]string `json:"locks,omitempty"`
|
||||
}
|
||||
|
||||
var lockFiles = []string{"go.sum", "package-lock.json"}
|
||||
|
||||
// moduleTrees are the trees, in the merge commit, of every directory of the tree that holds a module.json; the workspace
|
||||
// is at the merge, so the manifest and the locks are read from it.
|
||||
func (m *Module) moduleTrees(ctx context.Context, tree, merge string) (map[string]ModuleTree, error) {
|
||||
r := m.Run(ctx, m.baseEnv(), "git", "-C", tree, "ls-files", "--", "module.json", "*/module.json")
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
trees := map[string]ModuleTree{}
|
||||
for _, l := range strings.Split(r.Stdout, "\n") {
|
||||
if l = strings.TrimSpace(l); l == "" {
|
||||
continue
|
||||
}
|
||||
dir := filepath.ToSlash(filepath.Dir(l))
|
||||
spec := merge + ":" + dir
|
||||
if dir == "." {
|
||||
spec = merge + "^{tree}"
|
||||
}
|
||||
t := m.Run(ctx, m.baseEnv(), "git", "-C", tree, "rev-parse", spec)
|
||||
if t.Status != 0 || t.Err != "" {
|
||||
return nil, fmt.Errorf("git rev-parse %s: %s", spec, firstLine(t.Err+" "+t.Stderr))
|
||||
}
|
||||
mt := ModuleTree{Tree: strings.TrimSpace(t.Stdout)}
|
||||
raw, err := os.ReadFile(filepath.Join(tree, filepath.FromSlash(l)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mt.Manifest = sha256Hex(raw)
|
||||
for _, lock := range lockFiles {
|
||||
if raw, err := os.ReadFile(filepath.Join(tree, filepath.FromSlash(dir), lock)); err == nil {
|
||||
if mt.Locks == nil {
|
||||
mt.Locks = map[string]string{}
|
||||
}
|
||||
mt.Locks[lock] = sha256Hex(raw)
|
||||
}
|
||||
}
|
||||
trees[dir] = mt
|
||||
}
|
||||
return trees, nil
|
||||
}
|
||||
|
||||
func sha256Hex(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) }
|
||||
@@ -0,0 +1,95 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
// The fetch step's merge on real git repositories (issue 1676); preview_test.go proves the commands with a fake runner.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
func git(t *testing.T, dir string, args ...string) string {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", append([]string{"-C", dir, "-c", "user.name=t", "-c", "user.email=t@example.invalid", "-c", "commit.gpgsign=false"}, args...)...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("git %v: %v\n%s", args, err, out)
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
func TestRealFetchMergesAConflictAndAPlainFetch(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
bare, work := filepath.Join(root, "forge", "o", "r.git"), filepath.Join(root, "work")
|
||||
os.MkdirAll(bare, 0o755)
|
||||
os.MkdirAll(work, 0o755)
|
||||
git(t, bare, "init", "-q", "--bare")
|
||||
git(t, work, "init", "-q", "-b", "main")
|
||||
files := typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6"}}`)
|
||||
files["a.txt"] = "one\ntwo\nthree\n"
|
||||
for n, c := range files {
|
||||
write(t, filepath.Join(work, n), c, 0o644)
|
||||
}
|
||||
git(t, work, "add", ".")
|
||||
git(t, work, "commit", "-q", "-m", "fork")
|
||||
fork := git(t, work, "rev-parse", "HEAD")
|
||||
change := func(from, ref string, f map[string]string) string {
|
||||
git(t, work, "checkout", "-q", "-B", "tmp", from)
|
||||
for n, c := range f {
|
||||
write(t, filepath.Join(work, n), c, 0o644)
|
||||
}
|
||||
git(t, work, "add", ".")
|
||||
git(t, work, "commit", "-q", "-m", "c")
|
||||
git(t, work, "push", "-q", "-f", bare, "HEAD:"+ref)
|
||||
return git(t, work, "rev-parse", "HEAD")
|
||||
}
|
||||
trunk := change(fork, "refs/heads/main", map[string]string{"a.txt": "one\ntwo\nTRUNK\n"})
|
||||
clean := change(fork, "refs/pull/1/head", map[string]string{"modules/m/extra.ts": "export {}\n"})
|
||||
clash := change(fork, "refs/pull/2/head", map[string]string{"a.txt": "one\ntwo\nHEAD\n"})
|
||||
|
||||
m, _, _ := newMachine(t)
|
||||
m.Run = forgekey.ExecRunner
|
||||
m.RemoteBase = "file://" + filepath.Join(root, "forge") + "/"
|
||||
m.FetchOptions = []buildstep.Option{buildstep.AllowFileRemote()}
|
||||
|
||||
out, err := m.fetch(context.Background(), map[string]any{"repository": "o/r", "commit": clean, "ref": "refs/pull/1/head", "base": trunk})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := out.(FetchResult)
|
||||
tree := filepath.Join(m.Work, r.Workspace, "tree")
|
||||
if r.Merge == "" || strings.Join(r.Changed, ",") != "modules/m/extra.ts" || len(r.Conflicts) != 0 || len(r.Modules) != 1 {
|
||||
t.Fatalf("clean: %+v", r)
|
||||
}
|
||||
if want := git(t, tree, "rev-parse", r.Merge+":modules/m"); r.Trees["modules/m"].Tree != want {
|
||||
t.Errorf("tree = %s, want %s", r.Trees["modules/m"].Tree, want)
|
||||
}
|
||||
if got := git(t, tree, "rev-parse", "HEAD"); got != r.Merge {
|
||||
t.Errorf("the workspace is at %s, want the merge %s", got, r.Merge)
|
||||
}
|
||||
|
||||
out, err = m.fetch(context.Background(), map[string]any{"repository": "o/r", "commit": clash, "ref": "refs/pull/2/head", "base": trunk})
|
||||
if err != nil {
|
||||
t.Fatalf("a conflict failed the step: %v", err)
|
||||
}
|
||||
r = out.(FetchResult)
|
||||
if r.Merge != "" || strings.Join(r.Conflicts, ",") != "a.txt" {
|
||||
t.Fatalf("conflict: %+v", r)
|
||||
}
|
||||
if got := git(t, filepath.Join(m.Work, r.Workspace, "tree"), "rev-parse", "HEAD"); got != trunk {
|
||||
t.Errorf("after a conflict the workspace is at %s, want the trunk %s", got, trunk)
|
||||
}
|
||||
|
||||
out, err = m.fetch(context.Background(), map[string]any{"repository": "o/r", "commit": clean, "ref": "refs/pull/1/head"})
|
||||
if err != nil || out.(FetchResult).Merge != "" || out.(FetchResult).Trees != nil {
|
||||
t.Errorf("no base: %+v %v", out, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
const baseSHA = "abcdef0123456789abcdef0123456789abcdef01"
|
||||
|
||||
func commandLines(f *fakeMachine) string {
|
||||
var out []string
|
||||
for _, c := range f.calls {
|
||||
out = append(out, joined(c))
|
||||
}
|
||||
return strings.Join(out, "\n")
|
||||
}
|
||||
|
||||
// 1676/AC1: with a base the fetch merges the head onto it through the SDK's merge (the check merge's identity and flags,
|
||||
// hooks off), and answers the base, the merge commit, the changed paths and the tree, manifest and locks of each module
|
||||
// directory; the TypeScript modules are installed from the merged tree.
|
||||
func TestFetchWithABaseMergesAndAnswersThePreview(t *testing.T) {
|
||||
m, f := fetchMachine(t, typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6"}}`), nil)
|
||||
f.answer["git diff"] = forgekey.Ran{Stdout: "modules/m/package.json\n"}
|
||||
args := map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA, "base": baseSHA}
|
||||
out, err := m.fetch(context.Background(), args)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := out.(FetchResult)
|
||||
if r.Base != baseSHA || r.Merge != commitSHA || strings.Join(r.Changed, ",") != "modules/m/package.json" || len(r.Conflicts) != 0 {
|
||||
t.Errorf("result = %+v", r)
|
||||
}
|
||||
if mt, ok := r.Trees["modules/m"]; !ok || mt.Tree != commitSHA || mt.Manifest == "" {
|
||||
t.Errorf("trees = %+v", r.Trees)
|
||||
}
|
||||
if len(r.Modules) != 1 || r.Modules[0] != "modules/m" {
|
||||
t.Errorf("modules = %v", r.Modules)
|
||||
}
|
||||
for _, want := range []string{"fetch --no-tags origin " + baseSHA, "core.hooksPath=/dev/null", "merge --no-ff --no-edit --no-verify"} {
|
||||
if !strings.Contains(commandLines(f), want) {
|
||||
t.Errorf("git was not asked %q:\n%s", want, commandLines(f))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1676/AC2: a conflict is an answer: the step passes with the files named, no merge commit, and nothing installed.
|
||||
func TestFetchWithAConflictAnswersTheFilesAndInstallsNothing(t *testing.T) {
|
||||
files := typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6","lodash":"^4"}}`)
|
||||
files["modules/m/package-lock.json"] = `{"lockfileVersion":3}`
|
||||
m, f := fetchMachine(t, files, nil)
|
||||
f.answer["git merge"] = forgekey.Ran{Status: 1, Stderr: "CONFLICT\n"}
|
||||
f.answer["git diff"] = forgekey.Ran{Stdout: "a.ts\nsub/b.ts\n"}
|
||||
out, err := m.fetch(context.Background(), map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA, "base": baseSHA})
|
||||
if err != nil {
|
||||
t.Fatalf("a conflict failed the step: %v", err)
|
||||
}
|
||||
r := out.(FetchResult)
|
||||
if r.Merge != "" || strings.Join(r.Conflicts, ",") != "a.ts,sub/b.ts" || len(r.Changed) != 0 || len(npmCalls(f)) != 0 {
|
||||
t.Errorf("result = %+v, npm calls %d", r, len(npmCalls(f)))
|
||||
}
|
||||
if !strings.Contains(commandLines(f), "merge --abort") {
|
||||
t.Errorf("the merge was not undone:\n%s", commandLines(f))
|
||||
}
|
||||
}
|
||||
|
||||
// 1676/AC3: no base is today's fetch: no merge command, and none of the preview in the answer.
|
||||
func TestFetchWithoutABaseMakesNoMerge(t *testing.T) {
|
||||
m, f := fetchMachine(t, typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6"}}`), nil)
|
||||
out, err := m.fetch(context.Background(), fetchArgs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := out.(FetchResult)
|
||||
if strings.Contains(commandLines(f), "merge") || strings.Contains(commandLines(f), "unshallow") || r.Base != "" || r.Merge != "" || r.Trees != nil {
|
||||
t.Errorf("a fetch without a base merged: %+v\n%s", r, commandLines(f))
|
||||
}
|
||||
}
|
||||
|
||||
func TestABaseThatIsNotACommitIsRefusedBeforeAnythingRuns(t *testing.T) {
|
||||
m, f := fetchMachine(t, nil, nil)
|
||||
if _, err := m.fetch(context.Background(), map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA, "base": "main"}); err == nil || len(f.calls) != 0 {
|
||||
t.Errorf("err = %v after %d commands", err, len(f.calls))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user