Compare commits

..
27 changed files with 1079 additions and 138 deletions
@@ -17,6 +17,8 @@ import (
"git.novox.be/novox/mesh-sdk/go/buildstep"
)
var commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
// ForgeAlias is the ssh name the forge-account provision sets up in the account's ssh files (ADR 0409 §1); the
// remote of a fetch is ssh://<it>/<owner>/<repository>.git.
const ForgeAlias = "mesh-forge"
@@ -37,6 +39,18 @@ type FetchResult struct {
Workspace string `json:"workspace"`
Commit string `json:"commit"`
Modules []string `json:"modules"`
// The merge preview (ADR 0484 clause 3), only when the call gave a base: the trunk commit, the merge commit of the
// head onto it, the paths the merge changes, and the tree of each module directory (a directory holding a
// module.json) in the merge. A head that does not merge cleanly is an answer: Merge is empty, Conflicts names the
// files, and the workspace is left at the trunk commit.
Base string `json:"base,omitempty"`
Merge string `json:"merge,omitempty"`
Conflicts []string `json:"conflicts,omitempty"`
Changed []string `json:"changed,omitempty"`
Trees map[string]buildstep.ModuleTree `json:"trees,omitempty"`
// Beside are the sibling repositories placed as plain directories under the workspace's beside directory, which
// build_go_test hands to the tests as MESH_CHECK_BESIDE (ADR 0484 clause 4).
Beside []Sibling `json:"beside,omitempty"`
}
// fetchTool is the fetch step tool.
@@ -52,6 +66,12 @@ func (m *Module) fetchTool() StepTool {
"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
"commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
"ref": map[string]any{"type": "string", "description": "refs/pull/<n>/head when only a pull request's head carries the commit"},
"base": map[string]any{"type": "string", "description": "optional trunk commit, 40 hex digits: the head is merged onto it as the forge's merge would, and the answer carries the merge commit, the files in conflict (an answer, the workspace is then left at the trunk commit), the changed paths and the tree of each module directory"},
"beside": map[string]any{"type": "array", "description": "optional sibling repositories the tests read, each placed as a plain directory (no symlink) under the workspace's beside/<name>, which build_go_test gives the tests as MESH_CHECK_BESIDE",
"items": map[string]any{"type": "object", "properties": map[string]any{
"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
"commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
}, "required": []string{"repository", "commit"}}},
},
"required": []string{"repository", "commit"},
},
@@ -64,6 +84,14 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
repository, _ := args["repository"].(string)
commit, _ := args["commit"].(string)
ref, _ := args["ref"].(string)
base, _ := args["base"].(string)
if base != "" && !commitOK.MatchString(base) {
return nil, fmt.Errorf("%q is not a trunk commit (40 hex digits)", base)
}
siblings, err := siblingsArg(args["beside"])
if err != nil {
return nil, err
}
if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") {
return nil, fmt.Errorf("%q is not a repository, <owner>/<name>", repository)
}
@@ -74,14 +102,27 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
if err := os.MkdirAll(tree, 0o700); err != nil {
return nil, err
}
remote := "ssh://" + ForgeAlias + "/" + repository + ".git"
if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref); err != nil {
remote := m.remote(repository)
if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref, m.FetchOptions...); err != nil {
return nil, err
}
res := FetchResult{Workspace: id, Commit: commit}
if base != "" {
if err := m.merge(ctx, &res, tree, remote, base, commit); err != nil {
return nil, err
}
}
if res.Beside, err = m.placeSiblings(ctx, root, siblings); err != nil {
return nil, err
}
dirs, err := m.goModDirs(ctx, tree)
if err != nil {
return nil, err
}
res.Modules = dirs
if base != "" && res.Merge == "" {
return res, nil // a conflict: the tree is at the trunk, nothing of the head is built or tested
}
for _, d := range dirs {
if _, err := os.Stat(filepath.Join(tree, d, "vendor", "modules.txt")); err == nil {
continue // a module that vendors resolves nothing
@@ -93,7 +134,7 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
}
}
}
return FetchResult{Workspace: id, Commit: commit, Modules: dirs}, nil
return res, nil
}
// goModDirs are the directories of the tree that hold a go.mod, relative to it, in order.
@@ -30,7 +30,8 @@ type fakeStep struct {
gomods string
gate chan struct{} // when set, the first git command waits for it
goRule func(line string) forgekey.Ran
vendors []string // directories (relative to the tree) that get a vendor/modules.txt at checkout
vendors []string // directories (relative to the tree) that get a vendor/modules.txt at checkout
rule func(line string) (forgekey.Ran, bool) // answers a command before the defaults do
}
func (f *fakeStep) run(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
@@ -41,6 +42,11 @@ func (f *fakeStep) run(ctx context.Context, env []string, name string, args ...s
if f.gate != nil && strings.Contains(line, " init ") {
<-f.gate
}
if f.rule != nil {
if r, ok := f.rule(line); ok {
return r
}
}
switch {
case name == "git" && strings.Contains(line, " rev-parse HEAD"):
return forgekey.Ran{Stdout: commitSHA + "\n"}
@@ -60,7 +60,7 @@ func (m *Module) integration(ctx context.Context, workspace, tree string, packag
return nil, fmt.Errorf("the integration level's bus: %w", err)
}
defer stopBus()
env := []string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats}
env := append([]string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats}, m.besideEnv(workspace)...)
// Compiling counts as setup: the tests are built once, here, with the addresses a TestMain may read, and the run
// below finds them built. Whether they compile is for the run to say; its failure is not read here.
for _, d := range dirs {
@@ -37,6 +37,10 @@ type Module struct {
// both outside the account's home (ADR 0466 §1); Home is the account's home.
Run forgekey.Runner
Home, Work, Cache string
// RemoteBase is where repositories are fetched from, "ssh://mesh-forge/" when empty; FetchOptions are the options of
// the SDK's fetch and merge. A test points them at a local repository (buildstep.AllowFileRemote).
RemoteBase string
FetchOptions []buildstep.Option
// HTTP fetches the files an archive artifact is pinned to (url and sha256); a client of its own with a fifteen-minute
// timeout when nil. A test serves its own.
HTTP *http.Client
@@ -0,0 +1,119 @@
package main
// The merge preview and the siblings of a fetch (novox/hq ADR 0484 clauses 3 and 4; issues 1675 and 1677). The merge
// itself is the SDK's (buildstep.MergeInto), the one every build module's fetch step makes; what is here is the answer
// built from it and the placing of the sibling repositories the tests read.
import (
"context"
"fmt"
"io/fs"
"os"
"path/filepath"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/buildstep"
)
// Sibling is a repository placed beside the fetched tree: its name (the directory under beside/) and its commit.
type Sibling struct {
Repository string `json:"repository"`
Commit string `json:"commit"`
Dir string `json:"dir"`
}
// remote is the address a repository is fetched from: the forge over ssh as the account, or RemoteBase when a test set it.
func (m *Module) remote(repository string) string {
base := m.RemoteBase
if base == "" {
base = "ssh://" + ForgeAlias + "/"
}
return base + repository + ".git"
}
// merge makes the merge preview of the head onto base and writes the answer into res.
func (m *Module) merge(ctx context.Context, res *FetchResult, tree, remote, base, commit string) error {
mr, err := buildstep.MergeInto(ctx, m.Run, tree, remote, base, commit, m.FetchOptions...)
if err != nil {
return err
}
res.Base = mr.Base
res.Conflicts, res.Merge, res.Changed = mr.Conflicts, mr.Merge, mr.Changed
if res.Merge == "" {
return nil
}
res.Trees, err = buildstep.ModuleTrees(ctx, m.Run, nil, tree, res.Merge)
return err
}
type siblingArg struct{ repository, commit string }
func siblingsArg(v any) ([]siblingArg, error) {
if v == nil {
return nil, nil
}
list, ok := v.([]any)
if !ok {
return nil, fmt.Errorf("beside is a list of {repository, commit}")
}
var out []siblingArg
seen := map[string]bool{}
for _, e := range list {
o, _ := e.(map[string]any)
repository, _ := o["repository"].(string)
commit, _ := o["commit"].(string)
if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") {
return nil, fmt.Errorf("beside: %q is not a repository, <owner>/<name>", repository)
}
if !commitOK.MatchString(commit) {
return nil, fmt.Errorf("beside: %q is not a full commit (40 hex digits)", commit)
}
name := repository[strings.Index(repository, "/")+1:]
if seen[name] {
return nil, fmt.Errorf("beside: two repositories are named %s", name)
}
seen[name] = true
out = append(out, siblingArg{repository, commit})
}
return out, nil
}
// BesideDir is where a workspace's siblings are placed, and what MESH_CHECK_BESIDE names.
func (m *Module) besideDir(workspace string) string {
return filepath.Join(m.Work, workspace, "beside")
}
// placeSiblings fetches each sibling into beside/<name> of the workspace root, as a plain directory. A symlink anywhere
// in a placed tree fails the step naming it: the mesh creates none, and a test must not be led out of its sibling.
func (m *Module) placeSiblings(ctx context.Context, root string, siblings []siblingArg) ([]Sibling, error) {
var placed []Sibling
for _, s := range siblings {
name := s.repository[strings.Index(s.repository, "/")+1:]
dir := filepath.Join(root, "beside", name)
if err := os.MkdirAll(dir, 0o700); err != nil {
return nil, err
}
if err := buildstep.FetchCommit(ctx, m.Run, dir, m.remote(s.repository), s.commit, "", m.FetchOptions...); err != nil {
return nil, fmt.Errorf("beside %s: %w", s.repository, err)
}
if link := firstSymlink(dir); link != "" {
return nil, fmt.Errorf("beside %s holds a symlink, %s", s.repository, link)
}
placed = append(placed, Sibling{Repository: s.repository, Commit: s.commit, Dir: name})
}
sort.Slice(placed, func(i, j int) bool { return placed[i].Dir < placed[j].Dir })
return placed, nil
}
func firstSymlink(dir string) string {
var found string
filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
if err == nil && d.Type()&fs.ModeSymlink != 0 && found == "" {
found, _ = filepath.Rel(dir, p)
return fs.SkipAll
}
return nil
})
return found
}
@@ -0,0 +1,228 @@
//go:build integration
package main
// The fetch step's merge and siblings on real git repositories and the real go (issues 1675 and 1677): they start
// processes, so they carry the tag; preview_test.go proves the commands with a fake runner.
import (
"encoding/json"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func git(t *testing.T, dir string, args ...string) string {
t.Helper()
cmd := exec.Command("git", append([]string{"-C", dir, "-c", "user.name=t", "-c", "user.email=t@example.invalid", "-c", "commit.gpgsign=false"}, args...)...)
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("git %v: %v\n%s", args, err, out)
}
return strings.TrimSpace(string(out))
}
func put(t *testing.T, dir, name, body string) {
t.Helper()
p := filepath.Join(dir, name)
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
t.Fatal(err)
}
}
// forge is a directory of bare repositories <owner>/<name>.git standing in for the forge, and a module fetching from it.
type forge struct {
t *testing.T
root string
}
func newForge(t *testing.T) *forge { return &forge{t: t, root: t.TempDir()} }
// repo makes the repository owner/name with the files on main and answers its work directory and the commit.
func (f *forge) repo(name string, files map[string]string) (work, commit string) {
bare := filepath.Join(f.root, "forge", name+".git")
work = filepath.Join(f.root, "work", name)
os.MkdirAll(bare, 0o755)
os.MkdirAll(work, 0o755)
git(f.t, bare, "init", "-q", "--bare")
git(f.t, work, "init", "-q", "-b", "main")
for n, c := range files {
put(f.t, work, n, c)
}
git(f.t, work, "add", ".")
git(f.t, work, "commit", "-q", "-m", "one")
git(f.t, work, "push", "-q", bare, "HEAD:refs/heads/main")
return work, git(f.t, work, "rev-parse", "HEAD")
}
// commit makes a commit off `from` in the repository's work directory and publishes it as the pull request's head.
func (f *forge) commit(name, work, from string, files map[string]string, ref string) string {
git(f.t, work, "checkout", "-q", "-B", "b"+ref[len(ref)-3:], from)
for n, c := range files {
put(f.t, work, n, c)
}
git(f.t, work, "add", ".")
git(f.t, work, "commit", "-q", "-m", "change")
sha := git(f.t, work, "rev-parse", "HEAD")
git(f.t, work, "push", "-q", "-f", filepath.Join(f.root, "forge", name+".git"), "HEAD:"+ref)
return sha
}
func (f *forge) module(t *testing.T) (*Module, *keptStates) {
m, states := fetchModule(t, &fakeStep{})
m.Run = StepRunner(m.Home)
m.RemoteBase = "file://" + filepath.Join(f.root, "forge") + "/"
m.FetchOptions = []buildstep.Option{buildstep.AllowFileRemote()}
return m, states
}
// waitLong is waitEnd for a step that runs the real go, whose first build in a cold cache takes longer than 3 s.
func waitLong(t *testing.T, s *keptStates) delivery.Step {
t.Helper()
select {
case key := <-s.ended:
s.mu.Lock()
defer s.mu.Unlock()
v := s.steps[key]
return v[len(v)-1]
case <-time.After(2 * time.Minute):
t.Fatal("the step never ended on its key")
return delivery.Step{}
}
}
func fetched(t *testing.T, m *Module, states *keptStates, args map[string]any) (FetchResult, delivery.Step) {
t.Helper()
if _, err := fetchTool(t, m)(args); err != nil {
t.Fatal(err)
}
st := states.waitEnd(t)
var res FetchResult
if st.State == delivery.StepPassed {
if err := json.Unmarshal(st.Result, &res); err != nil {
t.Fatal(err)
}
}
return res, st
}
var baseFiles = map[string]string{
"go.mod": "module fixture/m\n\ngo 1.26\n",
"a.txt": "one\ntwo\nthree\n",
"modules/m/module.json": "{\"name\":\"m\"}\n",
"modules/m/go.sum": "",
"modules/m/m.txt": "m\n",
"modules/other/other.txt": "o\n",
}
// 1675/AC1 and AC2 on real git: a clean merge answers the merge commit, the changed paths (the trunk's change is not
// among them) and the tree of each module directory, which equals git's own; a conflict is a passed step naming the files,
// leaving the workspace at the trunk commit; and no base is today's fetch.
func TestRealFetchMergesConflictsAndPlainlyFetches(t *testing.T) {
f := newForge(t)
work, fork := f.repo("o/r", baseFiles)
trunk := f.commit("o/r", work, fork, map[string]string{"modules/other/other.txt": "trunk\n", "a.txt": "one\ntwo\nTRUNK\n"}, "refs/heads/main")
clean := f.commit("o/r", work, fork, map[string]string{"modules/m/m.txt": "head\n"}, "refs/pull/1/head")
clash := f.commit("o/r", work, fork, map[string]string{"a.txt": "one\ntwo\nHEAD\n"}, "refs/pull/2/head")
m, states := f.module(t)
res, st := fetched(t, m, states, map[string]any{"repository": "o/r", "commit": clean, "ref": "refs/pull/1/head", "base": trunk})
if st.State != delivery.StepPassed || res.Merge == "" || res.Base != trunk || res.Commit != clean || len(res.Conflicts) != 0 {
t.Fatalf("clean: %s %s", st.State, st.Result)
}
if strings.Join(res.Changed, ",") != "modules/m/m.txt" {
t.Errorf("changed = %v, want the head's own change alone", res.Changed)
}
tree := res.Trees["modules/m"]
if want := git(t, filepath.Join(m.Work, res.Workspace, "tree"), "rev-parse", res.Merge+":modules/m"); tree.Tree != want {
t.Errorf("tree = %s, want %s", tree.Tree, want)
}
if tree.Manifest != sha256Hex([]byte("{\"name\":\"m\"}\n")) || tree.Locks["go.sum"] != sha256Hex(nil) {
t.Errorf("manifest and locks = %+v", tree)
}
if got := git(t, filepath.Join(m.Work, res.Workspace, "tree"), "rev-parse", "HEAD"); got != res.Merge {
t.Errorf("the workspace is at %s, want the merge %s", got, res.Merge)
}
res, st = fetched(t, m, states, map[string]any{"repository": "o/r", "commit": clash, "ref": "refs/pull/2/head", "base": trunk})
if st.State != delivery.StepPassed || res.Merge != "" || strings.Join(res.Conflicts, ",") != "a.txt" {
t.Fatalf("conflict: %s %s", st.State, st.Result)
}
if got := git(t, filepath.Join(m.Work, res.Workspace, "tree"), "rev-parse", "HEAD"); got != trunk {
t.Errorf("after a conflict the workspace is at %s, want the trunk %s", got, trunk)
}
res, st = fetched(t, m, states, map[string]any{"repository": "o/r", "commit": clean, "ref": "refs/pull/1/head"})
if st.State != delivery.StepPassed || res.Merge != "" || res.Base != "" || res.Trees != nil || len(res.Modules) != 1 {
t.Errorf("no base: %s %s", st.State, st.Result)
}
}
// 1677/AC1 on the real go: the unit level of a repository whose test reads MESH_CHECK_BESIDE fails loudly without the
// sibling and passes with it; the sibling is a plain directory with no symlink in it; and a sibling that holds a
// symlink fails the fetch naming it.
func TestRealTestsReadTheSiblingBesideTheTree(t *testing.T) {
f := newForge(t)
files := map[string]string{}
for _, n := range []string{"go.mod", "beside.go", "beside_test.go"} {
raw, err := os.ReadFile(filepath.Join("testdata", "repos", "beside", n))
if err != nil {
t.Fatal(err)
}
files[n] = string(raw)
}
_, head := f.repo("o/r", files)
_, sibling := f.repo("o/sibling", map[string]string{"data.txt": "from the sibling\n"})
m, states := f.module(t)
without, _ := fetched(t, m, states, map[string]any{"repository": "o/r", "commit": head})
if _, err := m.testToolRun(t, without.Workspace); err != nil {
t.Fatal(err)
}
var res TestResult
st := waitLong(t, states)
json.Unmarshal(st.Result, &res)
if st.State != delivery.StepPassed || res.Passed || len(res.Failed) != 1 || !strings.Contains(res.Failed[0].Output, "MESH_CHECK_BESIDE is not set") {
t.Fatalf("without the sibling: %s %s", st.State, st.Result)
}
with, st2 := fetched(t, m, states, map[string]any{"repository": "o/r", "commit": head,
"beside": []any{map[string]any{"repository": "o/sibling", "commit": sibling}}})
if st2.State != delivery.StepPassed || len(with.Beside) != 1 {
t.Fatalf("with the sibling: %s %s", st2.State, st2.Result)
}
m.testToolRun(t, with.Workspace)
res = TestResult{}
st = waitLong(t, states)
json.Unmarshal(st.Result, &res)
if !res.Passed {
t.Errorf("with the sibling the level failed: %s", st.Result)
}
if link := firstSymlink(filepath.Join(m.Work, with.Workspace, "beside")); link != "" {
t.Errorf("the placed tree holds a symlink, %s", link)
}
_, bad := f.repo("o/linked", map[string]string{"data.txt": "x\n"})
work := filepath.Join(f.root, "work", "o/linked")
if err := os.Symlink("/etc", filepath.Join(work, "etc-link")); err != nil {
t.Fatal(err)
}
git(t, work, "add", ".")
git(t, work, "commit", "-q", "-m", "link")
bad = git(t, work, "rev-parse", "HEAD")
git(t, work, "push", "-q", "-f", filepath.Join(f.root, "forge", "o/linked.git"), "HEAD:refs/heads/main")
_, st3 := fetched(t, m, states, map[string]any{"repository": "o/r", "commit": head,
"beside": []any{map[string]any{"repository": "o/linked", "commit": bad}}})
if st3.State != delivery.StepFailed || !strings.Contains(st3.Reason, "etc-link") {
t.Errorf("a sibling with a symlink: %s %s", st3.State, st3.Reason)
}
}
@@ -0,0 +1,317 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
const (
baseSHA = "abcdef0123456789abcdef0123456789abcdef01"
mergeSHA = "fedcba9876543210fedcba9876543210fedcba98"
treeSHA = "1111111111111111111111111111111111111111"
)
// previewRules answers the merge's commands as git does: the merge commit, the changed paths, the manifests, the trees.
func previewRules(conflict bool) func(string) (forgekey.Ran, bool) {
merged := false
return func(line string) (forgekey.Ran, bool) {
switch {
case strings.Contains(line, "--is-shallow-repository"):
return forgekey.Ran{Stdout: "true\n"}, true
case strings.Contains(line, " merge --no-ff"):
if conflict {
return forgekey.Ran{Status: 1, Stderr: "CONFLICT (content): Merge conflict in a.go\n"}, true
}
merged = true
return forgekey.Ran{}, true
case strings.Contains(line, "diff --name-only --diff-filter=U"):
return forgekey.Ran{Stdout: "a.go\nsub/b.go\n"}, true
case strings.Contains(line, "diff --name-only --no-renames"):
return forgekey.Ran{Stdout: "a.go\nmodules/m/main.go\n"}, true
case strings.Contains(line, " rev-parse HEAD") && merged:
return forgekey.Ran{Stdout: mergeSHA + "\n"}, true
case strings.Contains(line, " ls-files ") && strings.Contains(line, "module.json"):
return forgekey.Ran{Stdout: "modules/m/module.json\n"}, true
case strings.Contains(line, " rev-parse "+mergeSHA+":modules/m"):
return forgekey.Ran{Stdout: treeSHA + "\n"}, true
}
return forgekey.Ran{}, false
}
}
// 1675/AC1: with a base the fetch merges the head onto it with the SDK's merge (the identity and flags of the check
// merge, hooks off), and answers base, merge commit, changed paths and, per module directory, tree, manifest hash and lock hashes.
func TestFetchWithABaseMergesAndAnswersThePreview(t *testing.T) {
f := &fakeStep{gomods: "go.mod\n", rule: previewRules(false)}
m, states := fetchModule(t, f)
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
// the workspace's module.json and go.sum are on disk once the fake checks the head out
for i, a := range args {
if a == "-C" && len(args) > i+1 && strings.Contains(strings.Join(args, " "), " checkout ") {
os.MkdirAll(filepath.Join(args[i+1], "modules", "m"), 0o755)
os.WriteFile(filepath.Join(args[i+1], "modules", "m", "module.json"), []byte("{}\n"), 0o644)
os.WriteFile(filepath.Join(args[i+1], "modules", "m", "go.sum"), []byte("sum\n"), 0o644)
}
}
return f.run(ctx, env, name, args...)
}
args := map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA, "base": baseSHA}
if _, err := fetchTool(t, m)(args); err != nil {
t.Fatal(err)
}
st := states.waitEnd(t)
if st.State != delivery.StepPassed {
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
}
var res FetchResult
if err := json.Unmarshal(st.Result, &res); err != nil {
t.Fatal(err)
}
if res.Base != baseSHA || res.Merge != mergeSHA || res.Commit != commitSHA || len(res.Conflicts) != 0 ||
strings.Join(res.Changed, ",") != "a.go,modules/m/main.go" {
t.Errorf("result = %s", st.Result)
}
mt, ok := res.Trees["modules/m"]
if !ok || mt.Tree != treeSHA || mt.Manifest != sha256Hex([]byte("{}\n")) || mt.Locks["go.sum"] != sha256Hex([]byte("sum\n")) {
t.Errorf("trees = %+v", res.Trees)
}
lines := strings.Join(f.lines(), "\n")
for _, want := range []string{"fetch --no-tags --unshallow origin " + baseSHA, "checkout -q --detach " + baseSHA,
"core.hooksPath=/dev/null", "merge --no-ff --no-edit --no-verify"} {
if !strings.Contains(lines, want) {
t.Errorf("git was not asked %q:\n%s", want, lines)
}
}
}
// 1675/AC2: a conflict is an answer: the step ends passed, the files are named, no merge commit, nothing downloaded.
func TestFetchWithAConflictEndsPassedAndNamesTheFiles(t *testing.T) {
f := &fakeStep{gomods: "go.mod\n", rule: previewRules(true)}
m, states := fetchModule(t, f)
args := map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA, "base": baseSHA}
if _, err := fetchTool(t, m)(args); err != nil {
t.Fatal(err)
}
st := states.waitEnd(t)
if st.State != delivery.StepPassed {
t.Fatalf("a conflict ended the step %s: %s", st.State, st.Reason)
}
var res FetchResult
json.Unmarshal(st.Result, &res)
if res.Merge != "" || res.Base != baseSHA || strings.Join(res.Conflicts, ",") != "a.go,sub/b.go" || len(res.Changed) != 0 {
t.Errorf("result = %s", st.Result)
}
lines := strings.Join(f.lines(), "\n")
if !strings.Contains(lines, "merge --abort") || strings.Contains(lines, "mod download") {
t.Errorf("the merge was not undone, or modules were downloaded:\n%s", lines)
}
}
// 1675/AC3: no base is today's fetch: no merge, no extra command, and the answer carries none of the preview.
func TestFetchWithoutABaseIsTodaysFetch(t *testing.T) {
f := &fakeStep{gomods: "go.mod\n"}
m, states := fetchModule(t, f)
fetchTool(t, m)(fetchArgs)
st := states.waitEnd(t)
lines := strings.Join(f.lines(), "\n")
if strings.Contains(lines, "merge") || strings.Contains(lines, "unshallow") {
t.Errorf("a fetch without a base merged:\n%s", lines)
}
for _, key := range []string{"base", "merge", "conflicts", "changed", "trees", "beside"} {
if strings.Contains(string(st.Result), `"`+key+`"`) {
t.Errorf("the answer carries %q without a base: %s", key, st.Result)
}
}
}
func TestABaseThatIsNotACommitIsRefusedBeforeAnythingRuns(t *testing.T) {
f := &fakeStep{}
m, states := fetchModule(t, f)
fetchTool(t, m)(map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA, "base": "main"})
if st := states.waitEnd(t); st.State != delivery.StepFailed || len(f.lines()) != 0 {
t.Errorf("the step ended %s after %d commands", st.State, len(f.lines()))
}
}
// 1677/AC1: each sibling is fetched into beside/<name> of the workspace, and the tests of the workspace are given
// MESH_CHECK_BESIDE naming that directory, at the unit level; a workspace without siblings is given none.
func TestSiblingsArePlacedBesideAndTheUnitLevelIsGivenTheirDirectory(t *testing.T) {
f := &fakeStep{gomods: "go.mod\n"}
m, states := fetchModule(t, f)
args := map[string]any{"repository": "novox/mesh-controller", "commit": commitSHA, "beside": []any{
map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA},
map[string]any{"repository": "novox/mesh-host", "commit": commitSHA}}}
if _, err := fetchTool(t, m)(args); err != nil {
t.Fatal(err)
}
st := states.waitEnd(t)
var res FetchResult
json.Unmarshal(st.Result, &res)
if st.State != delivery.StepPassed {
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
}
if len(res.Beside) != 2 || res.Beside[0].Dir != "mesh-catalog" || res.Beside[1].Dir != "mesh-host" {
t.Fatalf("result = %s", st.Result)
}
lines := strings.Join(f.lines(), "\n")
for _, d := range []string{"beside/mesh-catalog", "beside/mesh-host"} {
if !strings.Contains(lines, filepath.Join(m.Work, res.Workspace, d)+" init") && !strings.Contains(lines, "-C "+filepath.Join(m.Work, res.Workspace, d)) {
t.Errorf("nothing was fetched into %s:\n%s", d, lines)
}
}
if !strings.Contains(lines, "remote add origin ssh://mesh-forge/novox/mesh-catalog.git") {
t.Errorf("the sibling was not fetched over ssh:\n%s", lines)
}
calls := &[]cmdRecord{}
m.Run = capturedRunner(t, "failing", calls)
if err := os.MkdirAll(filepath.Join(m.Work, res.Workspace, "tree"), 0o755); err != nil {
t.Fatal(err)
}
if _, err := m.testToolRun(t, res.Workspace); err != nil {
t.Fatal(err)
}
states.waitEnd(t)
want := "MESH_CHECK_BESIDE=" + filepath.Join(m.Work, res.Workspace, "beside")
if !envHas(*calls, "go", want) {
t.Errorf("no go command was given %s", want)
}
}
func (m *Module) testToolRun(t *testing.T, ws string) (any, error) {
t.Helper()
for _, tool := range m.Tools(m.StepTools()) {
if tool.Name == "build_go_test" {
return tool.Run(map[string]any{"workspace": ws, "level": "unit"})
}
}
t.Fatal("no tool build_go_test")
return nil, nil
}
func envHas(calls []cmdRecord, name, want string) bool {
for _, c := range calls {
if strings.HasPrefix(c.line, name+" ") {
for _, e := range c.env {
if e == want {
return true
}
}
}
}
return false
}
func TestAWorkspaceWithoutSiblingsIsGivenNoBesideVariable(t *testing.T) {
m, states, ws, calls := workspaceFor(t, "failing")
runUnit(t, m, states, ws)
for _, c := range *calls {
for _, e := range c.env {
if strings.HasPrefix(e, "MESH_CHECK_BESIDE=") {
t.Errorf("%s was given %s", c.line, e)
}
}
}
}
func TestSiblingsThatAreNotRepositoriesOrCommitsAreRefused(t *testing.T) {
for name, beside := range map[string]any{
"not a list": "x",
"bad repository": []any{map[string]any{"repository": "../x/y", "commit": baseSHA}},
"bad commit": []any{map[string]any{"repository": "novox/a", "commit": "main"}},
"twice": []any{map[string]any{"repository": "novox/a", "commit": baseSHA}, map[string]any{"repository": "other/a", "commit": baseSHA}},
} {
f := &fakeStep{}
m, states := fetchModule(t, f)
fetchTool(t, m)(map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA, "beside": beside})
if st := states.waitEnd(t); st.State != delivery.StepFailed || len(f.lines()) != 0 {
t.Errorf("%s: the step ended %s after %d commands", name, st.State, len(f.lines()))
}
}
}
// 1288/AC5 (the checker's contract): an import path runs only in the go.mod directory whose module holds it; a
// relative pattern runs in every directory; no pattern is ./... everywhere.
func TestPackagesRunOnlyInTheModuleThatHoldsThem(t *testing.T) {
tree := t.TempDir()
for dir, mod := range map[string]string{".": "example.org/root", "tools": "example.org/root/tools", "other": "example.org/other"} {
os.MkdirAll(filepath.Join(tree, dir), 0o755)
os.WriteFile(filepath.Join(tree, dir, "go.mod"), []byte("module "+mod+"\n\ngo 1.26\n"), 0o644)
}
dirs := []string{".", "other", "tools"}
got, err := packagesByDir(tree, dirs, []string{"example.org/root/a", "example.org/root/tools/b", "example.org/other"})
if err != nil {
t.Fatal(err)
}
want := map[string]string{".": "example.org/root/a", "tools": "example.org/root/tools/b", "other": "example.org/other"}
for d, w := range want {
if strings.Join(got[d], ",") != w {
t.Errorf("%s runs %v, want %s", d, got[d], w)
}
}
got, _ = packagesByDir(tree, dirs, []string{"./x/..."})
if len(got["."]) != 1 || len(got["tools"]) != 1 || len(got["other"]) != 1 {
t.Errorf("a relative pattern = %v", got)
}
got, _ = packagesByDir(tree, dirs, nil)
if got["tools"][0] != "./..." {
t.Errorf("no pattern = %v", got)
}
if _, err := packagesByDir(tree, dirs, []string{"example.org/nowhere/x"}); err == nil || !strings.Contains(err.Error(), "example.org/nowhere/x") {
t.Errorf("a package no module holds: %v", err)
}
}
func sha256Hex(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) }
// 1677/AC (integration level): the integration level's go test is given MESH_CHECK_BESIDE when the workspace has siblings
// beside it, with the store's and the bus's addresses, and is given none when it has not.
func TestTheIntegrationLevelIsGivenTheBesideDirectoryOnlyWhenSiblingsWerePlaced(t *testing.T) {
for _, siblings := range []bool{true, false} {
f := newIntegFake(t, "slow")
m, states := integModule(t, f)
beside := filepath.Join(m.Work, "ws1", "beside")
if siblings {
if err := os.MkdirAll(filepath.Join(beside, "mesh-catalog"), 0o755); err != nil {
t.Fatal(err)
}
}
if _, st := runLevelTool(t, m, states, "integration"); st.State != delivery.StepPassed {
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
}
tested := 0
for _, c := range f.calls {
if !strings.HasPrefix(c.line, "go ") || !strings.Contains(c.line, "-tags integration") {
continue
}
tested++
got := ""
hasNATS := false
for _, e := range c.env {
if strings.HasPrefix(e, "MESH_CHECK_BESIDE=") {
got = strings.TrimPrefix(e, "MESH_CHECK_BESIDE=")
}
hasNATS = hasNATS || strings.HasPrefix(e, "MESH_TEST_NATS=")
}
want := ""
if siblings {
want = beside
}
if got != want || !hasNATS {
t.Errorf("siblings %v: %s was given MESH_CHECK_BESIDE=%q (want %q), MESH_TEST_NATS present %v", siblings, c.line, got, want, hasNATS)
}
}
if tested == 0 {
t.Errorf("siblings %v: no integration go test ran", siblings)
}
}
}
@@ -65,13 +65,23 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
}
switch level {
case LevelUnit:
return m.runLevel(ctx, tree, levelOpts{level: LevelUnit, packages: packages, bounds: Bounds{Run: m.unitBound()}})
return m.runLevel(ctx, tree, levelOpts{level: LevelUnit, packages: packages, bounds: Bounds{Run: m.unitBound()}, env: m.besideEnv(workspace)})
case LevelIntegration:
return m.integration(ctx, workspace, tree, packages)
}
return nil, fmt.Errorf("the level %q is neither unit nor integration", level)
}
// besideEnv is MESH_CHECK_BESIDE, naming the directory build_go_fetch placed the workspace's siblings in (ADR 0484
// clause 4), when it placed any; the tests that read other repositories look there (mesh-sdk go/internal/beside).
func (m *Module) besideEnv(workspace string) []string {
dir := m.besideDir(workspace)
if st, err := os.Stat(dir); err != nil || !st.IsDir() {
return nil
}
return []string{"MESH_CHECK_BESIDE=" + dir}
}
func packagesArg(v any) ([]string, error) {
if v == nil {
return nil, nil
@@ -115,12 +125,16 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR
return nil, err
}
res := &TestResult{Level: o.level, Packages: []PackageResult{}, Setup: o.setup.Seconds(), Store: o.store, Bus: o.bus}
packages := o.packages
if len(packages) == 0 {
packages = []string{"./..."}
byDir, err := packagesByDir(tree, dirs, o.packages)
if err != nil {
return nil, err
}
began := m.clock()
for _, d := range dirs {
packages := byDir[d]
if len(packages) == 0 {
continue // no package asked for lies in this module
}
args := []string{"-C", filepath.Join(tree, d), "test", "-json", "-count=1", "-timeout", testTimeout}
if len(o.tags) > 0 {
args = append(args, "-tags", strings.Join(o.tags, ","))
@@ -141,3 +155,50 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR
res.judge(o.bounds)
return res, nil
}
// packagesByDir says which patterns run in which go.mod directory. A relative pattern (./...) runs in every directory, as
// it always did; an import path, as build_go_digest keys its packages, runs only in the directory whose module holds it
// (the longest module path that is the path or its prefix), as another module's path would fail the step. No pattern
// means ./... everywhere. An import path no module of the tree holds is an error that names it.
func packagesByDir(tree string, dirs, patterns []string) (map[string][]string, error) {
out := map[string][]string{}
if len(patterns) == 0 {
patterns = []string{"./..."}
}
modules := map[string]string{}
for _, d := range dirs {
if raw, err := os.ReadFile(filepath.Join(tree, d, "go.mod")); err == nil {
modules[d] = modulePath(string(raw))
}
}
for _, p := range patterns {
if strings.HasPrefix(p, "./") || p == "." {
for _, d := range dirs {
out[d] = append(out[d], p)
}
continue
}
owner, best := "", -1
for _, d := range dirs {
mp := modules[d]
if mp != "" && (p == mp || strings.HasPrefix(p, mp+"/")) && len(mp) > best {
owner, best = d, len(mp)
}
}
if best < 0 {
return nil, fmt.Errorf("no module of the workspace holds the package %s", p)
}
out[owner] = append(out[owner], p)
}
return out, nil
}
// modulePath is the module path a go.mod declares, "" when it declares none.
func modulePath(gomod string) string {
for _, l := range strings.Split(gomod, "\n") {
if f := strings.Fields(l); len(f) == 2 && f[0] == "module" {
return strings.Trim(f[1], `"`)
}
}
return ""
}
@@ -0,0 +1 @@
package beside
@@ -0,0 +1,22 @@
package beside
import (
"os"
"path/filepath"
"testing"
)
// A test of a repository that reads another one, as the controller's, the node-engine's, the SDK's and the catalogue's do.
func TestReadsTheSiblingBesideIt(t *testing.T) {
root := os.Getenv("MESH_CHECK_BESIDE")
if root == "" {
t.Fatal("the sibling is not beside this check: MESH_CHECK_BESIDE is not set")
}
raw, err := os.ReadFile(filepath.Join(root, "sibling", "data.txt"))
if err != nil {
t.Fatal(err)
}
if string(raw) != "from the sibling\n" {
t.Fatalf("data.txt = %q", raw)
}
}
@@ -0,0 +1,3 @@
module fixture/beside
go 1.26
+1 -1
View File
@@ -3,7 +3,7 @@ module meshbuildgo
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc
github.com/nats-io/nats-server/v2 v2.11.17
)
+4
View File
@@ -1,5 +1,9 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3 h1:OKxU64kHN1yQ/GOa6LwoUwVmiQ74d83WMSZJRFPVm9c=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5 h1:1BiPKRPTYpgex8rSHWT9az0BteOafGWRDzd/Ba6GAcA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -36,6 +36,8 @@ const sdkPackage = "@novox/mesh-sdk"
// WorkspaceAge is how long a workspace is kept.
const WorkspaceAge = 24 * time.Hour
var commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
var repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
// FetchResult is what a passed fetch step says.
@@ -48,6 +50,15 @@ type FetchResult struct {
// Refused are the bundles whose packages were not installed, each with why (a module naming packages beyond the SDK
// with no package-lock.json). The fetch passes; a build of such a bundle fails naming it (issue 1459).
Refused []Refusal `json:"refused"`
// The merge preview (ADR 0484 clause 3), only when the call gave a base, as build_go_fetch answers it: the trunk
// commit, the merge commit of the head onto it, the paths the merge changes and the tree of each module directory.
// A head that does not merge cleanly is an answer: Merge is empty, Conflicts names the files, and the workspace is
// left at the trunk commit with nothing installed.
Base string `json:"base,omitempty"`
Merge string `json:"merge,omitempty"`
Conflicts []string `json:"conflicts,omitempty"`
Changed []string `json:"changed,omitempty"`
Trees map[string]buildstep.ModuleTree `json:"trees,omitempty"`
}
// Refusal is a module a fetch did not install the packages of.
@@ -68,6 +79,7 @@ func (m *Module) fetchTool() StepTool {
"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
"commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
"ref": map[string]any{"type": "string", "description": "refs/pull/<n>/head when only a pull request's head carries the commit"},
"base": map[string]any{"type": "string", "description": "optional trunk commit, 40 hex digits: the head is merged onto it as the forge's merge would, and the answer carries the merge commit, the files in conflict (an answer, the workspace is then left at the trunk commit), the changed paths and the tree of each module directory"},
},
"required": []string{"repository", "commit"},
},
@@ -80,6 +92,10 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
repository, _ := args["repository"].(string)
commit, _ := args["commit"].(string)
ref, _ := args["ref"].(string)
base, _ := args["base"].(string)
if base != "" && !commitOK.MatchString(base) {
return nil, fmt.Errorf("%q is not a trunk commit (40 hex digits)", base)
}
if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") {
return nil, fmt.Errorf("%q is not a repository, <owner>/<name>", repository)
}
@@ -94,15 +110,23 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
gitRun := func(ctx context.Context, env []string, name string, a ...string) forgekey.Ran {
return m.Run(ctx, append(m.baseEnv(), env...), name, a...)
}
remote := "ssh://" + ForgeAlias + "/" + repository + ".git"
if err := buildstep.FetchCommit(ctx, gitRun, tree, remote, commit, ref); err != nil {
remote := m.remote(repository)
if err := buildstep.FetchCommit(ctx, gitRun, tree, remote, commit, ref, m.FetchOptions...); err != nil {
return nil, err
}
res := FetchResult{Workspace: id, Commit: commit, Modules: []string{}, Installed: []string{}, Refused: []Refusal{}}
if base != "" {
if err := m.merge(ctx, gitRun, &res, tree, remote, base, commit); err != nil {
return nil, err
}
if res.Merge == "" {
return res, nil // a conflict: the tree is at the trunk, nothing of the head is installed
}
}
dirs, err := m.typeScriptModules(ctx, tree)
if err != nil {
return nil, err
}
res := FetchResult{Workspace: id, Commit: commit, Modules: []string{}, Installed: []string{}, Refused: []Refusal{}}
for _, d := range dirs {
res.Modules = append(res.Modules, d)
did, refused, err := m.installOwn(ctx, root, filepath.Join(tree, filepath.FromSlash(d)), d)
@@ -48,7 +48,7 @@ func (f *fakeMachine) run(_ context.Context, env []string, name string, args ...
}
} else if name == "git" {
for _, a := range args {
if slices.Contains([]string{"init", "remote", "fetch", "checkout", "rev-parse", "ls-files"}, a) {
if slices.Contains([]string{"init", "remote", "fetch", "checkout", "rev-parse", "ls-files", "merge", "diff"}, a) {
key = "git " + a
break
}
@@ -31,9 +31,13 @@ type Module struct {
Toolchain *Toolchain
// Work is the work directory, outside the home: a workspace is <Work>/<step id>/tree. Run runs the commands of a
// step (a fake in a test). Clock is the time the steps are timed by.
Work string
Run forgekey.Runner
Clock func() time.Time
Work string
Run forgekey.Runner
// RemoteBase is where repositories are fetched from, "ssh://mesh-forge/" when empty; FetchOptions are the options of
// the SDK's fetch and merge. A test points them at a local repository (buildstep.AllowFileRemote).
RemoteBase string
FetchOptions []buildstep.Option
Clock func() time.Time
// Toolchains and Steps are the Put of the states of those names.
Toolchains, Steps buildstep.Put
// ReadSteps and ReadToolchainsState read those two states back for the read tools (issue 1562).
@@ -0,0 +1,35 @@
package main
// The merge preview of a fetch (novox/hq ADR 0484 clause 3; issue 1676), as build_go_fetch answers it. The merge itself
// is the SDK's (buildstep.MergeInto), the one every build module's fetch step makes.
import (
"context"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
// remote is the address a repository is fetched from: the forge over ssh as the account, or RemoteBase when a test set it.
func (m *Module) remote(repository string) string {
base := m.RemoteBase
if base == "" {
base = "ssh://" + ForgeAlias + "/"
}
return base + repository + ".git"
}
// merge makes the merge preview of the head onto base and writes the answer into res.
func (m *Module) merge(ctx context.Context, run forgekey.Runner, res *FetchResult, tree, remote, base, commit string) error {
mr, err := buildstep.MergeInto(ctx, run, tree, remote, base, commit, m.FetchOptions...)
if err != nil {
return err
}
res.Base = mr.Base
res.Conflicts, res.Merge, res.Changed = mr.Conflicts, mr.Merge, mr.Changed
if res.Merge == "" {
return nil
}
res.Trees, err = buildstep.ModuleTrees(ctx, m.Run, m.baseEnv(), tree, res.Merge)
return err
}
@@ -0,0 +1,95 @@
//go:build integration
package main
// The fetch step's merge on real git repositories (issue 1676); preview_test.go proves the commands with a fake runner.
import (
"context"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
func git(t *testing.T, dir string, args ...string) string {
t.Helper()
cmd := exec.Command("git", append([]string{"-C", dir, "-c", "user.name=t", "-c", "user.email=t@example.invalid", "-c", "commit.gpgsign=false"}, args...)...)
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("git %v: %v\n%s", args, err, out)
}
return strings.TrimSpace(string(out))
}
func TestRealFetchMergesAConflictAndAPlainFetch(t *testing.T) {
root := t.TempDir()
bare, work := filepath.Join(root, "forge", "o", "r.git"), filepath.Join(root, "work")
os.MkdirAll(bare, 0o755)
os.MkdirAll(work, 0o755)
git(t, bare, "init", "-q", "--bare")
git(t, work, "init", "-q", "-b", "main")
files := typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6"}}`)
files["a.txt"] = "one\ntwo\nthree\n"
for n, c := range files {
write(t, filepath.Join(work, n), c, 0o644)
}
git(t, work, "add", ".")
git(t, work, "commit", "-q", "-m", "fork")
fork := git(t, work, "rev-parse", "HEAD")
change := func(from, ref string, f map[string]string) string {
git(t, work, "checkout", "-q", "-B", "tmp", from)
for n, c := range f {
write(t, filepath.Join(work, n), c, 0o644)
}
git(t, work, "add", ".")
git(t, work, "commit", "-q", "-m", "c")
git(t, work, "push", "-q", "-f", bare, "HEAD:"+ref)
return git(t, work, "rev-parse", "HEAD")
}
trunk := change(fork, "refs/heads/main", map[string]string{"a.txt": "one\ntwo\nTRUNK\n"})
clean := change(fork, "refs/pull/1/head", map[string]string{"modules/m/extra.ts": "export {}\n"})
clash := change(fork, "refs/pull/2/head", map[string]string{"a.txt": "one\ntwo\nHEAD\n"})
m, _, _ := newMachine(t)
m.Run = forgekey.ExecRunner
m.RemoteBase = "file://" + filepath.Join(root, "forge") + "/"
m.FetchOptions = []buildstep.Option{buildstep.AllowFileRemote()}
out, err := m.fetch(context.Background(), map[string]any{"repository": "o/r", "commit": clean, "ref": "refs/pull/1/head", "base": trunk})
if err != nil {
t.Fatal(err)
}
r := out.(FetchResult)
tree := filepath.Join(m.Work, r.Workspace, "tree")
if r.Merge == "" || strings.Join(r.Changed, ",") != "modules/m/extra.ts" || len(r.Conflicts) != 0 || len(r.Modules) != 1 {
t.Fatalf("clean: %+v", r)
}
if want := git(t, tree, "rev-parse", r.Merge+":modules/m"); r.Trees["modules/m"].Tree != want {
t.Errorf("tree = %s, want %s", r.Trees["modules/m"].Tree, want)
}
if got := git(t, tree, "rev-parse", "HEAD"); got != r.Merge {
t.Errorf("the workspace is at %s, want the merge %s", got, r.Merge)
}
out, err = m.fetch(context.Background(), map[string]any{"repository": "o/r", "commit": clash, "ref": "refs/pull/2/head", "base": trunk})
if err != nil {
t.Fatalf("a conflict failed the step: %v", err)
}
r = out.(FetchResult)
if r.Merge != "" || strings.Join(r.Conflicts, ",") != "a.txt" {
t.Fatalf("conflict: %+v", r)
}
if got := git(t, filepath.Join(m.Work, r.Workspace, "tree"), "rev-parse", "HEAD"); got != trunk {
t.Errorf("after a conflict the workspace is at %s, want the trunk %s", got, trunk)
}
out, err = m.fetch(context.Background(), map[string]any{"repository": "o/r", "commit": clean, "ref": "refs/pull/1/head"})
if err != nil || out.(FetchResult).Merge != "" || out.(FetchResult).Trees != nil {
t.Errorf("no base: %+v %v", out, err)
}
}
@@ -0,0 +1,87 @@
package main
import (
"context"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
const baseSHA = "abcdef0123456789abcdef0123456789abcdef01"
func commandLines(f *fakeMachine) string {
var out []string
for _, c := range f.calls {
out = append(out, joined(c))
}
return strings.Join(out, "\n")
}
// 1676/AC1: with a base the fetch merges the head onto it through the SDK's merge (the check merge's identity and flags,
// hooks off), and answers the base, the merge commit, the changed paths and the tree, manifest and locks of each module
// directory; the TypeScript modules are installed from the merged tree.
func TestFetchWithABaseMergesAndAnswersThePreview(t *testing.T) {
m, f := fetchMachine(t, typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6"}}`), nil)
f.answer["git diff"] = forgekey.Ran{Stdout: "modules/m/package.json\n"}
args := map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA, "base": baseSHA}
out, err := m.fetch(context.Background(), args)
if err != nil {
t.Fatal(err)
}
r := out.(FetchResult)
if r.Base != baseSHA || r.Merge != commitSHA || strings.Join(r.Changed, ",") != "modules/m/package.json" || len(r.Conflicts) != 0 {
t.Errorf("result = %+v", r)
}
if mt, ok := r.Trees["modules/m"]; !ok || mt.Tree != commitSHA || mt.Manifest == "" {
t.Errorf("trees = %+v", r.Trees)
}
if len(r.Modules) != 1 || r.Modules[0] != "modules/m" {
t.Errorf("modules = %v", r.Modules)
}
for _, want := range []string{"fetch --no-tags origin " + baseSHA, "core.hooksPath=/dev/null", "merge --no-ff --no-edit --no-verify"} {
if !strings.Contains(commandLines(f), want) {
t.Errorf("git was not asked %q:\n%s", want, commandLines(f))
}
}
}
// 1676/AC2: a conflict is an answer: the step passes with the files named, no merge commit, and nothing installed.
func TestFetchWithAConflictAnswersTheFilesAndInstallsNothing(t *testing.T) {
files := typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6","lodash":"^4"}}`)
files["modules/m/package-lock.json"] = `{"lockfileVersion":3}`
m, f := fetchMachine(t, files, nil)
f.answer["git merge"] = forgekey.Ran{Status: 1, Stderr: "CONFLICT\n"}
f.answer["git diff"] = forgekey.Ran{Stdout: "a.ts\nsub/b.ts\n"}
out, err := m.fetch(context.Background(), map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA, "base": baseSHA})
if err != nil {
t.Fatalf("a conflict failed the step: %v", err)
}
r := out.(FetchResult)
if r.Merge != "" || strings.Join(r.Conflicts, ",") != "a.ts,sub/b.ts" || len(r.Changed) != 0 || len(npmCalls(f)) != 0 {
t.Errorf("result = %+v, npm calls %d", r, len(npmCalls(f)))
}
if !strings.Contains(commandLines(f), "merge --abort") {
t.Errorf("the merge was not undone:\n%s", commandLines(f))
}
}
// 1676/AC3: no base is today's fetch: no merge command, and none of the preview in the answer.
func TestFetchWithoutABaseMakesNoMerge(t *testing.T) {
m, f := fetchMachine(t, typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6"}}`), nil)
out, err := m.fetch(context.Background(), fetchArgs)
if err != nil {
t.Fatal(err)
}
r := out.(FetchResult)
if strings.Contains(commandLines(f), "merge") || strings.Contains(commandLines(f), "unshallow") || r.Base != "" || r.Merge != "" || r.Trees != nil {
t.Errorf("a fetch without a base merged: %+v\n%s", r, commandLines(f))
}
}
func TestABaseThatIsNotACommitIsRefusedBeforeAnythingRuns(t *testing.T) {
m, f := fetchMachine(t, nil, nil)
if _, err := m.fetch(context.Background(), map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA, "base": "main"}); err == nil || len(f.calls) != 0 {
t.Errorf("err = %v after %d commands", err, len(f.calls))
}
}
+1 -1
View File
@@ -2,4 +2,4 @@ module meshbuildnode
go 1.26.0
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc
+4
View File
@@ -1,2 +1,6 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3 h1:OKxU64kHN1yQ/GOa6LwoUwVmiQ74d83WMSZJRFPVm9c=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5 h1:1BiPKRPTYpgex8rSHWT9az0BteOafGWRDzd/Ba6GAcA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
@@ -703,23 +703,3 @@ func TestNatsResticsFilesFillFromTheBackupPlacement(t *testing.T) {
b.assign(t, "restic", plan("# a\n/srv/a\n", "# a\nstate\n"))
eventually(t, "a module taken away refills backups.conf", func() bool { return e.read(t, backups) == head+"# a\n/srv/a\n" })
}
// defaults writes the defaults a module's manifest declares as mesh-config resolves them (source default), for a
// moved module whose templates read config values (issue 1780).
func (b *aBus) defaults(t *testing.T, module string, manifest []byte) {
t.Helper()
var m struct {
Config map[string]struct {
Default json.RawMessage `json:"default"`
} `json:"config"`
}
if err := json.Unmarshal(manifest, &m); err != nil {
t.Fatal(err)
}
for key, d := range m.Config {
if len(d.Default) > 0 && string(d.Default) != "null" {
b.put(t, ConfigBucket, node+"."+module+"."+key, fmt.Sprintf(`{"node":%q,"module":%q,"key":%q,"value":%s,`+
`"source":"default"}`, node, module, key, d.Default))
}
}
}
@@ -158,7 +158,6 @@ func TestNatsTheEnvironmentOfTheCataloguesModulesIsWhatTheOldFormComposed(t *tes
if !manifest.Moved || len(manifest.Items) == 0 {
t.Fatalf("%s is not a moved contributor of items", m)
}
b.defaults(t, m, read(m+"/module.json"))
texts := map[string]string{}
var planned []PlannedItem
for _, it := range manifest.Items {
@@ -63,7 +63,6 @@ func TestNatsTheSessionFilesFilledFromTheirContributorsItemsAreWhatTheOldFormCom
if !c.Moved || len(c.Shell) != 0 {
t.Fatalf("%s is not moved or still holds shell entries", m)
}
b.defaults(t, m, []byte(read(m+"/module.json")))
texts := map[string]string{}
var planned []PlannedItem
for _, it := range c.Items {
+5 -36
View File
@@ -256,22 +256,11 @@ export class MinioClient {
if (argv.some((a) => a.includes(this.rootPassword))) {
throw new Error("refused to run mc: its command line would carry the root password (novox/hq issue 282)");
}
const mcHostValue = mcHost(this.baseUrl, this.rootUser, this.rootPassword);
try {
const { stdout } = await execFileAsync(this.mcBin, argv, {
timeout: 30_000,
env: { ...process.env, MC_HOST_mesh: mcHostValue },
});
return stdout.trim();
} catch (err) {
// A fresh error with only the redacted message: execFile's own carries the whole command line, and
// its cmd, stdout and stderr properties would carry it on (issue 1766).
const known = [mcHostValue, this.rootPassword, encodeURIComponent(this.rootPassword), ...secretArgs(argv)];
const e = err as { stderr?: unknown; message?: string };
const reason = typeof e.stderr === "string" && e.stderr.trim() !== "" ? e.stderr.trim() : (e.message ?? String(err));
const shown = argv.filter((a) => a !== this.mcConfigDir).join(" ");
throw new Error(redactSecrets(`mc ${shown} failed: ${reason}`, known));
}
const { stdout } = await execFileAsync(this.mcBin, argv, {
timeout: 30_000,
env: { ...process.env, MC_HOST_mesh: mcHost(this.baseUrl, this.rootUser, this.rootPassword) },
});
return stdout.trim();
}
// --- SigV4 request plumbing ---------------------------------------------
@@ -371,26 +360,6 @@ function uriEncode(str: string, encodeSlash: boolean): string {
return out;
}
/** The values that follow --secret-key and --access-key in an argument list. */
function secretArgs(argv: string[]): string[] {
const out: string[] = [];
argv.forEach((a, i) => { if ((a === "--secret-key" || a === "--access-key") && i + 1 < argv.length) out.push(argv[i + 1]); });
return out;
}
/**
* The one redaction every `mc` failure goes through (issue 1766): the token after --secret-key or
* --access-key (as a separate word or after "="), any MC_HOST_ value, and each known secret, wherever they
* appear in the text, become "[redacted]".
*/
export function redactSecrets(text: string, known: string[] = []): string {
let out = text
.replace(/(--(?:secret|access)-key(?:=|\s+))('[^']*'|"[^"]*"|\S+)/g, "$1[redacted]")
.replace(/(MC_HOST_\w+=)\S+/g, "$1[redacted]");
for (const k of known) if (k !== "") out = out.split(k).join("[redacted]");
return out;
}
/**
* The MC_HOST_<alias> URL `mc` reads an alias from: the endpoint with the credentials in its userinfo,
* each percent-encoded so a reserved character in either cannot end it early.
+1 -7
View File
@@ -36,9 +36,6 @@ export function linkPolicy(bucket: string): string {
});
}
/** A link key's secret: 40 hex characters, minio's limit for a secret key (issue 1766). */
const LINK_SECRET = /^[0-9a-f]{40}$/;
export function linkSecretPath(dir: string, bucket: string): string {
return join(dir, `${bucket}.secret`);
}
@@ -50,12 +47,9 @@ export async function ensureLinkKey(admin: LinkAdmin, dir: string, bucket: strin
const path = linkSecretPath(dir, bucket);
mkdirSync(dir, { recursive: true, mode: 0o700 });
let secret = existsSync(path) ? readFileSync(path, "utf8").trim() : "";
// A stored secret of the wrong shape (an earlier version made 64 characters; minio takes at most 40) is
// replaced, and its key made again (issue 1766).
if (!LINK_SECRET.test(secret)) secret = "";
if (secret !== "" && (await admin.accessKeyExists(id))) return;
if (secret === "") {
secret = randomBytes(20).toString("hex");
secret = randomBytes(32).toString("hex");
writeFileSync(path, secret + "\n", { mode: 0o600 });
chmodSync(path, 0o600);
}
-55
View File
@@ -115,58 +115,3 @@ test("the link key id is the one the object-store tool derives (cmd/object-store
// 1516/AC1
assert.equal(linkKeyId("brother"), "link0b4bd77cec705cf5");
});
// Issue 1766: a link key's secret is minio's length, and no mc failure carries an argument's secret.
import { chmodSync, writeFileSync, mkdirSync } from "node:fs";
import { MinioClient } from "../client.ts";
test("a link key's secret is 40 characters, and a stored 64-character secret is replaced", async () => {
// 1766/AC1
const { minio, dir } = setup();
await ensureLinkKey(minio, dir, "fresh");
assert.equal(minio.keys.get(linkKeyId("fresh"))!.secret.length, 40);
assert.match(minio.keys.get(linkKeyId("fresh"))!.secret, /^[0-9a-f]{40}$/);
mkdirSync(dir, { recursive: true });
const old = "ab".repeat(32);
writeFileSync(linkSecretPath(dir, "old"), old + "\n");
await ensureLinkKey(minio, dir, "old");
const made = minio.keys.get(linkKeyId("old"))!.secret;
assert.equal(made.length, 40);
assert.notEqual(made, old);
assert.equal(readFileSync(linkSecretPath(dir, "old"), "utf8").trim(), made);
});
function failingMc(): { client: MinioClient; root: string } {
const d = mkdtempSync(join(process.env.TMPDIR ?? tmpdir(), "minio-mc-"));
const bin = join(d, "mc");
// Says its arguments and its alias on stderr, then fails, the way a refused command does.
writeFileSync(bin, '#!/bin/sh\necho "args: $*" >&2\necho "env: MC_HOST_mesh=$MC_HOST_mesh" >&2\nexit 1\n');
chmodSync(bin, 0o755);
const root = "RootPass-s3cret/ok";
const client = new MinioClient({ endpoint: "http://127.0.0.1:9000", rootUser: "root", rootPassword: root, region: "us-east-1", mcBin: bin, mcConfigDir: d });
return { client, root };
}
test("an mc failure's message holds no --secret-key value and no MC_HOST value", async () => {
// 1766/AC2
const { client, root } = failingMc();
const secret = "SuperSecretValue0123456789";
const err: any = await client.createAccessKey("b", "ACCESSKEYID1234567", secret).then(() => null, (e) => e);
assert.ok(err, "the failure is raised");
for (const text of [String(err.message), String(err.stack ?? ""), JSON.stringify(Object.getOwnPropertyNames(err).map((k) => err[k]))]) {
assert.ok(!text.includes(secret), "no secret key");
assert.ok(!text.includes("ACCESSKEYID1234567"), "no access key");
assert.ok(!text.includes(root) && !text.includes(encodeURIComponent(root)), "no root password");
assert.ok(!text.includes("root:"), "no MC_HOST value");
}
assert.match(err.message, /\[redacted\]/);
});
test("a consumer key's create passes the minted secret unchanged", async () => {
// 1766/AC3
const s = setup();
await s.h.create(provision);
assert.equal(s.minio.keys.get("brother-key")!.secret, "consumer-secret");
});