mesh-delivery/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go

package bridge

import (
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"os"
	"reflect"
	"sort"
	"strings"
	"sync"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/delivery"
)

// The bytes the forge answers in these tests are two real files of this repository, read from the working tree: the
// module.json and the go.sum of modules/mesh-delivery-forge-bridge (two directories up). The forge's own answer shape
// is mesh-catalog modules/gitea tools/index.ts gitea_get_file; the bridge sees only the bytes (issue 1448).
func real(t *testing.T, name string) []byte {
	t.Helper()
	b, err := os.ReadFile("../../" + name)
	if err != nil {
		t.Fatal(err)
	}
	return b
}

func hexSum(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) }

type fileCall struct{ owner, repo, path, ref string }

// forgeFiles answers as the forge does for a path: its bytes, ErrNotFound for a path in none, the error in errs.
type forgeFiles struct {
	have  map[string][]byte
	errs  map[string]error
	mu    sync.Mutex
	calls []fileCall
}

func (f *forgeFiles) get(owner, repo, path, ref string) ([]byte, error) {
	f.mu.Lock()
	f.calls = append(f.calls, fileCall{owner, repo, path, ref})
	f.mu.Unlock()
	if err, ok := f.errs[path]; ok {
		return nil, err
	}
	if b, ok := f.have[path]; ok {
		return b, nil
	}
	return nil, ErrNotFound
}

// anyFiles is a forge that has a module.json at every path and no lock file, for tests that are not about the files.
func anyFiles(owner, repo, path, ref string) ([]byte, error) {
	if strings.HasSuffix(path, "module.json") {
		return []byte("{}"), nil
	}
	return nil, ErrNotFound
}

func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1
	manifest, sum := real(t, "module.json"), real(t, "go.sum")
	forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest, "modules/a/go.sum": sum, "modules/b/module.json": []byte(`{"module":"b"}`)}}
	var calls []call
	msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1", "modules/b": "t2"}, nil), forge.get)
	if err != nil {
		t.Fatal(err)
	}
	f, err := delivery.DecodeForge(msg.Data)
	if err != nil {
		t.Fatal(err)
	}
	if want := map[string]string{"modules/a": string(manifest), "modules/b": `{"module":"b"}`}; !reflect.DeepEqual(f.Manifests, want) {
		t.Errorf("manifests = %v", f.Manifests)
	}
	if want := map[string]map[string]string{"modules/a": {"go.sum": hexSum(sum)}}; !reflect.DeepEqual(f.Locks, want) {
		t.Errorf("locks = %v, want %v", f.Locks, want)
	}
}

func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2
	manifest := real(t, "module.json")
	forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest}}
	msg, err := Convert(updatedSubject, updated(nil), nil, forge.get)
	if err != nil {
		t.Fatal(err)
	}
	f, _ := delivery.DecodeForge(msg.Data)
	if !reflect.DeepEqual(f.Manifests, map[string]string{"modules/a": string(manifest)}) || len(f.Locks) != 0 {
		t.Errorf("manifests %v locks %v", f.Manifests, f.Locks)
	}
	if len(forge.calls) == 0 || forge.calls[0].ref != head {
		t.Errorf("calls %v, want the head %s", forge.calls, head)
	}
	// Dirs not known: neither field, and the forge is not asked.
	forge = &forgeFiles{}
	msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs": nil, "module_dirs_said": nil}), nil, forge.get)
	if err != nil {
		t.Fatal(err)
	}
	f, _ = delivery.DecodeForge(msg.Data)
	if f.Manifests != nil || f.Locks != nil || len(forge.calls) != 0 {
		t.Errorf("manifests %v locks %v calls %v", f.Manifests, f.Locks, forge.calls)
	}
}

func TestAForgeErrorOrAMissingManifestIsRetryAndALockNotFoundIsLeftOut(t *testing.T) { // 1448/AC3
	boom := errors.New("Gitea API /repos/novox/mesh-delivery/contents/modules/a/go.sum?ref=x: 502 bad gateway")
	trees := func() Trees { var c []call; return treesOf(&c, "r0", map[string]string{"modules/a": "t1"}, nil) }
	for name, forge := range map[string]*forgeFiles{
		"a manifest error":     {errs: map[string]error{"modules/a/module.json": boom}},
		"a manifest not found": {},
		"a lock file error":    {have: map[string][]byte{"modules/a/module.json": []byte("{}")}, errs: map[string]error{"modules/a/go.sum": boom}},
		"a second lock error":  {have: map[string][]byte{"modules/a/module.json": []byte("{}")}, errs: map[string]error{"modules/a/package-lock.json": boom}},
	} {
		msg, err := Convert(mergedSubject, mergedBody(nil), trees(), forge.get)
		if !errors.Is(err, ErrRetry) || msg.Data != nil {
			t.Errorf("%s: err = %v data = %s, want ErrRetry and no message", name, err, msg.Data)
		}
		_, err = Convert(updatedSubject, updated(nil), nil, forge.get)
		if !errors.Is(err, ErrRetry) {
			t.Errorf("%s (pull): err = %v, want ErrRetry", name, err)
		}
	}
	forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": []byte("{}")}}
	msg, err := Convert(mergedSubject, mergedBody(nil), trees(), forge.get)
	if err != nil {
		t.Fatal(err)
	}
	if f, _ := delivery.DecodeForge(msg.Data); f.Locks != nil {
		t.Errorf("locks = %v, want none for lock files the forge does not have", f.Locks)
	}
}

func TestTheFilesAreAskedAtTheMergeCommitByOwnerRepoPathAndRef(t *testing.T) { // 1448/AC4 (the call's arguments; the manifest is in manifest_test)
	forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": []byte("{}")}}
	var calls []call
	if _, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get); err != nil {
		t.Fatal(err)
	}
	want := []fileCall{{"novox", "mesh-delivery", "modules/a/module.json", merge}, {"novox", "mesh-delivery", "modules/a/go.sum", merge},
		{"novox", "mesh-delivery", "modules/a/package-lock.json", merge}}
	sort.Slice(forge.calls, func(i, j int) bool { return forge.calls[i].path < forge.calls[j].path }) // they are asked for concurrently
	sort.Slice(want, func(i, j int) bool { return want[i].path < want[j].path })
	if !reflect.DeepEqual(forge.calls, want) {
		t.Errorf("calls = %v, want %v", forge.calls, want)
	}
}

// slowForge answers like a forge that takes d a call: a module.json of the directory's name everywhere, a go.sum in
// the directories whose number is even and in the root, and counts the calls in flight.
type slowForge struct {
	d                          time.Duration
	mu                         sync.Mutex
	inFlight, most, started, n int
	failOn                     string
	finished                   int
}

func (f *slowForge) get(_, _, path, _ string) ([]byte, error) {
	f.mu.Lock()
	f.inFlight++
	f.started++
	if f.inFlight > f.most {
		f.most = f.inFlight
	}
	f.mu.Unlock()
	time.Sleep(f.d)
	defer func() { f.mu.Lock(); f.inFlight--; f.finished++; f.mu.Unlock() }()
	switch {
	case path == f.failOn:
		return nil, errors.New("Gitea API " + path + ": 502 bad gateway")
	case strings.HasSuffix(path, "module.json"):
		return []byte(`{"module":"` + path + `"}`), nil
	case strings.HasSuffix(path, "go.sum") && (path == "go.sum" || path[len(path)-8]%2 == 0):
		return []byte("sum of " + path), nil
	}
	return nil, ErrNotFound
}

func manyDirs(n int) []string {
	dirs := []string{""}
	for i := 1; i < n; i++ {
		dirs = append(dirs, fmt.Sprintf("modules/m%02d", i))
	}
	return dirs
}

func TestFortyDirectoriesAreReadEightAtATimeWithTheResultOfOneAtATime(t *testing.T) { // 1458/AC1
	dirs := manyDirs(40)
	slow := &slowForge{d: 10 * time.Millisecond}
	manifests, locks, err := inputsOf("novox", "mesh-catalog", merge, dirs, slow.get)
	if err != nil {
		t.Fatal(err)
	}
	if slow.most > MaxReadsInFlight || slow.most < 2 {
		t.Errorf("at most %d calls were in flight, want more than one and at most %d", slow.most, MaxReadsInFlight)
	}
	if slow.started != 40*3 {
		t.Errorf("%d calls, want 120", slow.started)
	}
	// The same read one file at a time.
	wantM, wantL := map[string]string{}, map[string]map[string]string{}
	one := &slowForge{}
	for _, dir := range dirs {
		m, l, err := inputsOfOne(dir, one.get)
		if err != nil {
			t.Fatal(err)
		}
		wantM[dir] = m
		if l != nil {
			wantL[dir] = l
		}
	}
	if !reflect.DeepEqual(manifests, wantM) || !reflect.DeepEqual(locks, wantL) {
		t.Errorf("the concurrent read differs from a one-at-a-time read: %d manifests %d locks, want %d and %d", len(manifests), len(locks), len(wantM), len(wantL))
	}
	if len(locks) == 0 || len(locks) == 40 {
		t.Errorf("%d directories with locks, the fixture should have some and not all", len(locks))
	}
}

// inputsOfOne reads one directory with a single call at a time, as the bridge did before issue 1458.
func inputsOfOne(dir string, files Files) (string, map[string]string, error) {
	at := func(name string) string {
		if dir == "" {
			return name
		}
		return dir + "/" + name
	}
	raw, err := files("o", "r", at("module.json"), "x")
	if err != nil {
		return "", nil, err
	}
	var locks map[string]string
	for _, name := range LockFiles {
		b, err := files("o", "r", at(name), "x")
		if errors.Is(err, ErrNotFound) {
			continue
		}
		if err != nil {
			return "", nil, err
		}
		if locks == nil {
			locks = map[string]string{}
		}
		s := sha256.Sum256(b)
		locks[name] = hex.EncodeToString(s[:])
	}
	return string(raw), locks, nil
}

func TestOneFailingFileIsRetryAndEveryCallHasReturnedBeforeTheReadDoes(t *testing.T) { // 1458/AC2
	dirs := manyDirs(40)
	slow := &slowForge{d: 10 * time.Millisecond, failOn: "modules/m05/go.sum"}
	_, _, err := inputsOf("novox", "mesh-catalog", merge, dirs, slow.get)
	if err == nil || !strings.Contains(err.Error(), "modules/m05/go.sum") || !strings.Contains(err.Error(), "502") {
		t.Fatalf("err = %v, want the failing file named", err)
	}
	slow.mu.Lock()
	defer slow.mu.Unlock()
	if slow.inFlight != 0 || slow.finished != slow.started {
		t.Errorf("%d calls in flight, %d of %d finished, when the read returned", slow.inFlight, slow.finished, slow.started)
	}
	if slow.started >= 40*3 {
		t.Errorf("%d calls were made, want the read to stop asking after the failure", slow.started)
	}
	// Through Convert it is ErrRetry.
	var calls []call
	dirsMap := map[string]string{}
	for _, d := range dirs[1:] {
		dirsMap[d] = "t"
	}
	slow2 := &slowForge{failOn: "modules/m05/module.json"}
	if _, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", dirsMap, nil), slow2.get); !errors.Is(err, ErrRetry) {
		t.Errorf("err = %v, want ErrRetry", err)
	}
}