package bridge import ( "crypto/sha256" "encoding/hex" "errors" "fmt" "os" "reflect" "sort" "strings" "sync" "testing" "time" "git.novox.be/novox/mesh-sdk/go/delivery" ) // The bytes the forge answers in these tests are two real files of this repository, read from the working tree: the // module.json and the go.sum of modules/mesh-delivery-forge-bridge (two directories up). The forge's own answer shape // is mesh-catalog modules/gitea tools/index.ts gitea_get_file; the bridge sees only the bytes (issue 1448). func real(t *testing.T, name string) []byte { t.Helper() b, err := os.ReadFile("../../" + name) if err != nil { t.Fatal(err) } return b } func hexSum(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) } type fileCall struct{ owner, repo, path, ref string } // forgeFiles answers as the forge does for a path: its bytes, ErrNotFound for a path in none, the error in errs. type forgeFiles struct { have map[string][]byte errs map[string]error mu sync.Mutex calls []fileCall } func (f *forgeFiles) get(owner, repo, path, ref string) ([]byte, error) { f.mu.Lock() f.calls = append(f.calls, fileCall{owner, repo, path, ref}) f.mu.Unlock() if err, ok := f.errs[path]; ok { return nil, err } if b, ok := f.have[path]; ok { return b, nil } return nil, ErrNotFound } // anyFiles is a forge that has a module.json at every path and no lock file, for tests that are not about the files. func anyFiles(owner, repo, path, ref string) ([]byte, error) { if strings.HasSuffix(path, "module.json") { return []byte("{}"), nil } return nil, ErrNotFound } func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1 manifest, sum := real(t, "module.json"), real(t, "go.sum") forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest, "modules/a/go.sum": sum, "modules/b/module.json": []byte(`{"module":"b"}`)}} var calls []call msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1", "modules/b": "t2"}, nil), forge.get) if err != nil { t.Fatal(err) } f, err := delivery.DecodeForge(msg.Data) if err != nil { t.Fatal(err) } if want := map[string]string{"modules/a": string(manifest), "modules/b": `{"module":"b"}`}; !reflect.DeepEqual(f.Manifests, want) { t.Errorf("manifests = %v", f.Manifests) } if want := map[string]map[string]string{"modules/a": {"go.sum": hexSum(sum)}}; !reflect.DeepEqual(f.Locks, want) { t.Errorf("locks = %v, want %v", f.Locks, want) } } func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2 manifest := real(t, "module.json") forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest}} msg, err := Convert(updatedSubject, updated(nil), nil, forge.get) if err != nil { t.Fatal(err) } f, _ := delivery.DecodeForge(msg.Data) if !reflect.DeepEqual(f.Manifests, map[string]string{"modules/a": string(manifest)}) || len(f.Locks) != 0 { t.Errorf("manifests %v locks %v", f.Manifests, f.Locks) } if len(forge.calls) == 0 || forge.calls[0].ref != head { t.Errorf("calls %v, want the head %s", forge.calls, head) } // Dirs not known: neither field, and the forge is not asked. forge = &forgeFiles{} msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs": nil, "module_dirs_said": nil}), nil, forge.get) if err != nil { t.Fatal(err) } f, _ = delivery.DecodeForge(msg.Data) if f.Manifests != nil || f.Locks != nil || len(forge.calls) != 0 { t.Errorf("manifests %v locks %v calls %v", f.Manifests, f.Locks, forge.calls) } } func TestAForgeErrorOrAMissingManifestIsRetryAndALockNotFoundIsLeftOut(t *testing.T) { // 1448/AC3 boom := errors.New("Gitea API /repos/novox/mesh-delivery/contents/modules/a/go.sum?ref=x: 502 bad gateway") trees := func() Trees { var c []call; return treesOf(&c, "r0", map[string]string{"modules/a": "t1"}, nil) } for name, forge := range map[string]*forgeFiles{ "a manifest error": {errs: map[string]error{"modules/a/module.json": boom}}, "a manifest not found": {}, "a lock file error": {have: map[string][]byte{"modules/a/module.json": []byte("{}")}, errs: map[string]error{"modules/a/go.sum": boom}}, "a second lock error": {have: map[string][]byte{"modules/a/module.json": []byte("{}")}, errs: map[string]error{"modules/a/package-lock.json": boom}}, } { msg, err := Convert(mergedSubject, mergedBody(nil), trees(), forge.get) if !errors.Is(err, ErrRetry) || msg.Data != nil { t.Errorf("%s: err = %v data = %s, want ErrRetry and no message", name, err, msg.Data) } _, err = Convert(updatedSubject, updated(nil), nil, forge.get) if !errors.Is(err, ErrRetry) { t.Errorf("%s (pull): err = %v, want ErrRetry", name, err) } } forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": []byte("{}")}} msg, err := Convert(mergedSubject, mergedBody(nil), trees(), forge.get) if err != nil { t.Fatal(err) } if f, _ := delivery.DecodeForge(msg.Data); f.Locks != nil { t.Errorf("locks = %v, want none for lock files the forge does not have", f.Locks) } } func TestTheFilesAreAskedAtTheMergeCommitByOwnerRepoPathAndRef(t *testing.T) { // 1448/AC4 (the call's arguments; the manifest is in manifest_test) forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": []byte("{}")}} var calls []call if _, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get); err != nil { t.Fatal(err) } want := []fileCall{{"novox", "mesh-delivery", "modules/a/module.json", merge}, {"novox", "mesh-delivery", "modules/a/go.sum", merge}, {"novox", "mesh-delivery", "modules/a/package-lock.json", merge}} sort.Slice(forge.calls, func(i, j int) bool { return forge.calls[i].path < forge.calls[j].path }) // they are asked for concurrently sort.Slice(want, func(i, j int) bool { return want[i].path < want[j].path }) if !reflect.DeepEqual(forge.calls, want) { t.Errorf("calls = %v, want %v", forge.calls, want) } } // slowForge answers like a forge that takes d a call: a module.json of the directory's name everywhere, a go.sum in // the directories whose number is even and in the root, and counts the calls in flight. type slowForge struct { d time.Duration mu sync.Mutex inFlight, most, started, n int failOn string finished int } func (f *slowForge) get(_, _, path, _ string) ([]byte, error) { f.mu.Lock() f.inFlight++ f.started++ if f.inFlight > f.most { f.most = f.inFlight } f.mu.Unlock() time.Sleep(f.d) defer func() { f.mu.Lock(); f.inFlight--; f.finished++; f.mu.Unlock() }() switch { case path == f.failOn: return nil, errors.New("Gitea API " + path + ": 502 bad gateway") case strings.HasSuffix(path, "module.json"): return []byte(`{"module":"` + path + `"}`), nil case strings.HasSuffix(path, "go.sum") && (path == "go.sum" || path[len(path)-8]%2 == 0): return []byte("sum of " + path), nil } return nil, ErrNotFound } func manyDirs(n int) []string { dirs := []string{""} for i := 1; i < n; i++ { dirs = append(dirs, fmt.Sprintf("modules/m%02d", i)) } return dirs } func TestFortyDirectoriesAreReadEightAtATimeWithTheResultOfOneAtATime(t *testing.T) { // 1458/AC1 dirs := manyDirs(40) slow := &slowForge{d: 10 * time.Millisecond} manifests, locks, err := inputsOf("novox", "mesh-catalog", merge, dirs, slow.get) if err != nil { t.Fatal(err) } if slow.most > MaxReadsInFlight || slow.most < 2 { t.Errorf("at most %d calls were in flight, want more than one and at most %d", slow.most, MaxReadsInFlight) } if slow.started != 40*3 { t.Errorf("%d calls, want 120", slow.started) } // The same read one file at a time. wantM, wantL := map[string]string{}, map[string]map[string]string{} one := &slowForge{} for _, dir := range dirs { m, l, err := inputsOfOne(dir, one.get) if err != nil { t.Fatal(err) } wantM[dir] = m if l != nil { wantL[dir] = l } } if !reflect.DeepEqual(manifests, wantM) || !reflect.DeepEqual(locks, wantL) { t.Errorf("the concurrent read differs from a one-at-a-time read: %d manifests %d locks, want %d and %d", len(manifests), len(locks), len(wantM), len(wantL)) } if len(locks) == 0 || len(locks) == 40 { t.Errorf("%d directories with locks, the fixture should have some and not all", len(locks)) } } // inputsOfOne reads one directory with a single call at a time, as the bridge did before issue 1458. func inputsOfOne(dir string, files Files) (string, map[string]string, error) { at := func(name string) string { if dir == "" { return name } return dir + "/" + name } raw, err := files("o", "r", at("module.json"), "x") if err != nil { return "", nil, err } var locks map[string]string for _, name := range LockFiles { b, err := files("o", "r", at(name), "x") if errors.Is(err, ErrNotFound) { continue } if err != nil { return "", nil, err } if locks == nil { locks = map[string]string{} } s := sha256.Sum256(b) locks[name] = hex.EncodeToString(s[:]) } return string(raw), locks, nil } func TestOneFailingFileIsRetryAndEveryCallHasReturnedBeforeTheReadDoes(t *testing.T) { // 1458/AC2 dirs := manyDirs(40) slow := &slowForge{d: 10 * time.Millisecond, failOn: "modules/m05/go.sum"} _, _, err := inputsOf("novox", "mesh-catalog", merge, dirs, slow.get) if err == nil || !strings.Contains(err.Error(), "modules/m05/go.sum") || !strings.Contains(err.Error(), "502") { t.Fatalf("err = %v, want the failing file named", err) } slow.mu.Lock() defer slow.mu.Unlock() if slow.inFlight != 0 || slow.finished != slow.started { t.Errorf("%d calls in flight, %d of %d finished, when the read returned", slow.inFlight, slow.finished, slow.started) } if slow.started >= 40*3 { t.Errorf("%d calls were made, want the read to stop asking after the failure", slow.started) } // Through Convert it is ErrRetry. var calls []call dirsMap := map[string]string{} for _, d := range dirs[1:] { dirsMap[d] = "t" } slow2 := &slowForge{failOn: "modules/m05/module.json"} if _, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", dirsMap, nil), slow2.get); !errors.Is(err, ErrRetry) { t.Errorf("err = %v, want ErrRetry", err) } }