Files
mesh-delivery/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go
T
jschoubben b201ccfa72 Indexer: hold a module whose templates need a reading, and raise a hold past its bound (ADR 0632 §2; issue 1966)
docker's daemon file names the image seat's reach while its manifest stays, so
the hold read from the manifest alone let an engine before the reading lose the
file. The forge bridge reads each manifest's templates at the merge and carries
the module's needs whole; the indexer takes them, keeps them past the judge, and
holds any changed entry, not only a new manifest digest.

A hold standing past 30 minutes is the finding manifest-held, its bound a key
of the new state hold-bounds that expires; its expiry and the start weigh the
holds again, so a restart loses no bound. A hold moved by hand names the nodes
it strands, in its answer and its hand act. The hold test is a case of the
manifests test, not a function of its own.

Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn
2026-11-04 21:39:03 +01:00

413 lines
17 KiB
Go

package bridge
import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"os"
"reflect"
"slices"
"sort"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The bytes the forge answers in these tests are two real files of this repository, read from the working tree: the
// module.json and the go.sum of modules/mesh-delivery-forge-bridge (two directories up). The forge's own answer shape
// is mesh-catalog modules/gitea tools/index.ts gitea_get_file; the bridge sees only the bytes (issue 1448).
func real(t *testing.T, name string) []byte {
t.Helper()
b, err := os.ReadFile("../../" + name)
if err != nil {
t.Fatal(err)
}
return b
}
func hexSum(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) }
type fileCall struct{ owner, repo, path, ref string }
// forgeFiles answers as the forge does for a path: its bytes, ErrNotFound for a path in none, the error in errs.
type forgeFiles struct {
have map[string][]byte
errs map[string]error
mu sync.Mutex
calls []fileCall
}
func (f *forgeFiles) get(owner, repo, path, ref string) ([]byte, error) {
f.mu.Lock()
f.calls = append(f.calls, fileCall{owner, repo, path, ref})
f.mu.Unlock()
if err, ok := f.errs[path]; ok {
return nil, err
}
if b, ok := f.have[path]; ok {
return b, nil
}
return nil, ErrNotFound
}
// anyFiles is a forge that has a module.json at every path and no lock file, for tests that are not about the files.
func anyFiles(owner, repo, path, ref string) ([]byte, error) {
if strings.HasSuffix(path, "module.json") {
return []byte("{}"), nil
}
return nil, ErrNotFound
}
func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1
manifest, sum := real(t, "module.json"), real(t, "go.sum")
forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest, "modules/a/go.sum": sum, "modules/b/module.json": []byte(`{"module":"b"}`)}}
var calls []call
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1", "modules/b": "t2"}, nil), forge.get)
if err != nil {
t.Fatal(err)
}
f, err := delivery.DecodeForge(msg.Data)
if err != nil {
t.Fatal(err)
}
if want := map[string]string{"modules/a": string(manifest), "modules/b": `{"module":"b"}`}; !reflect.DeepEqual(f.Manifests, want) {
t.Errorf("manifests = %v", f.Manifests)
}
if want := map[string]map[string]string{"modules/a": {"go.sum": hexSum(sum)}}; !reflect.DeepEqual(f.Locks, want) {
t.Errorf("locks = %v, want %v", f.Locks, want)
}
// ADR 0593 §2 (D6): the bridge reads no mesh package; only the indexer computes what a fingerprint takes.
if len(f.Packages) != 0 || strings.Contains(string(msg.Data), `"packages"`) {
t.Errorf("the merged message carries packages: %v", f.Packages)
}
// ADR 0567: placement is the seat mesh-placement's: a merge of the catalogue carries no placement file and asks the
// forge for none.
forge.calls = nil
msg, err = Convert(mergedSubject, mergedBody(map[string]any{"repo": "mesh-catalog"}), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get)
if err != nil {
t.Fatal(err)
}
if f, err = delivery.DecodeForge(msg.Data); err != nil {
t.Fatal(err)
}
if len(f.RetiredPlacements) > 0 || f.RetiredRead || strings.Contains(string(msg.Data), `"placements`) {
t.Errorf("a merge of the catalogue carries placement: %s", msg.Data)
}
for _, c := range forge.calls {
if strings.HasPrefix(c.path, "placement") {
t.Errorf("the forge was asked for %s", c.path)
}
}
// 1953/D1 (ADR 0565 §2): the catalogue's merged message carries its predefined.json at the merge, or says it holds
// none; another repository's asks the forge for none.
if !f.PredefinedRead || len(f.Predefined) != 0 {
t.Errorf("a merge of the catalogue with no predefined.json does not say so: %s", msg.Data)
}
forge.have[PredefinedFile] = []byte(`{"font-size":{"type":"integer","value":13}}`)
msg, _ = Convert(mergedSubject, mergedBody(map[string]any{"repo": "mesh-catalog"}), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get)
if f, err = delivery.DecodeForge(msg.Data); err != nil || string(f.Predefined) != string(forge.have[PredefinedFile]) || f.PredefinedRead {
t.Errorf("predefined = %s (%v)", f.Predefined, err)
}
forge.calls = nil
msg, _ = Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get)
for _, c := range forge.calls {
if c.path == PredefinedFile {
t.Errorf("another repository's merge asked for %s", c.path)
}
}
forge.errs = map[string]error{PredefinedFile: errors.New("forge down")}
if _, err := Convert(mergedSubject, mergedBody(map[string]any{"repo": "mesh-catalog"}), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get); !errors.Is(err, ErrRetry) {
t.Errorf("a forge that cannot say is %v, want a retry", err)
}
// ADR 0632 §2: of a manifest naming a template, the template is read at the merge and the module's needs carried
// whole: docker's daemon file names the image seat's reach while its manifest stays the same. A manifest naming
// none carries no needs; a template the forge does not have is no need, and no retry.
forge.errs = nil
forge.have["modules/d/module.json"] = []byte(`{"module":"d","version":"1","resources":[{"id":"daemon","type":"file",` +
`"path":"/etc/d.json","template":"files/daemon.json"},{"id":"gone","type":"file","path":"/etc/g","template":"files/gone"}]}`)
forge.have["modules/d/files/daemon.json"] = []byte(`{"registry":"${seat:container-image-registry:reach}"}`)
msg, err = Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/b": "t2", "modules/d": "t4"}, nil), forge.get)
if err != nil {
t.Fatal(err)
}
if f, err = delivery.DecodeForge(msg.Data); err != nil || len(f.Needs) != 1 || !slices.Contains(f.Needs["modules/d"], "form:seat-reach") {
t.Errorf("needs = %v (%v), want modules/d's alone, with form:seat-reach from its template", f.Needs, err)
}
// Issue 1126: a merge reads the forge only for the module directories whose tree changed.
t.Run("a merge reads only the directories whose tree changed", mergeReadsOnlyChangedDirs)
}
func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2
manifest := real(t, "module.json")
forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest, "modules/a/go.sum": []byte("git.novox.be/novox/mesh-sdk/go v0.1.7 h1:x=\n")}}
msg, err := Convert(updatedSubject, updated(nil), nil, forge.get)
if err != nil {
t.Fatal(err)
}
f, _ := delivery.DecodeForge(msg.Data)
if !reflect.DeepEqual(f.Manifests, map[string]string{"modules/a": string(manifest)}) || len(f.Locks) != 1 || len(f.Packages) != 0 {
t.Errorf("manifests %v locks %v packages %v", f.Manifests, f.Locks, f.Packages)
}
if len(forge.calls) == 0 || forge.calls[0].ref != head {
t.Errorf("calls %v, want the head %s", forge.calls, head)
}
// Dirs not known: neither field, and the forge is not asked.
forge = &forgeFiles{}
msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs": nil, "module_dirs_said": nil}), nil, forge.get)
if err != nil {
t.Fatal(err)
}
f, _ = delivery.DecodeForge(msg.Data)
if f.Manifests != nil || f.Locks != nil || len(forge.calls) != 0 {
t.Errorf("manifests %v locks %v calls %v", f.Manifests, f.Locks, forge.calls)
}
}
func TestAForgeErrorOrAMissingManifestIsRetryAndALockNotFoundIsLeftOut(t *testing.T) { // 1448/AC3
boom := errors.New("Gitea API /repos/novox/mesh-delivery/contents/modules/a/go.sum?ref=x: 502 bad gateway")
trees := func() Trees { var c []call; return treesOf(&c, "r0", map[string]string{"modules/a": "t1"}, nil) }
for name, forge := range map[string]*forgeFiles{
"a manifest error": {errs: map[string]error{"modules/a/module.json": boom}},
"a manifest not found": {},
"a lock file error": {have: map[string][]byte{"modules/a/module.json": []byte("{}")}, errs: map[string]error{"modules/a/go.sum": boom}},
"a second lock error": {have: map[string][]byte{"modules/a/module.json": []byte("{}")}, errs: map[string]error{"modules/a/package-lock.json": boom}},
} {
msg, err := Convert(mergedSubject, mergedBody(nil), trees(), forge.get)
if !errors.Is(err, ErrRetry) || msg.Data != nil {
t.Errorf("%s: err = %v data = %s, want ErrRetry and no message", name, err, msg.Data)
}
_, err = Convert(updatedSubject, updated(nil), nil, forge.get)
if !errors.Is(err, ErrRetry) {
t.Errorf("%s (pull): err = %v, want ErrRetry", name, err)
}
}
forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": []byte("{}")}}
msg, err := Convert(mergedSubject, mergedBody(nil), trees(), forge.get)
if err != nil {
t.Fatal(err)
}
if f, _ := delivery.DecodeForge(msg.Data); f.Locks != nil {
t.Errorf("locks = %v, want none for lock files the forge does not have", f.Locks)
}
}
func TestTheFilesAreAskedAtTheMergeCommitByOwnerRepoPathAndRef(t *testing.T) { // 1448/AC4 (the call's arguments; the manifest is in manifest_test)
forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": []byte("{}")}}
var calls []call
if _, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get); err != nil {
t.Fatal(err)
}
want := []fileCall{{"novox", "mesh-delivery", "modules/a/module.json", merge}, {"novox", "mesh-delivery", "modules/a/go.sum", merge},
{"novox", "mesh-delivery", "modules/a/package-lock.json", merge}}
sort.Slice(forge.calls, func(i, j int) bool { return forge.calls[i].path < forge.calls[j].path }) // they are asked for concurrently
sort.Slice(want, func(i, j int) bool { return want[i].path < want[j].path })
if !reflect.DeepEqual(forge.calls, want) {
t.Errorf("calls = %v, want %v", forge.calls, want)
}
}
// slowForge answers like a forge that takes d a call: a module.json of the directory's name everywhere, a go.sum in
// the directories whose number is even and in the root, and counts the calls in flight.
type slowForge struct {
d time.Duration
mu sync.Mutex
inFlight, most, started, n int
failOn string
finished int
}
func (f *slowForge) get(_, _, path, _ string) ([]byte, error) {
f.mu.Lock()
f.inFlight++
f.started++
if f.inFlight > f.most {
f.most = f.inFlight
}
f.mu.Unlock()
time.Sleep(f.d)
defer func() { f.mu.Lock(); f.inFlight--; f.finished++; f.mu.Unlock() }()
switch {
case path == f.failOn:
return nil, errors.New("Gitea API " + path + ": 502 bad gateway")
case strings.HasSuffix(path, "module.json"):
return []byte(`{"module":"` + path + `"}`), nil
case strings.HasSuffix(path, "go.sum") && (path == "go.sum" || path[len(path)-8]%2 == 0):
return []byte("sum of " + path), nil
}
return nil, ErrNotFound
}
func manyDirs(n int) []string {
dirs := []string{""}
for i := 1; i < n; i++ {
dirs = append(dirs, fmt.Sprintf("modules/m%02d", i))
}
return dirs
}
func TestFortyDirectoriesAreReadEightAtATimeWithTheResultOfOneAtATime(t *testing.T) { // 1458/AC1
dirs := manyDirs(40)
slow := &slowForge{d: 10 * time.Millisecond}
manifests, locks, err := inputsOf("novox", "mesh-catalog", merge, dirs, slow.get)
if err != nil {
t.Fatal(err)
}
if slow.most > MaxReadsInFlight || slow.most < 2 {
t.Errorf("at most %d calls were in flight, want more than one and at most %d", slow.most, MaxReadsInFlight)
}
if slow.started != 40*3 {
t.Errorf("%d calls, want 120", slow.started)
}
// The same read one file at a time.
wantM, wantL := map[string]string{}, map[string]map[string]string{}
one := &slowForge{}
for _, dir := range dirs {
m, l, err := inputsOfOne(dir, one.get)
if err != nil {
t.Fatal(err)
}
wantM[dir] = m
if l != nil {
wantL[dir] = l
}
}
if !reflect.DeepEqual(manifests, wantM) || !reflect.DeepEqual(locks, wantL) {
t.Errorf("the concurrent read differs from a one-at-a-time read: %d manifests %d locks, want %d and %d", len(manifests), len(locks), len(wantM), len(wantL))
}
if len(locks) == 0 || len(locks) == 40 {
t.Errorf("%d directories with locks, the fixture should have some and not all", len(locks))
}
}
// inputsOfOne reads one directory with a single call at a time, as the bridge did before issue 1458.
func inputsOfOne(dir string, files Files) (string, map[string]string, error) {
at := func(name string) string {
if dir == "" {
return name
}
return dir + "/" + name
}
raw, err := files("o", "r", at("module.json"), "x")
if err != nil {
return "", nil, err
}
var locks map[string]string
for _, name := range LockFiles {
b, err := files("o", "r", at(name), "x")
if errors.Is(err, ErrNotFound) {
continue
}
if err != nil {
return "", nil, err
}
if locks == nil {
locks = map[string]string{}
}
s := sha256.Sum256(b)
locks[name] = hex.EncodeToString(s[:])
}
return string(raw), locks, nil
}
func TestOneFailingFileIsRetryAndEveryCallHasReturnedBeforeTheReadDoes(t *testing.T) { // 1458/AC2
dirs := manyDirs(40)
slow := &slowForge{d: 10 * time.Millisecond, failOn: "modules/m05/go.sum"}
_, _, err := inputsOf("novox", "mesh-catalog", merge, dirs, slow.get)
if err == nil || !strings.Contains(err.Error(), "modules/m05/go.sum") || !strings.Contains(err.Error(), "502") {
t.Fatalf("err = %v, want the failing file named", err)
}
slow.mu.Lock()
defer slow.mu.Unlock()
if slow.inFlight != 0 || slow.finished != slow.started {
t.Errorf("%d calls in flight, %d of %d finished, when the read returned", slow.inFlight, slow.finished, slow.started)
}
if slow.started >= 40*3 {
t.Errorf("%d calls were made, want the read to stop asking after the failure", slow.started)
}
// Through Convert it is ErrRetry.
var calls []call
dirsMap := map[string]string{}
for _, d := range dirs[1:] {
dirsMap[d] = "t"
}
slow2 := &slowForge{failOn: "modules/m05/module.json"}
if _, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", dirsMap, nil), slow2.get); !errors.Is(err, ErrRetry) {
t.Errorf("err = %v, want ErrRetry", err)
}
}
// memKept is a Kept in memory.
type memKept map[string]Inputs
func (k memKept) Get(tree string) (Inputs, bool) { in, ok := k[tree]; return in, ok }
func (k memKept) Put(tree string, in Inputs) { k[tree] = in }
// Issue 1126 (mesh-catalog#535; issue 1458 met the kind first): a merge reads the forge only for the module directories
// whose tree changed. Its message is the same as one read whole; a directory whose tree is kept costs no read.
func mergeReadsOnlyChangedDirs(t *testing.T) {
manifest, sum := real(t, "module.json"), real(t, "go.sum")
forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest, "modules/a/go.sum": sum, "modules/b/module.json": []byte(`{"module":"b"}`)}}
kept := memKept{}
var calls []call
first, err := ConvertKept(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1", "modules/b": "t2"}, nil), forge.get, kept)
if err != nil {
t.Fatal(err)
}
if len(forge.calls) != 6 {
t.Fatalf("the first merge read %d files, want 6 (a manifest and two lock files for each directory)", len(forge.calls))
}
// The next merge changes modules/b alone: only its three files are read, and the message equals one read whole.
forge.calls = nil
forge.have["modules/b/module.json"] = []byte(`{"module":"b","v":2}`)
trees := map[string]string{"modules/a": "t1", "modules/b": "t3"}
next, err := ConvertKept(mergedSubject, mergedBody(nil), treesOf(&calls, "r1", trees, nil), forge.get, kept)
if err != nil {
t.Fatal(err)
}
for _, c := range forge.calls {
if !strings.HasPrefix(c.path, "modules/b/") {
t.Errorf("%s was read although its directory's tree is kept", c.path)
}
}
if len(forge.calls) != 3 {
t.Errorf("the second merge read %d files, want 3", len(forge.calls))
}
whole, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r1", trees, nil), forge.get)
if err != nil {
t.Fatal(err)
}
if string(next.Data) != string(whole.Data) {
t.Errorf("the message from kept inputs differs from one read whole:\n%s\n%s", next.Data, whole.Data)
}
if string(first.Data) == string(next.Data) {
t.Error("the second merge's message did not change")
}
// A failed read keeps the directories it read whole and nothing of the one that failed, so the next try reads only the
// rest: a merge too large for one try's bound converges over its tries.
kept2 := memKept{}
failing := &forgeFiles{have: forge.have, errs: map[string]error{"modules/b/module.json": errors.New("context deadline exceeded")}}
if _, err := ConvertKept(mergedSubject, mergedBody(nil), treesOf(&calls, "r1", trees, nil), failing.get, kept2); !errors.Is(err, ErrRetry) {
t.Fatalf("err = %v, want ErrRetry", err)
}
if _, ok := kept2["t1"]; !ok || len(kept2) != 1 { // modules/a's files are asked first, before the failure is seen
t.Errorf("a failed read kept %v, want modules/a's tree t1 alone", kept2)
}
failing.calls = nil
failing.errs = nil
again, err := ConvertKept(mergedSubject, mergedBody(nil), treesOf(&calls, "r1", trees, nil), failing.get, kept2)
if err != nil {
t.Fatal(err)
}
if string(again.Data) != string(whole.Data) {
t.Errorf("the message of the try after a failure differs from one read whole")
}
if len(failing.calls) != 3 {
t.Errorf("the try after a failure read %d files, want only modules/b's 3", len(failing.calls))
}
// ADR 0593 §2: inputs an older bridge kept with the packages it read are taken as they are; their packages are left.
}