diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go index 9310a73..2c8789f 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go @@ -91,6 +91,33 @@ func treesVia(ctx context.Context, call caller) bridge.Trees { } } +// namedVia asks the forge for the git object of each named path through call: gitea_commit_trees with `paths`, by +// directory (the root spelled ""), the paths relative to it, answering {root, trees, paths} with paths by directory then +// path (the interface agreed in R4e between the lead, the catalogue's leaf lane and this lane; mesh-sdk +// delivery.Forge.Paths; novox/hq ADR 0499 clause 3). An answer without every path asked is an error. +func namedVia(ctx context.Context, call caller) bridge.Named { + return func(owner, repo, sha string, paths map[string][]string) (map[string]map[string]string, error) { + raw, err := call(ctx, commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha, "paths": paths}) + if err != nil { + return nil, err + } + var a struct { + Paths map[string]map[string]string `json:"paths"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the answer of %s is not {root, trees, paths}: %w", commitTreesTool, err) + } + for dir, want := range paths { + for _, p := range want { + if a.Paths[dir][p] == "" { + return nil, fmt.Errorf("the answer of %s names no object for %q of %q", commitTreesTool, p, dir) + } + } + } + return a.Paths, nil + } +} + // getFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea tools/index.ts gitea_get_file): // arguments owner, repo, path and ref, answer {file: {path, ref, sha, size, content}}, the content decoded to text // (client.ts getFile). A path the forge does not have fails with `Gitea API : 404 ` (client.ts request). diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go index fc1aaff..595f185 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go @@ -115,8 +115,8 @@ type publisher interface { // handle turns one event into its message on FORGE and acknowledges it. An event it cannot read is terminated, never // redelivered, and the log names its subject and why. A message it could not write stays unacknowledged: the server // delivers it again (the server's ack wait is the only clock). -func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, logger *log.Logger) { - msg, err := bridge.Convert(m.Subject(), m.Data(), trees, files) +func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, named bridge.Named, logger *log.Logger) { + msg, err := bridge.ConvertNamed(m.Subject(), m.Data(), trees, files, named) switch { case errors.Is(err, bridge.ErrRetry): logger.Printf("%v; heard again in %s", err, retryDelay) @@ -256,7 +256,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // Bounded: the turn is held, and no sync or other event goes on without it. hctx, cancel := context.WithTimeout(ctx, syncDeadline) defer cancel() - handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), logger) + handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), namedVia(hctx, call), logger) }) }, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) })) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go index c4ab010..bd0fe4c 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go @@ -68,7 +68,7 @@ func body(m map[string]any) []byte { func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit part; the bus is the integration test) var logs bytes.Buffer m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{} - handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0)) + handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0)) if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull" { t.Fatalf("published on %v", pub.subjects) } @@ -84,7 +84,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5 var logs bytes.Buffer m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(map[string]any{"head_sha": "short"})}, &fakePub{} - handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0)) + handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0)) if len(pub.subjects) != 0 { t.Errorf("a message was written: %v", pub.subjects) } @@ -99,7 +99,7 @@ func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5 func TestUnwrittenMessageIsNotAcked(t *testing.T) { // 1235 (a failed write is redelivered by the server, not retried here) var logs bytes.Buffer m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")} - handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0)) + handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0)) if m.acked != 0 || m.termd != 0 { t.Errorf("acked %d terminated %d, want neither", m.acked, m.termd) } @@ -121,7 +121,7 @@ func TestTreesRefusedLeavesTheEventForTenSecondsAndLogs(t *testing.T) { // 1236/ call := func(context.Context, string, any) (json.RawMessage, error) { return nil, errors.New("the tree listing of novox/mesh-delivery was cut") } - handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), log.New(&logs, "", 0)) + handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), namedVia(context.Background(), call), log.New(&logs, "", 0)) if len(pub.subjects) != 0 { t.Errorf("a message was written: %v", pub.subjects) } @@ -150,7 +150,7 @@ func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/ } return json.RawMessage(`{"root":"r0","trees":{"":"t0","modules/a":"t1"}}`), nil } - handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), log.New(&logs, "", 0)) + handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), namedVia(context.Background(), call), log.New(&logs, "", 0)) if len(keys) == 0 || keys[0] != "gitea.gitea_commit_trees" || slices.Index(keys[1:], "gitea.gitea_commit_trees") >= 0 { t.Fatalf("calls %v, want the trees once and first", keys) } @@ -230,3 +230,27 @@ func someFiles(_, _, path, _ string) ([]byte, error) { } return nil, bridge.ErrNotFound } + +func TestNamedViaAsksTheTreesToolWithThePathsAndTakesTheirObjects(t *testing.T) { // I6, ADR 0499 clause 3 + var asked map[string]any + call := func(_ context.Context, key string, body any) (json.RawMessage, error) { + if key != commitTreesTool { + t.Errorf("called %s", key) + } + asked = body.(map[string]any) + return json.RawMessage(`{"root":"r0","trees":{"":"r0"},"paths":{"":{"src":"1111111111111111111111111111111111111111","package.json":"3333333333333333333333333333333333333333"}}}`), nil + } + got, err := namedVia(context.Background(), call)("novox", "mesh-sdk", "s", map[string][]string{"": {"package.json", "src"}}) + if err != nil || got[""]["src"] != "1111111111111111111111111111111111111111" || got[""]["package.json"] == "" { + t.Fatalf("objects = %v, %v", got, err) + } + if asked["sha"] != "s" || asked["paths"] == nil { + t.Errorf("asked %v, want the commit and the paths", asked) + } + call = func(context.Context, string, any) (json.RawMessage, error) { + return json.RawMessage(`{"root":"r0","trees":{},"paths":{"":{"src":"1111111111111111111111111111111111111111"}}}`), nil + } + if _, err := namedVia(context.Background(), call)("novox", "mesh-sdk", "s", map[string][]string{"": {"package.json", "src"}}); err == nil { + t.Error("an answer missing a path asked was taken") + } +} diff --git a/modules/mesh-delivery-forge-bridge/go.mod b/modules/mesh-delivery-forge-bridge/go.mod index 383758b..7239d27 100644 --- a/modules/mesh-delivery-forge-bridge/go.mod +++ b/modules/mesh-delivery-forge-bridge/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-forge-bridge/go.sum b/modules/mesh-delivery-forge-bridge/go.sum index 3515fb8..bb52af2 100644 --- a/modules/mesh-delivery-forge-bridge/go.sum +++ b/modules/mesh-delivery-forge-bridge/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7 h1:wK9O1CgNk7mo3IdJqW/CCx/hebg+1F0rkvW8CvJfnRM= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go index a6cbc7d..0447e53 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go @@ -88,6 +88,13 @@ func unreadable(subject, why string, args ...any) error { // the trunk asks trees once and becomes a merged message; a merge into any other branch is a closed pull message, as the // pull request left its group. A forge that cannot say is ErrRetry; a merge with no module.json is ErrNoModules. func Convert(subject string, data []byte, trees Trees, files Files) (Message, error) { + return ConvertNamed(subject, data, trees, files, nil) +} + +// ConvertNamed is Convert that also asks named for the git object of each path a module's manifest names for every +// artifact (novox/hq ADR 0499 clause 3), at the commit its manifest is read at, and carries them in the message's Paths. +// Nil named asks nothing, and the indexer gives such a module none. +func ConvertNamed(subject string, data []byte, trees Trees, files Files, named Named) (Message, error) { var action string switch subject { case UpdatedSubject: @@ -114,7 +121,7 @@ func Convert(subject string, data []byte, trees Trees, files Files) (Message, er return Message{}, unreadable(subject, "no branch") } if subject == MergedSubject && p.Base == Trunk { - return merged(subject, p, trees, files) + return merged(subject, p, trees, files, named) } f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number, Head: p.HeadSHA, Action: action} @@ -125,6 +132,9 @@ func Convert(subject string, data []byte, trees Trees, files Files) (Message, er if f.Manifests, f.Locks, err = inputsOf(p.Owner, p.Repo, p.HeadSHA, p.ModuleDirs, files); err != nil { return Message{}, fmt.Errorf("%w: %s: the manifests of %s/%s at the head %s: %v", ErrRetry, subject, p.Owner, p.Repo, p.HeadSHA, err) } + if f.Paths, err = objectsOf(p.Owner, p.Repo, p.HeadSHA, f.Manifests, named); err != nil { + return Message{}, fmt.Errorf("%w: %s: %s/%s at the head %s: %v", ErrRetry, subject, p.Owner, p.Repo, p.HeadSHA, err) + } } event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull) if err != nil { @@ -227,7 +237,7 @@ func inputsOf(owner, repo, ref string, dirs []string, files Files) (manifests ma } // merged is the merged message of a merge into the trunk: the trees of the merge commit, asked once. -func merged(subject string, p pull, trees Trees, files Files) (Message, error) { +func merged(subject string, p pull, trees Trees, files Files, named Named) (Message, error) { if !sha40.MatchString(p.MergeSHA) { return Message{}, unreadable(subject, "the merge commit %q is not 40 hex digits", p.MergeSHA) } @@ -248,12 +258,16 @@ func merged(subject string, p pull, trees Trees, files Files) (Message, error) { if err != nil { return Message{}, fmt.Errorf("%w: %s: the manifests of %s at the merge commit %s: %v", ErrRetry, subject, repository, p.MergeSHA, err) } + objects, err := objectsOf(p.Owner, p.Repo, p.MergeSHA, manifests, named) + if err != nil { + return Message{}, fmt.Errorf("%w: %s: %s at the merge commit %s: %v", ErrRetry, subject, repository, p.MergeSHA, err) + } event, err := delivery.ForgeEvent(repository, delivery.ForgeMerged) if err != nil { return Message{}, unreadable(subject, "%v", err) } body, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number, - Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks}) + Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks, Paths: objects}) if err != nil { return Message{}, unreadable(subject, "%v", err) } diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/named.go b/modules/mesh-delivery-forge-bridge/internal/bridge/named.go new file mode 100644 index 0000000..fa31605 --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/named.go @@ -0,0 +1,63 @@ +package bridge + +import ( + "encoding/json" + "fmt" + "slices" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Named asks the forge for the git object of each named path of the module directories given, at a commit: by directory +// (the root spelled ""), the paths relative to it; answered by directory, then path (gitea_commit_trees with `paths`, +// mesh-sdk delivery.Forge.Paths; novox/hq ADR 0499 clause 3). A path the forge does not hold fails the call. +type Named func(owner, repo, sha string, paths map[string][]string) (map[string]map[string]string, error) + +// namedPathsOf are, of each manifest that names the paths of every artifact, those paths by directory (catalogue +// Manifest.NamedPaths, ADR 0499 clause 2). Only the build section is read: what the paths are is the catalogue's rule, +// and a manifest the catalogue would refuse for another reason is judged by the indexer, which gives it none. +func namedPathsOf(manifests map[string]string) map[string][]string { + out := map[string][]string{} + for dir, raw := range manifests { + var m struct { + Build *catalogue.Build `json:"build"` + } + if json.Unmarshal([]byte(raw), &m) != nil || m.Build == nil { + continue + } + if paths, every := (catalogue.Manifest{Build: m.Build}).NamedPaths(); every { + out[dir] = paths + } + } + return out +} + +// objectsOf asks named for the objects of the named paths the manifests give at sha, and answers none when no manifest +// names paths. A forge that cannot say is an error. +func objectsOf(owner, repo, sha string, manifests map[string]string, named Named) (map[string]map[string]string, error) { + paths := namedPathsOf(manifests) + if len(paths) == 0 || named == nil { + return nil, nil + } + objects, err := named(owner, repo, sha, paths) + if err != nil { + return nil, fmt.Errorf("the objects of the named paths of %s: %w", dirsOf(paths), err) + } + for dir, want := range paths { + for _, p := range want { + if objects[dir][p] == "" { + return nil, fmt.Errorf("the forge named no object for the named path %q of %q", p, dir) + } + } + } + return objects, nil +} + +func dirsOf(paths map[string][]string) []string { + var dirs []string + for d := range paths { + dirs = append(dirs, d) + } + slices.Sort(dirs) + return dirs +} diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/named_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/named_test.go new file mode 100644 index 0000000..5cf39c5 --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/named_test.go @@ -0,0 +1,98 @@ +package bridge + +import ( + "os" + "reflect" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// ADR 0499 clause 3 (I6): the merged and the pull message carry each named path's git object of a module whose every +// artifact names its paths, read at the same commit as its manifest. The manifest is mesh-sdk's module.json at 9cb4d59 (its +// lib names src, test, package.json, package-lock.json and tsconfig.json). + +type namedCall struct { + owner, repo, sha string + paths map[string][]string +} + +func namedOf(calls *[]namedCall, answer map[string]map[string]string, err error) Named { + return func(owner, repo, sha string, paths map[string][]string) (map[string]map[string]string, error) { + *calls = append(*calls, namedCall{owner, repo, sha, paths}) + return answer, err + } +} + +func sdkFiles(t *testing.T) Files { + t.Helper() + raw, err := os.ReadFile("testdata/mesh-sdk-9cb4d59.module.json") + if err != nil { + t.Fatal(err) + } + return func(owner, repo, path, ref string) ([]byte, error) { + if path == "module.json" { + return raw, nil + } + return nil, ErrNotFound + } +} + +var sdkObjects = map[string]map[string]string{"": {"src": "1111111111111111111111111111111111111111", "test": "2222222222222222222222222222222222222222", + "package.json": "3333333333333333333333333333333333333333", "package-lock.json": "4444444444444444444444444444444444444444", "tsconfig.json": "5555555555555555555555555555555555555555"}} + +func TestAMergedMessageCarriesTheNamedPathsObjectsOfAModuleNamingItsPaths(t *testing.T) { // I6, ADR 0499 clause 3 + var calls []call + var named []namedCall + body := mergedBody(map[string]any{"repo": "mesh-sdk", "module_dirs": []string{""}}) + msg, err := ConvertNamed(mergedSubject, body, treesOf(&calls, "r0", map[string]string{"": "r0"}, nil), sdkFiles(t), namedOf(&named, sdkObjects, nil)) + if err != nil { + t.Fatal(err) + } + f, err := delivery.DecodeForge(msg.Data) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(f.Paths, sdkObjects) { + t.Errorf("paths = %v, want the forge's objects of lib's named paths", f.Paths) + } + if len(named) != 1 || named[0].sha != merge || named[0].repo != "mesh-sdk" || + strings.Join(named[0].paths[""], ",") != "package-lock.json,package.json,src,test,tsconfig.json" { + t.Errorf("asked %+v, want the root's named paths at the merge commit", named) + } +} + +func TestAMessageOfModulesNamingNoPathsAsksNoObjects(t *testing.T) { // I6 + var calls []call + var named []namedCall + if _, err := ConvertNamed(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), anyFiles, namedOf(&named, nil, nil)); err != nil { + t.Fatal(err) + } + if len(named) != 0 { + t.Errorf("asked %+v, want nothing for modules naming no paths", named) + } +} + +func TestAPullMessageCarriesTheNamedPathsObjectsAtItsHead(t *testing.T) { // I6, ADR 0499 clause 3 + var named []namedCall + body := mergedBody(map[string]any{"repo": "mesh-sdk", "module_dirs": []string{""}, "base": "feat/y"}) + msg, err := ConvertNamed(UpdatedSubject, body, nil, sdkFiles(t), namedOf(&named, sdkObjects, nil)) + if err != nil { + t.Fatal(err) + } + f, _ := delivery.DecodeForge(msg.Data) + if !reflect.DeepEqual(f.Paths, sdkObjects) || len(named) != 1 || named[0].sha != head { + t.Errorf("paths = %v, asked %+v; want the objects read at the head", f.Paths, named) + } +} + +func TestAForgeThatCannotSayTheObjectsIsARetry(t *testing.T) { // I6: a message without them would give the module none + var calls []call + var named []namedCall + body := mergedBody(map[string]any{"repo": "mesh-sdk", "module_dirs": []string{""}}) + _, err := ConvertNamed(mergedSubject, body, treesOf(&calls, "r0", map[string]string{"": "r0"}, nil), sdkFiles(t), namedOf(&named, nil, os.ErrDeadlineExceeded)) + if err == nil || !strings.Contains(err.Error(), "named paths") { + t.Errorf("err = %v, want a retry naming the named paths", err) + } +} diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/testdata/mesh-sdk-9cb4d59.module.json b/modules/mesh-delivery-forge-bridge/internal/bridge/testdata/mesh-sdk-9cb4d59.module.json new file mode 100644 index 0000000..404139a --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/testdata/mesh-sdk-9cb4d59.module.json @@ -0,0 +1,16 @@ +{ + "module": "mesh-sdk", + "version": "1", + "slug": "sdk", + "build": { + "artifacts": [ + { + "name": "lib", + "kind": "package", + "language": "typescript", + "paths": ["src", "test", "package.json", "package-lock.json", "tsconfig.json"] + } + ] + }, + "resources": [] +}