From 543d91d62656cf1cc4c53c82430f3de49048d8b3 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 18:51:27 +0000 Subject: [PATCH] Read each module's manifest and lock files in the forge bridge, so the indexer gets them (issue 1448, design 47 R4c-2) --- .../cmd/mesh-delivery-forge-bridge/call.go | 32 ++++++++ .../cmd/mesh-delivery-forge-bridge/main.go | 6 +- modules/mesh-delivery-forge-bridge/go.mod | 2 +- modules/mesh-delivery-forge-bridge/go.sum | 8 +- .../internal/bridge/bridge.go | 77 ++++++++++++++++++- .../manifest_test.go | 2 +- .../mesh-delivery-forge-bridge/module.json | 3 +- 7 files changed, 114 insertions(+), 16 deletions(-) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go index f1c1461..9310a73 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "regexp" "strings" "time" @@ -89,3 +90,34 @@ func treesVia(ctx context.Context, call caller) bridge.Trees { return a.Root, a.Trees, nil } } + +// getFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea tools/index.ts gitea_get_file): +// arguments owner, repo, path and ref, answer {file: {path, ref, sha, size, content}}, the content decoded to text +// (client.ts getFile). A path the forge does not have fails with `Gitea API : 404 ` (client.ts request). +const getFileTool = "gitea.gitea_get_file" + +// notFound is how the forge's error says 404 (client.ts request: `Gitea API ${path}: ${status} ${text}`; client.ts:670 +// recognises it as /: 404 /). +var notFound = regexp.MustCompile(`: 404 `) + +// filesVia asks the forge for a file through call. A file the forge says is not there is bridge.ErrNotFound. +func filesVia(ctx context.Context, call caller) bridge.Files { + return func(owner, repo, path, ref string) ([]byte, error) { + raw, err := call(ctx, getFileTool, map[string]string{"owner": owner, "repo": repo, "path": path, "ref": ref}) + if err != nil { + if notFound.MatchString(err.Error()) { + return nil, fmt.Errorf("%w: %v", bridge.ErrNotFound, err) + } + return nil, err + } + var a struct { + File *struct { + Content *string `json:"content"` + } `json:"file"` + } + if err := json.Unmarshal(raw, &a); err != nil || a.File == nil || a.File.Content == nil { + return nil, fmt.Errorf("the answer of %s is not {file: {content}}: %v", getFileTool, err) + } + return []byte(*a.File.Content), nil + } +} diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go index d9dd739..0438f1c 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go @@ -115,8 +115,8 @@ type publisher interface { // handle turns one event into its message on FORGE and acknowledges it. An event it cannot read is terminated, never // redelivered, and the log names its subject and why. A message it could not write stays unacknowledged: the server // delivers it again (the server's ack wait is the only clock). -func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, logger *log.Logger) { - msg, err := bridge.Convert(m.Subject(), m.Data(), trees) +func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, logger *log.Logger) { + msg, err := bridge.Convert(m.Subject(), m.Data(), trees, files) switch { case errors.Is(err, bridge.ErrRetry): logger.Printf("%v; heard again in %s", err, retryDelay) @@ -238,7 +238,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // Bounded: the turn is held, and no sync or other event goes on without it. hctx, cancel := context.WithTimeout(ctx, syncDeadline) defer cancel() - handle(hctx, m, rec, treesVia(hctx, call), logger) + handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), logger) }) }, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) })) diff --git a/modules/mesh-delivery-forge-bridge/go.mod b/modules/mesh-delivery-forge-bridge/go.mod index f3a0279..25936a8 100644 --- a/modules/mesh-delivery-forge-bridge/go.mod +++ b/modules/mesh-delivery-forge-bridge/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-forge-bridge/go.sum b/modules/mesh-delivery-forge-bridge/go.sum index 856a7bd..0e67d23 100644 --- a/modules/mesh-delivery-forge-bridge/go.sum +++ b/modules/mesh-delivery-forge-bridge/go.sum @@ -1,9 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019195003-4580d80a32d9 h1:XL1dbtSleGAixg6FX4gEnSG/AJxRG/tWw6/7U1RuzHc= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019195003-4580d80a32d9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69 h1:kvtV/SPsdRVUL/BZ3xxGS6U8YrXy/t7TRBjVrD6VLw8= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef h1:xy3wuGWJpqMAXYdXolocYG6w9xnWqBltVUffoaziIKo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960 h1:FCId3IuRScG03X7Gx9vfKfGNZH9jWnfx9RChfJ3eWfo= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go index e845d91..b1c74f8 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go @@ -2,6 +2,8 @@ package bridge import ( + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -34,6 +36,17 @@ const Trunk = "main" // (the root directory spelled ""), as the forge's gitea_commit_trees answers (issue 1234/AC4). type Trees func(owner, repo, sha string) (root string, trees map[string]string, err error) +// ErrNotFound says the forge has no file at that path and ref (the forge's "404", issue 1448). +var ErrNotFound = errors.New("the forge has no such file") + +// Files asks the forge for one file's bytes at a ref (a commit), as the forge's gitea_get_file answers; a file the forge +// says is not there is ErrNotFound, any other failure any other error (issue 1448). +type Files func(owner, repo, path, ref string) ([]byte, error) + +// LockFiles are the lock files read beside a module's manifest, by name relative to its directory (novox/hq ADR 0465 +// clause 1: go.sum for a Go module, package-lock.json for a Node one). +var LockFiles = []string{"go.sum", "package-lock.json"} + // Message is a message for FORGE: the event name (forge...) and its JSON body. type Message struct { Event string @@ -72,7 +85,7 @@ func unreadable(subject, why string, args ...any) error { // is an ErrUnreadable error naming the subject and why; an event it writes nothing for is ErrNotHandled. A merge into // the trunk asks trees once and becomes a merged message; a merge into any other branch is a closed pull message, as the // pull request left its group. A forge that cannot say is ErrRetry; a merge with no module.json is ErrNoModules. -func Convert(subject string, data []byte, trees Trees) (Message, error) { +func Convert(subject string, data []byte, trees Trees, files Files) (Message, error) { var action string switch subject { case UpdatedSubject: @@ -99,13 +112,17 @@ func Convert(subject string, data []byte, trees Trees) (Message, error) { return Message{}, unreadable(subject, "no branch") } if subject == MergedSubject && p.Base == Trunk { - return merged(subject, p, trees) + return merged(subject, p, trees, files) } f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number, Head: p.HeadSHA, Action: action} // The directories are the forge's word only when it said it could read them (module_dirs_said). if action == delivery.PullUpdated && p.ModuleDirsSaid { f.Dirs, f.DirsKnown = p.ModuleDirs, true + var err error + if f.Manifests, f.Locks, err = inputsOf(p.Owner, p.Repo, p.HeadSHA, p.ModuleDirs, files); err != nil { + return Message{}, fmt.Errorf("%w: %s: the manifests of %s/%s at the head %s: %v", ErrRetry, subject, p.Owner, p.Repo, p.HeadSHA, err) + } } event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull) if err != nil { @@ -118,8 +135,51 @@ func Convert(subject string, data []byte, trees Trees) (Message, error) { return Message{Event: event, Data: body}, nil } +// inputsOf reads, at ref, the module.json of each directory (the root spelled "") and the lock files beside it +// (LockFiles): the manifests as the forge answered them and the lock files' hex sha256 by name. A manifest the forge +// does not have is an error, since the directory was said to hold one; a lock file it does not have is left out; any other +// failure is an error (issue 1448). +func inputsOf(owner, repo, ref string, dirs []string, files Files) (manifests map[string]string, locks map[string]map[string]string, err error) { + manifests = map[string]string{} + locks = map[string]map[string]string{} + at := func(dir, name string) string { + if dir == "" { + return name + } + return dir + "/" + name + } + for _, dir := range dirs { + raw, err := files(owner, repo, at(dir, "module.json"), ref) + if err != nil { + return nil, nil, fmt.Errorf("%s: %w", at(dir, "module.json"), err) + } + manifests[dir] = string(raw) + for _, name := range LockFiles { + raw, err := files(owner, repo, at(dir, name), ref) + if errors.Is(err, ErrNotFound) { + continue + } + if err != nil { + return nil, nil, fmt.Errorf("%s: %w", at(dir, name), err) + } + sum := sha256.Sum256(raw) + if locks[dir] == nil { + locks[dir] = map[string]string{} + } + locks[dir][name] = hex.EncodeToString(sum[:]) + } + } + if len(manifests) == 0 { + manifests = nil + } + if len(locks) == 0 { + locks = nil + } + return manifests, locks, nil +} + // merged is the merged message of a merge into the trunk: the trees of the merge commit, asked once. -func merged(subject string, p pull, trees Trees) (Message, error) { +func merged(subject string, p pull, trees Trees, files Files) (Message, error) { if !sha40.MatchString(p.MergeSHA) { return Message{}, unreadable(subject, "the merge commit %q is not 40 hex digits", p.MergeSHA) } @@ -131,12 +191,21 @@ func merged(subject string, p pull, trees Trees) (Message, error) { if len(dirs) == 0 { return Message{}, fmt.Errorf("%w: %s: %s at the merge commit %s", ErrNoModules, subject, repository, p.MergeSHA) } + names := make([]string, 0, len(dirs)) + for dir := range dirs { + names = append(names, dir) + } + slices.Sort(names) + manifests, locks, err := inputsOf(p.Owner, p.Repo, p.MergeSHA, names, files) + if err != nil { + return Message{}, fmt.Errorf("%w: %s: the manifests of %s at the merge commit %s: %v", ErrRetry, subject, repository, p.MergeSHA, err) + } event, err := delivery.ForgeEvent(repository, delivery.ForgeMerged) if err != nil { return Message{}, unreadable(subject, "%v", err) } body, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number, - Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs}) + Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks}) if err != nil { return Message{}, unreadable(subject, "%v", err) } diff --git a/modules/mesh-delivery-forge-bridge/manifest_test.go b/modules/mesh-delivery-forge-bridge/manifest_test.go index f1bb6ea..d6682cc 100644 --- a/modules/mesh-delivery-forge-bridge/manifest_test.go +++ b/modules/mesh-delivery-forge-bridge/manifest_test.go @@ -93,7 +93,7 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1 t.Errorf("reads = %v, want %v", m.Reads, want) } if want := []string{"gitea.gitea_add_comment", "gitea.gitea_commit_status", "gitea.gitea_commit_trees", "gitea.gitea_delivery_view", - "gitea.gitea_merge_pull_request", "gitea.gitea_note_append", "gitea.gitea_open_pulls"}; !reflect.DeepEqual(sorted(m.Invokes), want) { + "gitea.gitea_get_file", "gitea.gitea_merge_pull_request", "gitea.gitea_note_append", "gitea.gitea_open_pulls"}; !reflect.DeepEqual(sorted(m.Invokes), want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { diff --git a/modules/mesh-delivery-forge-bridge/module.json b/modules/mesh-delivery-forge-bridge/module.json index 641fa86..25b68be 100644 --- a/modules/mesh-delivery-forge-bridge/module.json +++ b/modules/mesh-delivery-forge-bridge/module.json @@ -25,7 +25,8 @@ "gitea.gitea_note_append", "gitea.gitea_add_comment", "gitea.gitea_open_pulls", - "gitea.gitea_commit_trees" + "gitea.gitea_commit_trees", + "gitea.gitea_get_file" ], "own-secrets": { "broker": "${dir:state}/broker"