From d650a41ba6b8583954323b81f9e3039ed9f27aae Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 18:48:08 +0000 Subject: [PATCH 1/9] Copy the real manifests the indexer's new-form tests read (issues 1450, 1444) Fixtures are copied from the writer, never written by hand: four catalogue manifests at mesh-catalog a67f0196 and build-agent's at mesh-controller f2d7bc94. --- ...atalogue-a67f0196-audit-logger.module.json | 69 +++++++++ .../catalogue-a67f0196-avahi.module.json | 43 ++++++ .../catalogue-a67f0196-ca-trust.module.json | 66 +++++++++ .../catalogue-a67f0196-netcheck.module.json | 57 +++++++ ...ontroller-f2d7bc94-build-agent.module.json | 140 ++++++++++++++++++ 5 files changed, 375 insertions(+) create mode 100644 modules/mesh-delivery-indexer/internal/index/testdata/catalogue-a67f0196-audit-logger.module.json create mode 100644 modules/mesh-delivery-indexer/internal/index/testdata/catalogue-a67f0196-avahi.module.json create mode 100644 modules/mesh-delivery-indexer/internal/index/testdata/catalogue-a67f0196-ca-trust.module.json create mode 100644 modules/mesh-delivery-indexer/internal/index/testdata/catalogue-a67f0196-netcheck.module.json create mode 100644 modules/mesh-delivery-indexer/internal/index/testdata/controller-f2d7bc94-build-agent.module.json diff --git a/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-a67f0196-audit-logger.module.json b/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-a67f0196-audit-logger.module.json new file mode 100644 index 0000000..122b94e --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-a67f0196-audit-logger.module.json @@ -0,0 +1,69 @@ +{ + "module": "audit-logger", + "version": "1", + "slug": "audit", + "consumes": [ + "**" + ], + "build": { + "artifacts": [ + { + "name": "code", + "kind": "bundle", + "language": "typescript", + "entrypoints": [ + "index.js", + "tools/index.js" + ], + "loads": [ + "index.js", + "tools/index.js" + ], + "env": { + "AUDIT_LOG": "${dir:trail}/audit.log", + "AUDIT_SPOOL": "${dir:spool}" + } + } + ] + }, + "data": { + "own": [ + { + "id": "trail", + "path": "${dir:trail}", + "class": "valuable", + "why": "the audit trail, written nowhere else" + }, + { + "id": "spool", + "path": "${dir:spool}", + "class": "valuable", + "why": "events the trail could not take yet; after the bus gives one up, the only copy" + } + ] + }, + "resources": [ + { + "id": "state", + "type": "directory", + "mode": "0700", + "place": ".", + "unknown": "program" + }, + { + "id": "trail", + "type": "directory", + "mode": "0700", + "unknown": "program" + }, + { + "id": "spool", + "type": "directory", + "mode": "0700", + "unknown": "program" + } + ], + "capabilities": [ + "container-runtime" + ] +} diff --git a/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-a67f0196-avahi.module.json b/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-a67f0196-avahi.module.json new file mode 100644 index 0000000..de99c7c --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-a67f0196-avahi.module.json @@ -0,0 +1,43 @@ +{ + "module": "avahi", + "version": "1", + "capabilities": [ + "package-manager", + "service-manager" + ], + "tools": [ + "avahi_status", + "avahi_browse", + "avahi_resolve", + "avahi_services" + ], + "resources": [ + { + "id": "package", + "type": "package", + "package": "avahi" + }, + { + "id": "daemon", + "type": "service", + "unit": "avahi-daemon.service", + "state": "running", + "boot": "enabled" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/avahi-tools", + "binary": "avahi-tools", + "loads": [ + "avahi-tools" + ] + } + ] + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-a67f0196-ca-trust.module.json b/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-a67f0196-ca-trust.module.json new file mode 100644 index 0000000..b3c9dc9 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-a67f0196-ca-trust.module.json @@ -0,0 +1,66 @@ +{ + "module": "ca-trust", + "version": "1", + "slug": "catrust", + "capabilities": [ + "service-manager" + ], + "requires": [ + "internal-acme-ca" + ], + "seats": [ + { + "name": "the-mesh-trust-anchor", + "scope": "node" + } + ], + "claims": [ + { + "name": "the-mesh-trust-anchor", + "scope": "node" + } + ], + "resources": [ + { + "id": "state", + "type": "directory", + "mode": "0700", + "place": ".", + "unknown": "program" + }, + { + "id": "anchor", + "type": "file", + "path": "${dir:state}/anchor", + "mode": "0755", + "template": "files/anchor" + }, + { + "id": "unit", + "type": "file", + "path": "/etc/systemd/system/mesh-ca-trust.service", + "mode": "0644", + "template": "files/mesh-ca-trust.service" + }, + { + "id": "trust", + "type": "service", + "unit": "mesh-ca-trust.service", + "state": "running", + "boot": "enabled", + "restart-on": [ + "anchor", + "unit" + ] + } + ], + "build": { + "artifacts": [ + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-a67f0196-netcheck.module.json b/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-a67f0196-netcheck.module.json new file mode 100644 index 0000000..ec01003 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/testdata/catalogue-a67f0196-netcheck.module.json @@ -0,0 +1,57 @@ +{ + "module": "netcheck", + "version": "1", + "emits": [ + "tunnel.endpoint-changed", + "tunnel.handshake-stale", + "tunnel.handshake-fresh" + ], + "tools": [ + "netcheck_tunnel", + "netcheck_tcp", + "netcheck_dns", + "netcheck_http", + "netcheck_listening" + ], + "replaces": { + "netcheck_listening": [ + "ss -lunt", + "ss -ltn", + "netstat -lnt" + ], + "netcheck_dns": [ + "dig", + "nslookup" + ], + "netcheck_tunnel": [ + "wg show", + "wg show mesh0 dump" + ] + }, + "build": { + "artifacts": [ + { + "name": "tools-go", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/netcheck", + "binary": "netcheck", + "loads": [ + "netcheck" + ] + }, + { + "name": "tools-typescript", + "kind": "bundle", + "language": "typescript", + "entrypoints": [ + "tools/index.js" + ], + "loads": [ + "tools/index.js" + ] + } + ] + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/testdata/controller-f2d7bc94-build-agent.module.json b/modules/mesh-delivery-indexer/internal/index/testdata/controller-f2d7bc94-build-agent.module.json new file mode 100644 index 0000000..5a550ef --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/testdata/controller-f2d7bc94-build-agent.module.json @@ -0,0 +1,140 @@ +{ + "module": "build-agent", + "version": "1", + "slug": "agent", + "capabilities": [ + "container-runtime" + ], + "claims": [ + { + "name": "node-build-agent", + "scope": "node", + "serves": ["current", "kill", "pause", "resume"] + } + ], + "reads": [ + "mesh-delivery-gate.pulls", + "mesh-delivery.pulls" + ], + "requires": [ + "artifact-store", + { + "name": "npm-package-registry", + "optional": true + } + ], + "binds": { + "npm-package-registry": "${dir:mesh-state}/package-registry.json" + }, + "secrets": { + "npm-package-registry": "${dir:mesh-state}/package-registry.secret" + }, + "own-secrets": { + "broker": "${dir:mesh-state}/broker", + "genesis-package-registry": { + "path": "${dir:mesh-state}/genesis-package-registry.json", + "issued-by": "outside" + } + }, + "data": { + "own": [ + { + "id": "workspace", + "path": "${dir:workspace}", + "class": "cache", + "why": "a build's working copy, cloned again for every build" + }, + { + "id": "judge-cache", + "path": "${dir:judge-cache}", + "class": "cache", + "why": "the Go cache every merge check's judge is built with and the group judges kept per head; mounted into judge builds only, never into a check's containers (novox/hq ADR 0358 §6)" + } + ] + }, + "resources": [ + { + "id": "mesh-state", + "type": "directory", + "mode": "0700", + "place": "mesh" + }, + { + "id": "workspace", + "type": "directory", + "mode": "0700" + }, + { + "id": "judge-cache", + "type": "directory", + "mode": "0700" + }, + { + "id": "agent-env", + "type": "file", + "path": "${dir:mesh-state}/build-agent.env", + "mode": "0600", + "template": "files/build-agent.env" + }, + { + "id": "server", + "type": "container", + "name": "mesh-build-agent", + "artifact": "server", + "env-file": [ + "${dir:mesh-state}/build-agent.env" + ], + "volumes": [ + "${dir:mesh-state}:/run/mesh:ro", + "${dir:workspace}:${dir:workspace}", + "${dir:judge-cache}:${dir:judge-cache}", + "/var/run/docker.sock:/var/run/docker.sock" + ], + "restart-on": [ + "agent-env" + ], + "network": "host", + "health": { + "kind": "exec", + "command": "/usr/local/bin/mesh-builder ready", + "interval": "10s", + "timeout": "5s", + "looks": 3, + "grace": "60s" + }, + "busy": { + "command": "/usr/local/bin/mesh-builder busy" + } + } + ], + "build": { + "artifacts": [ + { + "name": "server", + "kind": "image", + "from": "Dockerfile", + "compiles": "cmd/mesh-builder", + "context": { + "seat": "git", + "repository": "novox/mesh-controller", + "ref": "main" + } + }, + { + "name": "files", + "kind": "archive", + "from": "files" + } + ], + "on": [ + { + "arg": "GO_BASE", + "image": "golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c" + }, + { + "arg": "ALPINE_BASE", + "image": "alpine@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc" + } + ] + } +} -- 2.54.0 From 543d91d62656cf1cc4c53c82430f3de49048d8b3 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 18:51:27 +0000 Subject: [PATCH 2/9] Read each module's manifest and lock files in the forge bridge, so the indexer gets them (issue 1448, design 47 R4c-2) --- .../cmd/mesh-delivery-forge-bridge/call.go | 32 ++++++++ .../cmd/mesh-delivery-forge-bridge/main.go | 6 +- modules/mesh-delivery-forge-bridge/go.mod | 2 +- modules/mesh-delivery-forge-bridge/go.sum | 8 +- .../internal/bridge/bridge.go | 77 ++++++++++++++++++- .../manifest_test.go | 2 +- .../mesh-delivery-forge-bridge/module.json | 3 +- 7 files changed, 114 insertions(+), 16 deletions(-) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go index f1c1461..9310a73 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "regexp" "strings" "time" @@ -89,3 +90,34 @@ func treesVia(ctx context.Context, call caller) bridge.Trees { return a.Root, a.Trees, nil } } + +// getFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea tools/index.ts gitea_get_file): +// arguments owner, repo, path and ref, answer {file: {path, ref, sha, size, content}}, the content decoded to text +// (client.ts getFile). A path the forge does not have fails with `Gitea API : 404 ` (client.ts request). +const getFileTool = "gitea.gitea_get_file" + +// notFound is how the forge's error says 404 (client.ts request: `Gitea API ${path}: ${status} ${text}`; client.ts:670 +// recognises it as /: 404 /). +var notFound = regexp.MustCompile(`: 404 `) + +// filesVia asks the forge for a file through call. A file the forge says is not there is bridge.ErrNotFound. +func filesVia(ctx context.Context, call caller) bridge.Files { + return func(owner, repo, path, ref string) ([]byte, error) { + raw, err := call(ctx, getFileTool, map[string]string{"owner": owner, "repo": repo, "path": path, "ref": ref}) + if err != nil { + if notFound.MatchString(err.Error()) { + return nil, fmt.Errorf("%w: %v", bridge.ErrNotFound, err) + } + return nil, err + } + var a struct { + File *struct { + Content *string `json:"content"` + } `json:"file"` + } + if err := json.Unmarshal(raw, &a); err != nil || a.File == nil || a.File.Content == nil { + return nil, fmt.Errorf("the answer of %s is not {file: {content}}: %v", getFileTool, err) + } + return []byte(*a.File.Content), nil + } +} diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go index d9dd739..0438f1c 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go @@ -115,8 +115,8 @@ type publisher interface { // handle turns one event into its message on FORGE and acknowledges it. An event it cannot read is terminated, never // redelivered, and the log names its subject and why. A message it could not write stays unacknowledged: the server // delivers it again (the server's ack wait is the only clock). -func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, logger *log.Logger) { - msg, err := bridge.Convert(m.Subject(), m.Data(), trees) +func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, logger *log.Logger) { + msg, err := bridge.Convert(m.Subject(), m.Data(), trees, files) switch { case errors.Is(err, bridge.ErrRetry): logger.Printf("%v; heard again in %s", err, retryDelay) @@ -238,7 +238,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // Bounded: the turn is held, and no sync or other event goes on without it. hctx, cancel := context.WithTimeout(ctx, syncDeadline) defer cancel() - handle(hctx, m, rec, treesVia(hctx, call), logger) + handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), logger) }) }, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) })) diff --git a/modules/mesh-delivery-forge-bridge/go.mod b/modules/mesh-delivery-forge-bridge/go.mod index f3a0279..25936a8 100644 --- a/modules/mesh-delivery-forge-bridge/go.mod +++ b/modules/mesh-delivery-forge-bridge/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-forge-bridge/go.sum b/modules/mesh-delivery-forge-bridge/go.sum index 856a7bd..0e67d23 100644 --- a/modules/mesh-delivery-forge-bridge/go.sum +++ b/modules/mesh-delivery-forge-bridge/go.sum @@ -1,9 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019195003-4580d80a32d9 h1:XL1dbtSleGAixg6FX4gEnSG/AJxRG/tWw6/7U1RuzHc= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019195003-4580d80a32d9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69 h1:kvtV/SPsdRVUL/BZ3xxGS6U8YrXy/t7TRBjVrD6VLw8= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef h1:xy3wuGWJpqMAXYdXolocYG6w9xnWqBltVUffoaziIKo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960 h1:FCId3IuRScG03X7Gx9vfKfGNZH9jWnfx9RChfJ3eWfo= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go index e845d91..b1c74f8 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go @@ -2,6 +2,8 @@ package bridge import ( + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -34,6 +36,17 @@ const Trunk = "main" // (the root directory spelled ""), as the forge's gitea_commit_trees answers (issue 1234/AC4). type Trees func(owner, repo, sha string) (root string, trees map[string]string, err error) +// ErrNotFound says the forge has no file at that path and ref (the forge's "404", issue 1448). +var ErrNotFound = errors.New("the forge has no such file") + +// Files asks the forge for one file's bytes at a ref (a commit), as the forge's gitea_get_file answers; a file the forge +// says is not there is ErrNotFound, any other failure any other error (issue 1448). +type Files func(owner, repo, path, ref string) ([]byte, error) + +// LockFiles are the lock files read beside a module's manifest, by name relative to its directory (novox/hq ADR 0465 +// clause 1: go.sum for a Go module, package-lock.json for a Node one). +var LockFiles = []string{"go.sum", "package-lock.json"} + // Message is a message for FORGE: the event name (forge...) and its JSON body. type Message struct { Event string @@ -72,7 +85,7 @@ func unreadable(subject, why string, args ...any) error { // is an ErrUnreadable error naming the subject and why; an event it writes nothing for is ErrNotHandled. A merge into // the trunk asks trees once and becomes a merged message; a merge into any other branch is a closed pull message, as the // pull request left its group. A forge that cannot say is ErrRetry; a merge with no module.json is ErrNoModules. -func Convert(subject string, data []byte, trees Trees) (Message, error) { +func Convert(subject string, data []byte, trees Trees, files Files) (Message, error) { var action string switch subject { case UpdatedSubject: @@ -99,13 +112,17 @@ func Convert(subject string, data []byte, trees Trees) (Message, error) { return Message{}, unreadable(subject, "no branch") } if subject == MergedSubject && p.Base == Trunk { - return merged(subject, p, trees) + return merged(subject, p, trees, files) } f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number, Head: p.HeadSHA, Action: action} // The directories are the forge's word only when it said it could read them (module_dirs_said). if action == delivery.PullUpdated && p.ModuleDirsSaid { f.Dirs, f.DirsKnown = p.ModuleDirs, true + var err error + if f.Manifests, f.Locks, err = inputsOf(p.Owner, p.Repo, p.HeadSHA, p.ModuleDirs, files); err != nil { + return Message{}, fmt.Errorf("%w: %s: the manifests of %s/%s at the head %s: %v", ErrRetry, subject, p.Owner, p.Repo, p.HeadSHA, err) + } } event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull) if err != nil { @@ -118,8 +135,51 @@ func Convert(subject string, data []byte, trees Trees) (Message, error) { return Message{Event: event, Data: body}, nil } +// inputsOf reads, at ref, the module.json of each directory (the root spelled "") and the lock files beside it +// (LockFiles): the manifests as the forge answered them and the lock files' hex sha256 by name. A manifest the forge +// does not have is an error, since the directory was said to hold one; a lock file it does not have is left out; any other +// failure is an error (issue 1448). +func inputsOf(owner, repo, ref string, dirs []string, files Files) (manifests map[string]string, locks map[string]map[string]string, err error) { + manifests = map[string]string{} + locks = map[string]map[string]string{} + at := func(dir, name string) string { + if dir == "" { + return name + } + return dir + "/" + name + } + for _, dir := range dirs { + raw, err := files(owner, repo, at(dir, "module.json"), ref) + if err != nil { + return nil, nil, fmt.Errorf("%s: %w", at(dir, "module.json"), err) + } + manifests[dir] = string(raw) + for _, name := range LockFiles { + raw, err := files(owner, repo, at(dir, name), ref) + if errors.Is(err, ErrNotFound) { + continue + } + if err != nil { + return nil, nil, fmt.Errorf("%s: %w", at(dir, name), err) + } + sum := sha256.Sum256(raw) + if locks[dir] == nil { + locks[dir] = map[string]string{} + } + locks[dir][name] = hex.EncodeToString(sum[:]) + } + } + if len(manifests) == 0 { + manifests = nil + } + if len(locks) == 0 { + locks = nil + } + return manifests, locks, nil +} + // merged is the merged message of a merge into the trunk: the trees of the merge commit, asked once. -func merged(subject string, p pull, trees Trees) (Message, error) { +func merged(subject string, p pull, trees Trees, files Files) (Message, error) { if !sha40.MatchString(p.MergeSHA) { return Message{}, unreadable(subject, "the merge commit %q is not 40 hex digits", p.MergeSHA) } @@ -131,12 +191,21 @@ func merged(subject string, p pull, trees Trees) (Message, error) { if len(dirs) == 0 { return Message{}, fmt.Errorf("%w: %s: %s at the merge commit %s", ErrNoModules, subject, repository, p.MergeSHA) } + names := make([]string, 0, len(dirs)) + for dir := range dirs { + names = append(names, dir) + } + slices.Sort(names) + manifests, locks, err := inputsOf(p.Owner, p.Repo, p.MergeSHA, names, files) + if err != nil { + return Message{}, fmt.Errorf("%w: %s: the manifests of %s at the merge commit %s: %v", ErrRetry, subject, repository, p.MergeSHA, err) + } event, err := delivery.ForgeEvent(repository, delivery.ForgeMerged) if err != nil { return Message{}, unreadable(subject, "%v", err) } body, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number, - Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs}) + Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks}) if err != nil { return Message{}, unreadable(subject, "%v", err) } diff --git a/modules/mesh-delivery-forge-bridge/manifest_test.go b/modules/mesh-delivery-forge-bridge/manifest_test.go index f1bb6ea..d6682cc 100644 --- a/modules/mesh-delivery-forge-bridge/manifest_test.go +++ b/modules/mesh-delivery-forge-bridge/manifest_test.go @@ -93,7 +93,7 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1 t.Errorf("reads = %v, want %v", m.Reads, want) } if want := []string{"gitea.gitea_add_comment", "gitea.gitea_commit_status", "gitea.gitea_commit_trees", "gitea.gitea_delivery_view", - "gitea.gitea_merge_pull_request", "gitea.gitea_note_append", "gitea.gitea_open_pulls"}; !reflect.DeepEqual(sorted(m.Invokes), want) { + "gitea.gitea_get_file", "gitea.gitea_merge_pull_request", "gitea.gitea_note_append", "gitea.gitea_open_pulls"}; !reflect.DeepEqual(sorted(m.Invokes), want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { diff --git a/modules/mesh-delivery-forge-bridge/module.json b/modules/mesh-delivery-forge-bridge/module.json index 641fa86..25b68be 100644 --- a/modules/mesh-delivery-forge-bridge/module.json +++ b/modules/mesh-delivery-forge-bridge/module.json @@ -25,7 +25,8 @@ "gitea.gitea_note_append", "gitea.gitea_add_comment", "gitea.gitea_open_pulls", - "gitea.gitea_commit_trees" + "gitea.gitea_commit_trees", + "gitea.gitea_get_file" ], "own-secrets": { "broker": "${dir:state}/broker" -- 2.54.0 From 41225bf4599fe1ea0b0faec0408467deb344430f Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 18:52:27 +0000 Subject: [PATCH 3/9] Test the forge bridge's file reads against the stand-in forge and the bus (issue 1448, design 47 R4c-2) --- .../integration_test.go | 83 +++++++++++++++++-- .../mesh-delivery-forge-bridge/main_test.go | 69 +++++++++++++-- 2 files changed, 138 insertions(+), 14 deletions(-) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go index aa86ce5..3281f5c 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go @@ -4,6 +4,8 @@ package main import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "io" @@ -29,8 +31,8 @@ import ( // bucket performed is the bridge's own (its state), and the five forge tools of the gate's acts are invoked (issue 1341, 1342). // // The lists are what mesh-controller's broker.PermissionsFor answers (internal/broker/nats.go:577) for a module principal -// with the manifest of module.json: consumes the three gitea.pull events, invokes gitea.gitea_open_pulls and -// gitea.gitea_commit_trees, state heads, and the stream forge; `**` is written `>` on the wire (modstreams.go +// with the manifest of module.json: consumes the three gitea.pull events, invokes gitea.gitea_open_pulls, +// gitea.gitea_commit_trees and gitea.gitea_get_file, state heads, and the stream forge; `**` is written `>` on the wire (modstreams.go // WirePattern). The consumer's durable is _ on EVENTS (nats.go consumerDurable). The controller's own // user keeps all rights, as it has. const busConf = ` @@ -59,6 +61,10 @@ accounts { "mesh.mod.gitea.call.gitea_open_pulls.n1~mesh-delivery-forge-bridge" "mesh.mod.gitea.tool.gitea_commit_trees" "mesh.mod.gitea.tool.gitea_commit_trees.*" + "mesh.mod.gitea.call.gitea_get_file.*.n1~mesh-delivery-forge-bridge" + "mesh.mod.gitea.call.gitea_get_file.n1~mesh-delivery-forge-bridge" + "mesh.mod.gitea.tool.gitea_get_file" + "mesh.mod.gitea.tool.gitea_get_file.*" "mesh.mod.gitea.tool.gitea_open_pulls" "mesh.mod.gitea.tool.gitea_open_pulls.*" "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_performed.>" @@ -160,6 +166,21 @@ type bed struct { ctx context.Context done <-chan error reply func(string) + // files sets what the stand-in forge answers to gitea_get_file from then on; it is asked with the tool's arguments. + files func(func(owner, repo, path, ref string) (string, bool)) +} + +// getFileReply is the reply envelope for gitea_get_file as mesh-catalog modules/gitea tools/index.ts and client.ts getFile +// answer it ({file: {path, ref, sha, size, content}}), and for a path the forge does not have the error client.ts request +// writes: `Gitea API : 404 `. +func getFileReply(path, ref, content string, found bool) []byte { + if !found { + b, _ := json.Marshal(map[string]any{"error": "Gitea API /repos/novox/mesh-delivery/contents/" + path + "?ref=" + ref + `: 404 {"message":"GetContentsOrList","url":"https://forge.example/api/swagger"}`}) + return b + } + b, _ := json.Marshal(map[string]any{"result": map[string]any{"file": map[string]any{"path": path, "ref": ref, "sha": "0123456789abcdef0123456789abcdef01234567", + "size": len(content), "content": content}}, "node": "n0"}) + return b } // startBed is startBridge with a step between the streams and the consumer: prepare writes what the streams hold before @@ -222,6 +243,26 @@ func startBed(t *testing.T, openReply string, prepare func(ctx context.Context, t.Fatal(err) } + // A stand-in answers the forge's gitea_get_file: by default a module.json of {} at every path and no other file. + var fmu sync.Mutex + filesOf := func(_, _, path, _ string) (string, bool) { return "{}", strings.HasSuffix(path, "module.json") } + getFile, err := ctl.Subscribe("mesh.mod.gitea.tool.gitea_get_file", func(m *nats.Msg) { + var a struct{ Owner, Repo, Path, Ref string } + _ = json.Unmarshal(m.Data, &a) + fmu.Lock() + f := filesOf + fmu.Unlock() + content, found := f(a.Owner, a.Repo, a.Path, a.Ref) + _ = m.Respond(getFileReply(a.Path, a.Ref, content, found)) + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = getFile.Unsubscribe() }) + if err := ctl.Flush(); err != nil { + t.Fatal(err) + } + broker := filepath.Join(t.TempDir(), "broker") cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n1", Module: "mesh-delivery-forge-bridge", User: "n1.mesh-delivery-forge-bridge", Password: "pw"}) if err := os.WriteFile(broker, cred, 0o600); err != nil { @@ -232,7 +273,8 @@ func startBed(t *testing.T, openReply string, prepare func(ctx context.Context, done := make(chan error, 1) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(stop) - return &bed{srv: srv, ctl: ctl, js: js, ctx: ctx, done: done, reply: func(s string) { mu.Lock(); current = s; mu.Unlock() }} + return &bed{srv: srv, ctl: ctl, js: js, ctx: ctx, done: done, reply: func(s string) { mu.Lock(); current = s; mu.Unlock() }, + files: func(f func(owner, repo, path, ref string) (string, bool)) { fmu.Lock(); filesOf = f; fmu.Unlock() }} } // TestBridgeWritesPullMessages starts the program the way the node does (its entry, run, with MESH_BROKER_FILE). @@ -270,7 +312,7 @@ func TestBridgeWritesPullMessages(t *testing.T) { // 1235/AC2 t.Fatal(err) } want := delivery.Forge{V: delivery.Version, Kind: "pull", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, - Action: "updated", Dirs: []string{"modules/a"}, DirsKnown: true} + Action: "updated", Dirs: []string{"modules/a"}, DirsKnown: true, Manifests: map[string]string{"modules/a": "{}"}} if !delivery.SameForge(f, want) { t.Errorf("message = %+v, want %+v", f, want) } @@ -295,8 +337,31 @@ const merge = "89abcdef0123456789abcdef0123456789abcdef" // TestBridgeWritesMergedMessage has a stand-in answer the forge's gitea_commit_trees on its tool subject // (mesh-control internal/broker/nats.go invokedSubjects: mesh.mod..tool.), in the reply envelope of // mesh-sdk go/process/call.go ({result, error, node}). -func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4 - ctl, js, ctx, _ := startBridge(t, `{"result":{"pulls":[]}}`) +func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4, 1448/AC5 + bed := startBed(t, `{"result":{"pulls":[]}}`, nil) + ctl, js, ctx := bed.ctl, bed.js, bed.ctx + // The stand-in forge's files are two real files of this repository (this module's module.json and go.sum, read from the + // working tree), in gitea_get_file's real answer shape: the root has a manifest only, modules/a a manifest and go.sum. + manifest, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + sum, err := os.ReadFile("../../go.sum") + if err != nil { + t.Fatal(err) + } + bed.files(func(_, _, path, ref string) (string, bool) { + if ref != merge { + t.Errorf("the forge was asked for %s at %s, want the merge commit %s", path, ref, merge) + } + switch path { + case "module.json", "modules/a/module.json": + return string(manifest), true + case "modules/a/go.sum": + return string(sum), true + } + return "", false + }) var asked chan []byte = make(chan []byte, 4) sub, err := ctl.Subscribe("mesh.mod.gitea.tool.gitea_commit_trees", func(m *nats.Msg) { asked <- m.Data @@ -342,7 +407,9 @@ func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4 t.Fatal(err) } want := delivery.Forge{V: delivery.Version, Kind: "merged", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, - Merge: merge, Root: "r0", Trees: map[string]string{"": "t0", "modules/a": "t1"}} + Merge: merge, Root: "r0", Trees: map[string]string{"": "t0", "modules/a": "t1"}, + Manifests: map[string]string{"": string(manifest), "modules/a": string(manifest)}, + Locks: map[string]map[string]string{"modules/a": {"go.sum": hex.EncodeToString(sha256Sum(sum))}}} if !delivery.SameForge(f, want) { t.Errorf("message = %+v, want %+v", f, want) } @@ -637,3 +704,5 @@ func forgeMerged(t *testing.T, b *bed) []string { } return out } + +func sha256Sum(b []byte) []byte { s := sha256.Sum256(b); return s[:] } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go index 123bd49..b74d8da 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "log" + "slices" "strings" "testing" "time" @@ -14,6 +15,8 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-forge-bridge/internal/bridge" ) const head = "0123456789abcdef0123456789abcdef01234567" @@ -64,7 +67,7 @@ func body(m map[string]any) []byte { func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit part; the bus is the integration test) var logs bytes.Buffer m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{} - handle(context.Background(), m, pub, nil, log.New(&logs, "", 0)) + handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0)) if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull" { t.Fatalf("published on %v", pub.subjects) } @@ -80,7 +83,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5 var logs bytes.Buffer m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(map[string]any{"head_sha": "short"})}, &fakePub{} - handle(context.Background(), m, pub, nil, log.New(&logs, "", 0)) + handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0)) if len(pub.subjects) != 0 { t.Errorf("a message was written: %v", pub.subjects) } @@ -95,7 +98,7 @@ func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5 func TestUnwrittenMessageIsNotAcked(t *testing.T) { // 1235 (a failed write is redelivered by the server, not retried here) var logs bytes.Buffer m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")} - handle(context.Background(), m, pub, nil, log.New(&logs, "", 0)) + handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0)) if m.acked != 0 || m.termd != 0 { t.Errorf("acked %d terminated %d, want neither", m.acked, m.termd) } @@ -117,7 +120,7 @@ func TestTreesRefusedLeavesTheEventForTenSecondsAndLogs(t *testing.T) { // 1236/ call := func(context.Context, string, any) (json.RawMessage, error) { return nil, errors.New("the tree listing of novox/mesh-delivery was cut") } - handle(context.Background(), m, pub, treesVia(context.Background(), call), log.New(&logs, "", 0)) + handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), log.New(&logs, "", 0)) if len(pub.subjects) != 0 { t.Errorf("a message was written: %v", pub.subjects) } @@ -138,11 +141,14 @@ func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/ var args []any call := func(_ context.Context, key string, body any) (json.RawMessage, error) { keys, args = append(keys, key), append(args, body) + if key == "gitea.gitea_get_file" { // the manifests, 1448; the trees are what this test is about + return json.RawMessage(`{"file":{"path":"p","sha":"0123","size":2,"content":"{}"}}`), nil + } return json.RawMessage(`{"root":"r0","trees":{"":"t0","modules/a":"t1"}}`), nil } - handle(context.Background(), m, pub, treesVia(context.Background(), call), log.New(&logs, "", 0)) - if len(keys) != 1 || keys[0] != "gitea.gitea_commit_trees" { - t.Fatalf("calls %v", keys) + handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), log.New(&logs, "", 0)) + if len(keys) == 0 || keys[0] != "gitea.gitea_commit_trees" || slices.Index(keys[1:], "gitea.gitea_commit_trees") >= 0 { + t.Fatalf("calls %v, want the trees once and first", keys) } if got, _ := json.Marshal(args[0]); string(got) != `{"owner":"novox","repo":"mesh-delivery","sha":"89abcdef0123456789abcdef0123456789abcdef"}` { t.Errorf("arguments %s", got) @@ -171,3 +177,52 @@ func TestAnAnswerWithoutARootIsNotUsed(t *testing.T) { // 1236 (a shape the tool t.Error("an answer that is not an object was taken") } } + +// The answer of gitea_get_file as mesh-catalog modules/gitea tools/index.ts and client.ts getFile write it, and the +// error of a path the forge does not have as client.ts request writes it (`Gitea API ${path}: ${status} ${text}`). +const getFileAnswer = `{"file":{"path":"modules/a/go.sum","ref":"89abcdef0123456789abcdef0123456789abcdef","sha":"0123","size":4,"content":"x y\n"}}` +const getFile404 = "Gitea API /repos/novox/mesh-delivery/contents/modules/a/go.sum?ref=89abcdef: 404 {\"message\":\"GetContentsOrList\",\"url\":\"https://forge.example/api/swagger\"}" + +func TestFilesViaAsksByOwnerRepoPathRefAndTakesTheContent(t *testing.T) { // 1448/AC4 + var keys []string + var args []any + call := func(_ context.Context, key string, body any) (json.RawMessage, error) { + keys, args = append(keys, key), append(args, body) + return json.RawMessage(getFileAnswer), nil + } + got, err := filesVia(context.Background(), call)("novox", "mesh-delivery", "modules/a/go.sum", "89abcdef0123456789abcdef0123456789abcdef") + if err != nil || string(got) != "x y\n" { + t.Fatalf("got %q err %v", got, err) + } + if len(keys) != 1 || keys[0] != "gitea.gitea_get_file" { + t.Fatalf("calls %v", keys) + } + if b, _ := json.Marshal(args[0]); string(b) != `{"owner":"novox","path":"modules/a/go.sum","ref":"89abcdef0123456789abcdef0123456789abcdef","repo":"mesh-delivery"}` { + t.Errorf("arguments %s", b) + } +} + +func TestFilesViaTellsNotFoundFromOtherErrorsAndOtherShapes(t *testing.T) { // 1448/AC3 + call := func(err error, raw string) caller { + return func(context.Context, string, any) (json.RawMessage, error) { return json.RawMessage(raw), err } + } + if _, err := filesVia(context.Background(), call(errors.New(getFile404), ""))("o", "r", "p", "s"); !errors.Is(err, bridge.ErrNotFound) { + t.Errorf("a 404 is %v, want ErrNotFound", err) + } + if _, err := filesVia(context.Background(), call(errors.New("Gitea API /repos/o/r/contents/p: 502 bad gateway"), ""))("o", "r", "p", "s"); err == nil || errors.Is(err, bridge.ErrNotFound) { + t.Errorf("a 502 is %v, want an error that is not ErrNotFound", err) + } + for _, raw := range []string{`{"file":null}`, `{}`, `[1]`, `{"file":{"path":"p"}}`} { + if _, err := filesVia(context.Background(), call(nil, raw))("o", "r", "p", "s"); err == nil || errors.Is(err, bridge.ErrNotFound) { + t.Errorf("the answer %s gave %v, want an error", raw, err) + } + } +} + +// someFiles is a forge that has a module.json at every path and no other file, for the tests that are not about the files. +func someFiles(_, _, path, _ string) ([]byte, error) { + if strings.HasSuffix(path, "module.json") { + return []byte("{}"), nil + } + return nil, bridge.ErrNotFound +} -- 2.54.0 From 3c1e7ad4642a2b81b65585d9eb72750648340564 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 18:53:07 +0000 Subject: [PATCH 4/9] Test the bridge's manifest and lock file conversion (issue 1448, design 47 R4c-2) --- .../internal/bridge/bridge_test.go | 28 ++-- .../internal/bridge/files_test.go | 142 ++++++++++++++++++ 2 files changed, 156 insertions(+), 14 deletions(-) create mode 100644 modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge_test.go index 60a7ca5..c2424a5 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge_test.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge_test.go @@ -47,7 +47,7 @@ const updatedSubject = "mesh.mod.gitea.event.pull.updated" const closedSubject = "mesh.mod.gitea.event.pull.closed" func TestUpdatedBecomesAPullMessage(t *testing.T) { // 1235/AC2 (the conversion; the bus is the integration test) - msg, err := Convert(updatedSubject, updated(nil), nil) + msg, err := Convert(updatedSubject, updated(nil), nil, anyFiles) if err != nil { t.Fatal(err) } @@ -59,14 +59,14 @@ func TestUpdatedBecomesAPullMessage(t *testing.T) { // 1235/AC2 (the conversion; t.Fatal(err) } want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, - Head: head, Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true} + Head: head, Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true, Manifests: map[string]string{"modules/a": "{}"}} if !delivery.SameForge(f, want) { t.Errorf("message = %+v, want %+v", f, want) } } func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3 - msg, err := Convert(updatedSubject, updated(map[string]any{"module_dirs": nil, "module_dirs_said": nil}), nil) + msg, err := Convert(updatedSubject, updated(map[string]any{"module_dirs": nil, "module_dirs_said": nil}), nil, anyFiles) if err != nil { t.Fatal(err) } @@ -75,7 +75,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3 t.Errorf("dirs = %v known = %v, want not known", f.Dirs, f.DirsKnown) } // Said and empty is known and empty, not unknown. - msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs": []string{}, "module_dirs_said": true}), nil) + msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs": []string{}, "module_dirs_said": true}), nil, anyFiles) if err != nil { t.Fatal(err) } @@ -84,7 +84,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3 t.Errorf("dirs = %v known = %v, want known and empty", f.Dirs, f.DirsKnown) } // Dirs listed without the forge saying it could read them are not trusted. - msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs_said": nil}), nil) + msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs_said": nil}), nil, anyFiles) if err != nil { t.Fatal(err) } @@ -94,7 +94,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3 } func TestClosedBecomesAClosedPullMessage(t *testing.T) { // 1235/AC4 - msg, err := Convert(closedSubject, closed(), nil) + msg, err := Convert(closedSubject, closed(), nil, anyFiles) if err != nil { t.Fatal(err) } @@ -130,7 +130,7 @@ func TestUnreadableEventsWriteNothing(t *testing.T) { // 1235/AC5 "closed no head": {closedSubject, mustDrop(closed(), "head_sha"), "head"}, "closed no repo": {closedSubject, mustDrop(closed(), "repo"), "repo"}, } { - msg, err := Convert(c.subject, c.data, nil) + msg, err := Convert(c.subject, c.data, nil, anyFiles) if err == nil { t.Errorf("%s: a message %q was made", name, msg.Event) continue @@ -157,7 +157,7 @@ func mustDrop(body []byte, key string) []byte { func TestOtherSubjectsAreNotHandled(t *testing.T) { // 1235 for _, s := range []string{"mesh.mod.gitea.event.something.else", "mesh.mod.other.event.pull.updated"} { - if _, err := Convert(s, updated(nil), nil); !errors.Is(err, ErrNotHandled) { + if _, err := Convert(s, updated(nil), nil, anyFiles); !errors.Is(err, ErrNotHandled) { t.Errorf("%s: err = %v, want ErrNotHandled", s, err) } } @@ -200,7 +200,7 @@ func treesOf(calls *[]call, root string, trees map[string]string, err error) Tre func TestMergeIntoMainBecomesAMergedMessage(t *testing.T) { // 1236/AC1 var calls []call answer := map[string]string{"modules/a": "t1", "modules/b": "t2"} - msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", answer, nil)) + msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", answer, nil), anyFiles) if err != nil { t.Fatal(err) } @@ -212,7 +212,7 @@ func TestMergeIntoMainBecomesAMergedMessage(t *testing.T) { // 1236/AC1 t.Fatal(err) } want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, - Head: head, Merge: merge, Root: "r0", Trees: answer} + Head: head, Merge: merge, Root: "r0", Trees: answer, Manifests: map[string]string{"modules/a": "{}", "modules/b": "{}"}} if !delivery.SameForge(f, want) { t.Errorf("message = %+v, want %+v", f, want) } @@ -223,7 +223,7 @@ func TestMergeIntoMainBecomesAMergedMessage(t *testing.T) { // 1236/AC1 func TestMergeIntoAnotherBranchIsAClosedPull(t *testing.T) { // 1236/AC2 var calls []call - msg, err := Convert(mergedSubject, mergedBody(map[string]any{"base": "feat/y"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil)) + msg, err := Convert(mergedSubject, mergedBody(map[string]any{"base": "feat/y"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil), anyFiles) if err != nil { t.Fatal(err) } @@ -243,7 +243,7 @@ func TestMergeIntoAnotherBranchIsAClosedPull(t *testing.T) { // 1236/AC2 func TestTreesRefusedWritesNothingAndSaysWhy(t *testing.T) { // 1236/AC3 (the conversion; the delay is in main_test) var calls []call - msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "", nil, errors.New("the listing for novox/mesh-delivery was cut"))) + msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "", nil, errors.New("the listing for novox/mesh-delivery was cut")), anyFiles) if !errors.Is(err, ErrRetry) || msg.Data != nil { t.Fatalf("err = %v data = %s, want ErrRetry and no message", err, msg.Data) } @@ -256,7 +256,7 @@ func TestTreesRefusedWritesNothingAndSaysWhy(t *testing.T) { // 1236/AC3 (the co func TestNoModuleJSONAnywhereWritesNoMergedMessage(t *testing.T) { // 1236 (notes: a repository with no module.json) var calls []call - msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{}, nil)) + msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{}, nil), anyFiles) if !errors.Is(err, ErrNoModules) || msg.Data != nil { t.Errorf("err = %v data = %s", err, msg.Data) } @@ -264,7 +264,7 @@ func TestNoModuleJSONAnywhereWritesNoMergedMessage(t *testing.T) { // 1236 (note func TestMergedWithoutAMergeCommitIsUnreadable(t *testing.T) { // 1236 var calls []call - _, err := Convert(mergedSubject, mergedBody(map[string]any{"merge_commit_sha": "abc"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil)) + _, err := Convert(mergedSubject, mergedBody(map[string]any{"merge_commit_sha": "abc"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil), anyFiles) if !errors.Is(err, ErrUnreadable) || len(calls) != 0 { t.Errorf("err = %v calls = %v", err, calls) } diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go new file mode 100644 index 0000000..67e33b1 --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go @@ -0,0 +1,142 @@ +package bridge + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "os" + "reflect" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The bytes the forge answers in these tests are two real files of this repository, read from the working tree: the +// module.json and the go.sum of modules/mesh-delivery-forge-bridge (two directories up). The forge's own answer shape +// is mesh-catalog modules/gitea tools/index.ts gitea_get_file; the bridge sees only the bytes (issue 1448). +func real(t *testing.T, name string) []byte { + t.Helper() + b, err := os.ReadFile("../../" + name) + if err != nil { + t.Fatal(err) + } + return b +} + +func hexSum(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) } + +type fileCall struct{ owner, repo, path, ref string } + +// forgeFiles answers as the forge does for a path: its bytes, ErrNotFound for a path in none, the error in errs. +type forgeFiles struct { + have map[string][]byte + errs map[string]error + calls []fileCall +} + +func (f *forgeFiles) get(owner, repo, path, ref string) ([]byte, error) { + f.calls = append(f.calls, fileCall{owner, repo, path, ref}) + if err, ok := f.errs[path]; ok { + return nil, err + } + if b, ok := f.have[path]; ok { + return b, nil + } + return nil, ErrNotFound +} + +// anyFiles is a forge that has a module.json at every path and no lock file, for tests that are not about the files. +func anyFiles(owner, repo, path, ref string) ([]byte, error) { + if strings.HasSuffix(path, "module.json") { + return []byte("{}"), nil + } + return nil, ErrNotFound +} + +func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1 + manifest, sum := real(t, "module.json"), real(t, "go.sum") + forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest, "modules/a/go.sum": sum, "modules/b/module.json": []byte(`{"module":"b"}`)}} + var calls []call + msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1", "modules/b": "t2"}, nil), forge.get) + if err != nil { + t.Fatal(err) + } + f, err := delivery.DecodeForge(msg.Data) + if err != nil { + t.Fatal(err) + } + if want := map[string]string{"modules/a": string(manifest), "modules/b": `{"module":"b"}`}; !reflect.DeepEqual(f.Manifests, want) { + t.Errorf("manifests = %v", f.Manifests) + } + if want := map[string]map[string]string{"modules/a": {"go.sum": hexSum(sum)}}; !reflect.DeepEqual(f.Locks, want) { + t.Errorf("locks = %v, want %v", f.Locks, want) + } +} + +func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2 + manifest := real(t, "module.json") + forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest}} + msg, err := Convert(updatedSubject, updated(nil), nil, forge.get) + if err != nil { + t.Fatal(err) + } + f, _ := delivery.DecodeForge(msg.Data) + if !reflect.DeepEqual(f.Manifests, map[string]string{"modules/a": string(manifest)}) || len(f.Locks) != 0 { + t.Errorf("manifests %v locks %v", f.Manifests, f.Locks) + } + if len(forge.calls) == 0 || forge.calls[0].ref != head { + t.Errorf("calls %v, want the head %s", forge.calls, head) + } + // Dirs not known: neither field, and the forge is not asked. + forge = &forgeFiles{} + msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs": nil, "module_dirs_said": nil}), nil, forge.get) + if err != nil { + t.Fatal(err) + } + f, _ = delivery.DecodeForge(msg.Data) + if f.Manifests != nil || f.Locks != nil || len(forge.calls) != 0 { + t.Errorf("manifests %v locks %v calls %v", f.Manifests, f.Locks, forge.calls) + } +} + +func TestAForgeErrorOrAMissingManifestIsRetryAndALockNotFoundIsLeftOut(t *testing.T) { // 1448/AC3 + boom := errors.New("Gitea API /repos/novox/mesh-delivery/contents/modules/a/go.sum?ref=x: 502 bad gateway") + trees := func() Trees { var c []call; return treesOf(&c, "r0", map[string]string{"modules/a": "t1"}, nil) } + for name, forge := range map[string]*forgeFiles{ + "a manifest error": {errs: map[string]error{"modules/a/module.json": boom}}, + "a manifest not found": {}, + "a lock file error": {have: map[string][]byte{"modules/a/module.json": []byte("{}")}, errs: map[string]error{"modules/a/go.sum": boom}}, + "a second lock error": {have: map[string][]byte{"modules/a/module.json": []byte("{}")}, errs: map[string]error{"modules/a/package-lock.json": boom}}, + } { + msg, err := Convert(mergedSubject, mergedBody(nil), trees(), forge.get) + if !errors.Is(err, ErrRetry) || msg.Data != nil { + t.Errorf("%s: err = %v data = %s, want ErrRetry and no message", name, err, msg.Data) + } + _, err = Convert(updatedSubject, updated(nil), nil, forge.get) + if !errors.Is(err, ErrRetry) { + t.Errorf("%s (pull): err = %v, want ErrRetry", name, err) + } + } + forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": []byte("{}")}} + msg, err := Convert(mergedSubject, mergedBody(nil), trees(), forge.get) + if err != nil { + t.Fatal(err) + } + if f, _ := delivery.DecodeForge(msg.Data); f.Locks != nil { + t.Errorf("locks = %v, want none for lock files the forge does not have", f.Locks) + } +} + +func TestTheFilesAreAskedAtTheMergeCommitByOwnerRepoPathAndRef(t *testing.T) { // 1448/AC4 (the call's arguments; the manifest is in manifest_test) + forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": []byte("{}")}} + var calls []call + if _, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get); err != nil { + t.Fatal(err) + } + want := []fileCall{{"novox", "mesh-delivery", "modules/a/module.json", merge}, {"novox", "mesh-delivery", "modules/a/go.sum", merge}, + {"novox", "mesh-delivery", "modules/a/package-lock.json", merge}} + if !reflect.DeepEqual(forge.calls, want) { + t.Errorf("calls = %v, want %v", forge.calls, want) + } +} -- 2.54.0 From ba7ce5588712c59b943a29e5df47aad69eec105f Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 18:53:40 +0000 Subject: [PATCH 5/9] Give the indexer a manifests bucket and the SDK that knows the new keys (issue 1449, design 47 R4c-2) --- .../cmd/mesh-delivery-indexer/main.go | 6 +++++- modules/mesh-delivery-indexer/go.mod | 2 +- modules/mesh-delivery-indexer/go.sum | 10 ++-------- modules/mesh-delivery-indexer/manifest_test.go | 2 +- modules/mesh-delivery-indexer/module.json | 1 + 5 files changed, 10 insertions(+), 11 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index a64e005..ca8db63 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -251,6 +251,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } + manifests, err := bucket(delivery.ManifestsBucket) + if err != nil { + return err + } builds, err := bucket(delivery.BuildsBucket) if err != nil { return err @@ -270,7 +274,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro errs := make(chan error, 1) go func() { errs <- follow(ctx, w, each) }() - ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Builds: known, Logf: logger.Printf} + ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf} // The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key // main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does // not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate. diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index 8b8d6d1..b1780db 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-indexer go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-indexer/go.sum b/modules/mesh-delivery-indexer/go.sum index 962db0d..0e67d23 100644 --- a/modules/mesh-delivery-indexer/go.sum +++ b/modules/mesh-delivery-indexer/go.sum @@ -1,11 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019203925-84a764372337 h1:cZyFnvDiBpnNx8ifSUG9WA61OHsn7PC2ml2s4eD7AkA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019203925-84a764372337/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019223929-58373d65b47a h1:Vjk2Kq+g/Q86N8CCHq3OatJIJ0JSy6Kh980ekGxNLmE= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019223929-58373d65b47a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69 h1:kvtV/SPsdRVUL/BZ3xxGS6U8YrXy/t7TRBjVrD6VLw8= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef h1:xy3wuGWJpqMAXYdXolocYG6w9xnWqBltVUffoaziIKo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960 h1:FCId3IuRScG03X7Gx9vfKfGNZH9jWnfx9RChfJ3eWfo= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 45b62cf..84d4dfc 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -76,7 +76,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1 if m.Module != "mesh-delivery-indexer" { t.Errorf("module = %q", m.Module) } - if want := []state{{Name: "candidates", History: 10}, {Name: "modules", History: 20}}; !reflect.DeepEqual(m.State, want) { + if want := []state{{Name: "candidates", History: 10}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } if want := []string{"mesh-controller.builds", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 335599f..814948c 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -7,6 +7,7 @@ ], "state": [ { "name": "candidates", "history": 10 }, + { "name": "manifests", "history": 1 }, { "name": "modules", "history": 20 } ], "reads": [ -- 2.54.0 From a75d9cc851212e7a1bae9ee027b38830284002d5 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 18:54:28 +0000 Subject: [PATCH 6/9] Keep manifests and say why no new-form fingerprint exists, so the group check can start from facts (issues 1449, 1450, 1444, 1451, design 47 R4c-2) --- .../internal/index/candidates.go | 67 ++++++++-- .../internal/index/inputs.go | 115 ++++++++++++++++++ .../internal/index/manifests.go | 83 +++++++++++++ .../internal/index/modules.go | 52 +++++++- 4 files changed, 307 insertions(+), 10 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/inputs.go create mode 100644 modules/mesh-delivery-indexer/internal/index/manifests.go diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 7be504a..747700c 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -43,8 +43,10 @@ type Indexer struct { Store Store // candidates // Modules is the modules bucket; nil writes no fingerprints. Modules Store - Builds *Builds - Logf func(format string, args ...any) + // Manifests is the manifests bucket; nil writes no manifests (issue 1449). + Manifests Store + Builds *Builds + Logf func(format string, args ...any) mu sync.Mutex // guards source, which the source watch and the forge messages both reach source sourceNow @@ -72,6 +74,13 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error { return err } } + var digests map[string]string + if f.Kind == delivery.ForgePull { + var err error + if digests, err = ix.keepManifests(ctx, f); err != nil { + return err + } + } if trunkLike[f.Branch] { ix.logf("%s#%d: its branch %q is no delivery group, so no candidate", f.Repository, f.Number, f.Branch) return nil @@ -89,7 +98,12 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error { } cur = &c } - next := ix.next(cur, f) + next := ix.next(cur, f, digests) + if next != nil { + if err := ix.refuseDisagreement(*next); err != nil { + return nil, false, err + } + } switch { case next == nil && cur == nil, next != nil && cur != nil && delivery.SameCandidate(*cur, *next): return nil, false, nil @@ -185,7 +199,12 @@ func write(ctx context.Context, st Store, key string, compute func(value []byte, } // next is the group after the message; nil when the group is left with no pull request. -func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge) *delivery.Candidate { +// +// A pull message that carries manifests (issue 1448) gives each module it touches, in the pull request's Fps, the digest of +// its manifest and why it has no fingerprint of the new form (NoneNoPreview), and the candidate its FPSet (issue 1451); a +// pull request whose head is held again without its directories keeps the Fps it had. digests are the manifests' digests by +// directory. +func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[string]string) *delivery.Candidate { var pulls []delivery.CandidatePull var base uint64 if cur != nil { @@ -202,13 +221,20 @@ func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge) *delivery.Can switch { case f.DirsKnown: for _, dir := range f.Dirs { - if m := ix.Builds.ModuleOf(f.Repository, dir); !slices.Contains(entry.Modules, m) { + m := ix.Builds.ModuleOf(f.Repository, dir) + if !slices.Contains(entry.Modules, m) { entry.Modules = append(entry.Modules, m) } + if digest, ok := digests[dir]; ok { + if entry.Fps == nil { + entry.Fps = map[string]delivery.CandidateModule{} + } + entry.Fps[m] = delivery.CandidateModule{Manifest: digest, None2: NoneNoPreview} + } } case at >= 0 && pulls[at].Head.Head == f.Head: // Its directories are not known now, for the head the group already holds: what it knew stands. - entry.Modules = pulls[at].Modules + entry.Modules, entry.Fps = pulls[at].Modules, pulls[at].Fps } if at >= 0 { pulls[at] = entry @@ -230,5 +256,32 @@ func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge) *delivery.Can heads[i] = p.Head } hash, _ := delivery.HeadsHash(heads) - return &delivery.Candidate{V: delivery.Version, Group: f.Branch, Base: base, HeadsHash: hash, Pulls: pulls} + return &delivery.Candidate{V: delivery.Version, Group: f.Branch, Base: base, HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)} +} + +// NoneNoPreview is why a candidate's touched module has no fingerprint of the new form: the merge preview is not +// computed in this release (novox/hq ADR 0465 clause 3; the checker comes with issue 1445). +const NoneNoPreview = "no merge preview is computed" + +// refuseDisagreement refuses a candidate in which two pull requests give one module different manifests, as the +// candidate's own check does (mesh-sdk Candidate.check), naming both pull requests so the log says which; nothing is +// written, and the message is dropped as unreadable and said in the log (main.go handle): the next message of either pull +// request is judged again. +func (ix *Indexer) refuseDisagreement(c delivery.Candidate) error { + first := map[string]delivery.CandidatePull{} + for _, p := range c.Pulls { + for module, m := range p.Fps { + before, ok := first[module] + if !ok { + first[module] = p + continue + } + if before.Fps[module] != m { + err := fmt.Errorf("%w: candidates.%s: %s#%d and %s#%d give the module %q different manifests (%s and %s), so the group is not written", + ErrUnreadable, c.Group, before.Repository, before.Number, p.Repository, p.Number, module, before.Fps[module].Manifest, m.Manifest) + return err + } + } + } + return nil } diff --git a/modules/mesh-delivery-indexer/internal/index/inputs.go b/modules/mesh-delivery-indexer/internal/index/inputs.go new file mode 100644 index 0000000..52f189b --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/inputs.go @@ -0,0 +1,115 @@ +package index + +import ( + "fmt" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The reasons a module has no fingerprint of the new form while the indexer writes none for any module (novox/hq +// ADR 0465 clause 2; issue 1444). Toolchains missing is delivery.ToolchainNoReport. +const ( + // None2NoManifest is for a module whose directory the merged message carries no manifest of (an older bridge). + None2NoManifest = "the message carries no manifest" + // None2Builder is for a module with no artifact of a toolchain: only the builder's fingerprint is missing. + None2Builder = "builder's fingerprint not yet known" + // None2Context is for a module an artifact of which reads another repository's tree (today's form gives it none + // either, NoneContext). + None2Context = "context not pinned" +) + +// lockFile is the lock file of a language's artifact that resolves packages when it is built, by name in the module's +// directory (the bridge reads these, internal/bridge LockFiles; issue 1450). +var lockFile = map[string]string{"go": "go.sum", "typescript": "package-lock.json"} + +// moduleInputs is what a module's manifest and lock hashes give the indexer at a merge (issue 1450). +type moduleInputs struct { + // Kept is the new-form inputs the merge gave the module; nil when the manifest could not be read or gave none. + Kept *delivery.KeptInputs2 + // Languages is the language of each artifact that has a toolchain (a bundle or a package of go or typescript), by + // artifact name. The toolchain versions are joined to them when the build modules report them (issue 1444, the + // watching half); the SDK's KeptInputs2 keeps no languages, so they are read from the manifest again then. + Languages map[string]string + // None2 is why the module has no fingerprint of the new form. + None2 string +} + +// baseFP answers a base module's new-form fingerprint, or why it has none. +type baseFP func(module string) (fp, none string) + +// inputsOf turns the manifest of a module (raw, as the forge answered it), the tree of its directory and the lock +// hashes of that directory (lock file name to hash) into its new-form inputs. The manifest is read by the catalogue's +// parser; a manifest it refuses, or one that is of another module, leaves Kept nil and says why. +// +// The rules are the controller's builder (mesh-controller internal/builder/builder.go): a bundle or a package of a +// language with a toolchain (go, typescript) is the artifact that has one; a package, and a bundle of those languages, +// resolves packages when it is built (wantsPackages, :1017), so it has an entry in Locks, with the hash of its +// language's lock file or an empty hash when none was carried (the fingerprint is then none, "no lock"); an archive, +// an image and an upstream resolve nothing and have no toolchain. A base is a build.on entry that names a module. +func inputsOf(module, raw, tree string, locks map[string]string, base baseFP) moduleInputs { + m, err := catalogue.ParseManifest([]byte(raw)) + if err != nil { + return moduleInputs{None2: "the manifest cannot be read: " + err.Error()} + } + if m.Module != module { + return moduleInputs{None2: fmt.Sprintf("the manifest is of module %q, not %q", m.Module, module)} + } + if tree == "" { + return moduleInputs{None2: "the merge names no tree for the module's directory"} + } + kept := delivery.KeptInputs2{Tree: tree} + langs := map[string]string{} + contexts := false + var missing []string + if m.Build != nil { + for _, a := range m.Build.Artifacts { + if a.Context != nil { + contexts = true + } + if a.Kind != catalogue.ArtifactBundle && a.Kind != catalogue.ArtifactPackage { + continue + } + name, ok := lockFile[a.Language] + if !ok { + continue + } + langs[a.Name] = a.Language + if kept.Locks == nil { + kept.Locks = map[string]string{} + } + kept.Locks[a.Name] = locks[name] + } + seen := map[string]bool{} + for _, on := range m.Build.On { + if on.Module == "" || seen[on.Module] { + continue + } + seen[on.Module] = true + if fp, none := base(on.Module); fp != "" { + kept.Bases = append(kept.Bases, fp) + } else { + missing = append(missing, fmt.Sprintf("the base %s has no fingerprint (%s)", on.Module, none)) + } + } + } + sort.Strings(kept.Bases) + sort.Strings(missing) + out := moduleInputs{Kept: &kept} + if len(langs) > 0 { + out.Languages = langs + } + switch { + case contexts: + out.None2 = None2Context + case len(missing) > 0: + out.None2 = strings.Join(missing, "; ") + case len(langs) > 0: + out.None2 = delivery.ToolchainNoReport + default: + out.None2 = None2Builder + } + return out +} diff --git a/modules/mesh-delivery-indexer/internal/index/manifests.go b/modules/mesh-delivery-indexer/internal/index/manifests.go new file mode 100644 index 0000000..d1883b1 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/manifests.go @@ -0,0 +1,83 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// moduleNameOf is the module a manifest is of: the name the manifest itself gives (read here only to key the manifests +// value, so one digest always has one module; the inputs are read by the catalogue's parser, inputs.go), else the +// module a build record names the directory the path of. +func (ix *Indexer) moduleNameOf(repository, dir string, raw string) string { + var m struct { + Module string `json:"module"` + } + if json.Unmarshal([]byte(raw), &m) == nil && m.Module != "" { + return m.Module + } + name, _ := ix.Builds.Module(repository, dir) + return name +} + +// keepManifest writes manifests. for a manifest as the forge answered it, create-once (issue 1449): a key that +// exists with the same value is left and is no error; one that holds another value is an error (it cannot happen for +// one digest, ManifestDigest being a hash of the bytes, so it says the bucket was written by hand). It answers the +// digest. A manifest that cannot be a value (empty, not UTF-8, not JSON) is an error the caller says; it is never tried +// again. With no manifests store nothing is written and the digest is still answered. +func (ix *Indexer) keepManifest(ctx context.Context, module, raw string) (string, error) { + digest := delivery.ManifestDigest([]byte(raw)) + if ix.Manifests == nil { + return digest, nil + } + key, err := delivery.ManifestsKey(digest) + if err != nil { + return "", err + } + data, err := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: raw}) + if err != nil { + return "", fmt.Errorf("%w: manifests.%s: %v", errCannotBeAValue, key, err) + } + err = ix.Manifests.Create(ctx, key, data) + if err == nil { + return digest, nil + } + if !errors.Is(err, ErrConflict) { + return "", fmt.Errorf("writing manifests.%s: %w", key, err) + } + held, _, found, err := ix.Manifests.Get(ctx, key) + if err != nil { + return "", fmt.Errorf("reading manifests.%s: %w", key, err) + } + if !found { + return "", fmt.Errorf("manifests.%s was there and is not: written again at the next delivery", key) + } + if cur, err := delivery.DecodeManifest(held); err == nil && delivery.SameManifest(cur, delivery.Manifest{V: delivery.Version, Module: module, JSON: raw}) { + return digest, nil + } + return "", fmt.Errorf("manifests.%s holds another value than the manifest of %s, which hashes to it", key, module) +} + +// keepManifests writes the manifest of each directory a message carries, and answers the digest of each by directory. +// A manifest that cannot be kept is left out and said in the log, so the module reads as one with no manifest. +func (ix *Indexer) keepManifests(ctx context.Context, f delivery.Forge) (map[string]string, error) { + digests := make(map[string]string, len(f.Manifests)) + for dir, raw := range f.Manifests { + module := ix.moduleNameOf(f.Repository, dir, raw) + digest, err := ix.keepManifest(ctx, module, raw) + switch { + case err == nil: + digests[dir] = digest + case errors.Is(err, errCannotBeAValue): + ix.logf("%s#%d: the manifest of %q cannot be kept: %v", f.Repository, f.Number, dir, err) + default: + return nil, err + } + } + return digests, nil +} + +var errCannotBeAValue = errors.New("not a manifest value") diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 1243ff5..753df87 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -3,6 +3,7 @@ package index import ( "context" "fmt" + "reflect" "sort" "strings" @@ -47,8 +48,14 @@ func fingerprintOf(k Known, trees map[string]string) (fp, none string) { // applyMerged writes, for each module built from the merged repository, its fingerprint at the merge commit (or none and // why), with the source revision that names the merge commit as the repository's head when the indexer has seen it // (ApplySource), and Src 0 when it has not. A module of another repository is not written. A value the module already -// holds is not written again. +// holds is not written again. From the manifests and lock hashes the message carries (the forge bridge, issue 1448) it +// also writes the module's manifest digest, its new-form inputs and why it has no new-form fingerprint (issues 1444, 1449, +// 1450); this release writes no fingerprint of the new form for any module. func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error { + digests, err := ix.keepManifests(ctx, f) + if err != nil { + return err + } for _, k := range ix.Builds.Modules(f.Repository) { key, err := delivery.ModulesKey(k.Module) if err != nil { @@ -56,10 +63,22 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error { continue } fp, none := fingerprintOf(k, f.Trees) - next := delivery.Module{Repository: f.Repository, Commit: f.Merge, FP: fp, None: none, Src: ix.srcOf(f.Repository, f.Merge)} + next := delivery.Module{Repository: f.Repository, Commit: f.Merge, FP: fp, None: none, Src: ix.srcOf(f.Repository, f.Merge), Manifest: digests[k.Path]} + // The new form (issue 1444, R4c-2): no module has a fingerprint of it yet, so each has the reason it has none, which + // names what is missing, from its manifest and lock hashes (inputs.go). + raw, has := f.Manifests[k.Path] + switch { + case !has: + next.None2 = None2NoManifest + case next.Manifest == "": + next.None2 = "the manifest cannot be read: it cannot be kept as a value" + default: + in := inputsOf(k.Module, raw, f.Trees[k.Path], f.Locks[k.Path], ix.baseFP(ctx)) + next.Inputs2, next.None2 = in.Kept, in.None2 + } err = write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) { if found { - if cur, err := delivery.DecodeModule(value); err == nil && delivery.SameModule(cur, withV(next)) { + if cur, err := delivery.DecodeModule(value); err == nil && reflect.DeepEqual(cur, withV(next)) { return nil, false, nil } } @@ -73,4 +92,31 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error { return nil } +// baseFP answers the new-form fingerprint of a base module as modules. holds it, or why it has none. +func (ix *Indexer) baseFP(ctx context.Context) baseFP { + return func(module string) (string, string) { + key, err := delivery.ModulesKey(module) + if err != nil { + return "", err.Error() + } + value, _, found, err := ix.Modules.Get(ctx, key) + if err != nil { + return "", "its record cannot be read: " + err.Error() + } + if !found { + return "", "the indexer holds no record of it" + } + cur, err := delivery.DecodeModule(value) + switch { + case err != nil: + return "", "its record cannot be read: " + err.Error() + case cur.FP2 != "": + return cur.FP2, "" + case cur.None2 != "": + return "", cur.None2 + } + return "", "its record carries no fingerprint of the new form" + } +} + func withV(m delivery.Module) delivery.Module { m.V = delivery.Version; return m } -- 2.54.0 From 27bfd83fa02c4097aaa02a9ec0d0d16ee892cd48 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 18:55:29 +0000 Subject: [PATCH 7/9] Test the kept manifests, the new-form reasons and the candidate's manifests against real files (issues 1449, 1444, 1451, design 47 R4c-2) --- .../mesh-delivery-indexer/integration_test.go | 62 +++++++++++ .../index/candidate_manifests_test.go | 105 ++++++++++++++++++ .../internal/index/manifests_test.go | 102 +++++++++++++++++ .../internal/index/newform_test.go | 85 ++++++++++++++ 4 files changed, 354 insertions(+) create mode 100644 modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go create mode 100644 modules/mesh-delivery-indexer/internal/index/manifests_test.go create mode 100644 modules/mesh-delivery-indexer/internal/index/newform_test.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index bf98098..dc88ca3 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -145,6 +145,10 @@ func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue)) ( if err != nil { t.Fatal(err) } + // The bucket of the manifests, made by the controller from the manifest's state entry manifests (history 1, issue 1449). + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ManifestsBucket, History: delivery.ManifestsHistory}); err != nil { + t.Fatal(err) + } builds, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.BuildsBucket, History: 5}) if err != nil { t.Fatal(err) @@ -203,3 +207,61 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9 t.Errorf("module = %+v within 5 s, want fp %s", m, want) } } + +// TestIndexerKeepsManifestsAtAMerge: a merged message of mesh-catalog carrying the real manifest of the module avahi and the +// hash of its go.sum (the bridge's Manifests and Locks, issue 1448) makes, within 5 s, modules.avahi with its manifest digest, +// its inputs and the reason it has no fingerprint of the new form, and manifests. with the manifest byte for byte. +func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4 + // The manifest is avahi's module.json from mesh-catalog at a67f01963f0d9ef0913a63720b89e573ca7a16ef (copied into the + // index package's testdata); the lock hash is the sha256 of modules/avahi/go.sum there. + manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json") + if err != nil { + t.Fatal(err) + } + const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381" + const inputs = `{"Module":"avahi","Tree":"oldtree","Bases":null,"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}` + js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) { + rec := `{"id":"build-1","repository":"novox/mesh-catalog","ref":"main","module":"avahi","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/avahi","at":"2026-10-19T09:38:00Z","source-inputs":` + inputs + `}` + if _, err := builds.Put(ctx, delivery.BuildKey("avahi", "build-1"), []byte(rec)); err != nil { + t.Fatal(err) + } + if _, err := builds.Put(ctx, delivery.NewestKey("avahi"), []byte("build-1")); err != nil { + t.Fatal(err) + } + }) + data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, + Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0", + Trees: map[string]string{"modules/avahi": "newtree"}, + Manifests: map[string]string{"modules/avahi": string(manifest)}, + Locks: map[string]map[string]string{"modules/avahi": {"go.sum": goSum}}}) + if err != nil { + t.Fatal(err) + } + if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.merged", data); err != nil { + t.Fatal(err) + } + var m delivery.Module + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + if e, err := modules.Get(ctx, "avahi"); err == nil { + if m, err = delivery.DecodeModule(e.Value()); err == nil { + break + } + } + } + digest := delivery.ManifestDigest(manifest) + if m.Manifest != digest || m.None2 != "toolchain not yet reported" || m.FP2 != "" || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" || + m.Inputs2.Locks["tools"] != goSum { + t.Errorf("module = %+v within 5 s, want its manifest digest %s, its inputs and none2 'toolchain not yet reported'", m, digest) + } + kv, err := js.KeyValue(ctx, delivery.ManifestsBucket) + if err != nil { + t.Fatal(err) + } + e, err := kv.Get(ctx, digest) + if err != nil { + t.Fatalf("manifests.%s: %v", digest, err) + } + if got, err := delivery.DecodeManifest(e.Value()); err != nil || got.Module != "avahi" || got.JSON != string(manifest) { + t.Errorf("manifest = %+v err %v, want avahi and the file's bytes", got, err) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go b/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go new file mode 100644 index 0000000..bdadf94 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go @@ -0,0 +1,105 @@ +package index + +import ( + "context" + "errors" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +func manifestPull(t *testing.T, number int, head, manifest string) delivery.Forge { + f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: number, Head: head, + Action: delivery.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true} + if manifest != "" { + f.Manifests = map[string]string{"modules/avahi": manifest} + } + return f +} + +func candidateOf(t *testing.T, st *memStore) delivery.Candidate { + t.Helper() + key, err := delivery.CandidatesKey("feat/x") + if err != nil { + t.Fatal(err) + } + raw, ok := st.vals[key] + if !ok { + t.Fatalf("candidates holds %d keys, none %s", len(st.vals), key) + } + c, err := delivery.DecodeCandidate(raw) + if err != nil { + t.Fatal(err) + } + return c +} + +func TestACandidateGivesEachTouchedModuleItsManifestAndTheReasonItHasNoPreview(t *testing.T) { // 1451/AC1 + avahi := catalogueManifest(t, "avahi") + ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) + if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, avahi)); err != nil { + t.Fatal(err) + } + c := candidateOf(t, ix.Store.(*memStore)) // DecodeCandidate has run Candidate.check + want := delivery.CandidateModule{Manifest: delivery.ManifestDigest([]byte(avahi)), None2: "no merge preview is computed"} + if len(c.Pulls) != 1 || len(c.Pulls[0].Fps) != 1 || c.Pulls[0].Fps["avahi"] != want { + t.Fatalf("pulls = %+v, want Fps[avahi] = %+v", c.Pulls, want) + } + if c.FPSet == "" || c.FPSet != delivery.FingerprintSet(c.Pulls) { + t.Errorf("fpset = %q, want %q", c.FPSet, delivery.FingerprintSet(c.Pulls)) + } + // The same pull request heard again without its directories (the head unchanged) keeps what it had. + again := manifestPull(t, 301, h1, "") + again.Dirs, again.DirsKnown = nil, false + if err := ix.Apply(context.Background(), again); err != nil { + t.Fatal(err) + } + if c2 := candidateOf(t, ix.Store.(*memStore)); c2.Pulls[0].Fps["avahi"] != want || c2.FPSet != c.FPSet { + t.Errorf("after a message without directories: %+v fpset %q", c2.Pulls[0].Fps, c2.FPSet) + } +} + +func TestAPullWithoutManifestsWritesTheCandidateAsToday(t *testing.T) { // 1451/AC2 + ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) + if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, "")); err != nil { + t.Fatal(err) + } + c := candidateOf(t, ix.Store.(*memStore)) + if len(c.Pulls) != 1 || c.Pulls[0].Fps != nil || c.FPSet != "" || len(c.Pulls[0].Modules) != 1 || c.Pulls[0].Modules[0] != "avahi" { + t.Errorf("candidate = %+v, want the module avahi and no Fps and no fpset", c) + } +} + +func TestTwoPullsGivingOneModuleDifferentManifestsAreRefusedNamingBoth(t *testing.T) { // 1451/AC3 + avahi := catalogueManifest(t, "avahi") + // The second pull request carries another real manifest (ca-trust's, mesh-catalog a67f0196) for the same directory. + other := catalogueManifest(t, "ca-trust") + ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) + if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, avahi)); err != nil { + t.Fatal(err) + } + before := candidateOf(t, ix.Store.(*memStore)) + writes := ix.Store.(*memStore).writes + err := ix.Apply(context.Background(), manifestPull(t, 302, h2, other)) + if !errors.Is(err, ErrUnreadable) { + t.Fatalf("err = %v, want a refusal", err) + } + for _, w := range []string{catalogueRepo + "#301", catalogueRepo + "#302", `"avahi"`} { + if !strings.Contains(err.Error(), w) { + t.Errorf("the refusal %q does not name %s", err, w) + } + } + // Candidate.check refuses the group too, which is what the refusal stands for. + if _, encErr := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat.x", HeadsHash: before.HeadsHash, Pulls: []delivery.CandidatePull{before.Pulls[0], { + Head: delivery.Head{Repository: catalogueRepo, Number: 302, Head: h2}, Modules: []string{"avahi"}, + Fps: map[string]delivery.CandidateModule{"avahi": {Manifest: delivery.ManifestDigest([]byte(other)), None2: NoneNoPreview}}}}}); encErr == nil { + t.Error("the candidate check accepts two manifests for one module") + } + if ix.Store.(*memStore).writes != writes { + t.Error("a partial write was made") + } + if after := candidateOf(t, ix.Store.(*memStore)); len(after.Pulls) != 1 || after.FPSet != before.FPSet { + t.Errorf("the candidate changed: %+v", after) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/manifests_test.go b/modules/mesh-delivery-indexer/internal/index/manifests_test.go new file mode 100644 index 0000000..8c26287 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/manifests_test.go @@ -0,0 +1,102 @@ +package index + +import ( + "context" + "os" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The manifests these tests carry are real files, copied unchanged into testdata: the module.json of modules of +// mesh-catalog at its main commit a67f01963f0d9ef0913a63720b89e573ca7a16ef (testdata/catalogue-a67f0196-.module.json) +// and of build-agent from mesh-controller at its main commit f2d7bc94 (testdata/controller-f2d7bc94-build-agent.module.json). +func fixture(t *testing.T, name string) string { + t.Helper() + b, err := os.ReadFile("testdata/" + name + ".module.json") + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func catalogueManifest(t *testing.T, module string) string { + return fixture(t, "catalogue-a67f0196-"+module) +} + +// avahiGoSum is the sha256 of modules/avahi/go.sum in mesh-catalog at a67f0196 (sha256sum of that file). +const avahiGoSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381" + +const catalogueRepo = "novox/mesh-catalog" + +func manifestsIndexer(t *testing.T, recs ...[2]any) (*Indexer, *memStore, *memStore) { + t.Helper() + ix, mods := modulesIndexer(t, recs...) + manifests := newMem() + ix.Manifests = manifests + return ix, mods, manifests +} + +func withManifests(f delivery.Forge, manifests map[string]string, locks map[string]map[string]string) delivery.Forge { + f.Manifests, f.Locks = manifests, locks + return f +} + +func TestAMergedManifestIsKeptByItsDigestAndNamedByTheModule(t *testing.T) { // 1449/AC1 + avahi := catalogueManifest(t, "avahi") + ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) + f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, nil) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + digest := delivery.ManifestDigest([]byte(avahi)) + raw, ok := manifests.vals[digest] + if !ok || len(manifests.vals) != 1 { + t.Fatalf("manifests holds %d keys, want only %s", len(manifests.vals), digest) + } + m, err := delivery.DecodeManifest(raw) + if err != nil || m.Module != "avahi" || m.JSON != avahi { + t.Errorf("manifest = %+v err %v, want avahi and the bytes of the file", m, err) + } + if mod, ok := moduleOf(t, mods, "avahi"); !ok || mod.Manifest != digest { + t.Errorf("module = %+v (%v), want the manifest digest %s", mod, ok, digest) + } +} + +func TestTheSameManifestAppliedAgainIsWrittenOnce(t *testing.T) { // 1449/AC2 + avahi := catalogueManifest(t, "avahi") + ix, _, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) + f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, nil) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + // A second merge, and a pull message carrying it, the digest being the same. + f.Merge = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + if err := ix.Apply(context.Background(), f); err != nil { + t.Errorf("a second merge: %v", err) + } + pull := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: 301, Head: h1, + Action: delivery.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true, Manifests: map[string]string{"modules/avahi": avahi}} + if err := ix.Apply(context.Background(), pull); err != nil { + t.Errorf("a pull message: %v", err) + } + if manifests.writes != 1 || len(manifests.vals) != 1 { + t.Errorf("manifests written %d times into %d keys, want once into one", manifests.writes, len(manifests.vals)) + } +} + +func TestAManifestThatCannotBeAValueIsLeftOutAndSaid(t *testing.T) { // 1449 (a manifest that is not JSON cannot be kept byte for byte) + var logs []string + ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) + ix.Logf = func(format string, args ...any) { logs = append(logs, format) } + f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": "not json"}, nil) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + if len(manifests.vals) != 0 || len(logs) == 0 { + t.Errorf("manifests %d keys, log %v", len(manifests.vals), logs) + } + if mod, ok := moduleOf(t, mods, "avahi"); !ok || mod.Manifest != "" { + t.Errorf("module = %+v (%v), want no manifest digest", mod, ok) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/newform_test.go b/modules/mesh-delivery-indexer/internal/index/newform_test.go new file mode 100644 index 0000000..a610a9c --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/newform_test.go @@ -0,0 +1,85 @@ +package index + +import ( + "context" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// wantAvahiFP is today's fingerprint, at the tree "newtree", of a module with the recorded inputs aInputs (modules_test.go). +func wantAvahiFP(t *testing.T) string { + t.Helper() + fp, none := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"}, + Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}}) + if none != "" { + t.Fatal(none) + } + return fp +} + +func TestAGoModuleAtAMergeHasItsManifestItsInputsAndNoFingerprintOfTheNewForm(t *testing.T) { // 1444/AC1 + avahi := catalogueManifest(t, "avahi") + ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", + `{"Module":"avahi","Tree":"oldtree","Bases":["registry.example/base@sha256:aaa"],"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}`)) + f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, + map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}}) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + m, ok := moduleOf(t, mods, "avahi") + if !ok { + t.Fatal("no module avahi") + } + todays, _ := delivery.SourceFingerprint(delivery.SourceInputs{Module: "avahi", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"}, + Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}}) + if m.Manifest != delivery.ManifestDigest([]byte(avahi)) || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" || m.Inputs2.Locks["tools"] != avahiGoSum { + t.Errorf("module = %+v, want its manifest digest and inputs", m) + } + if m.None2 != "toolchain not yet reported" || m.FP2 != "" { + t.Errorf("none2 = %q fp2 = %q, want none2 'toolchain not yet reported'", m.None2, m.FP2) + } + if m.FP != todays || m.None != "" { + t.Errorf("fp = %q none = %q, want today's fingerprint %s unchanged", m.FP, m.None, todays) + } +} + +func TestAModuleWithNoToolchainArtifactNamesTheBuildersFingerprint(t *testing.T) { // 1444/AC2 + ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "ca-trust", catalogueRepo, "modules/ca-trust", "main", "", "2026-10-19T09:00:00Z", aInputs)) + f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/ca-trust": "newtree"}), map[string]string{"modules/ca-trust": catalogueManifest(t, "ca-trust")}, nil) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + if m, ok := moduleOf(t, mods, "ca-trust"); !ok || m.None2 != "builder's fingerprint not yet known" || m.Inputs2 == nil { + t.Errorf("module = %+v (%v), want none2 naming the builder's fingerprint", m, ok) + } +} + +func TestAMergedMessageWithoutManifestsIsWrittenAsTodayWithItsReason(t *testing.T) { // 1444/AC3 + ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs)) + if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil { + t.Fatal(err) + } + m, ok := moduleOf(t, mods, "a") + if !ok || m.FP != wantAvahiFP(t) || m.None2 != "the message carries no manifest" || m.Inputs2 != nil || m.Manifest != "" || m.FP2 != "" { + t.Errorf("module = %+v (%v), want today's fingerprint, none2 'the message carries no manifest' and no inputs", m, ok) + } + if len(manifests.vals) != 0 { + t.Errorf("manifests holds %d keys", len(manifests.vals)) + } +} + +func TestTheSameMergeAppliedAgainWritesNothing(t *testing.T) { // 1444 (a value the module already holds is not written again, with inputs kept) + avahi := catalogueManifest(t, "avahi") + ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) + f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, + map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}}) + for i := 0; i < 2; i++ { + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + } + if mods.writes != 1 { + t.Errorf("modules written %d times, want once", mods.writes) + } +} -- 2.54.0 From f69a561a1693abf5e2b71eaf1905dbbd02535e54 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 18:55:54 +0000 Subject: [PATCH 8/9] Test how a manifest and lock hashes become a module's new-form inputs (issue 1450, design 47 R4c-2) --- .../internal/index/inputs_test.go | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 modules/mesh-delivery-indexer/internal/index/inputs_test.go diff --git a/modules/mesh-delivery-indexer/internal/index/inputs_test.go b/modules/mesh-delivery-indexer/internal/index/inputs_test.go new file mode 100644 index 0000000..faee51b --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/inputs_test.go @@ -0,0 +1,146 @@ +package index + +import ( + "context" + "reflect" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// aBase is the new-form fingerprint of a base module, the one SourceFingerprint2 gives for the inputs of avahi at a tree. +func aBase(t *testing.T) string { + t.Helper() + fp, none := delivery.SourceFingerprint2(delivery.SourceInputs2{Module: "mesh-tools", Tree: "basetree"}) + if none != "" { + t.Fatal(none) + } + return fp +} + +func noBases(string) (string, string) { return "", "none are held in this test" } + +func TestAGoModuleKeepsItsTreeAndTheHashOfItsLockFile(t *testing.T) { // 1450/AC1 + in := inputsOf("avahi", catalogueManifest(t, "avahi"), "avahitree", map[string]string{"go.sum": avahiGoSum}, noBases) + want := &delivery.KeptInputs2{Tree: "avahitree", Locks: map[string]string{"tools": avahiGoSum}} + if !reflect.DeepEqual(in.Kept, want) { + t.Errorf("kept = %+v, want %+v", in.Kept, want) + } + if !reflect.DeepEqual(in.Languages, map[string]string{"tools": "go"}) { + t.Errorf("languages = %v, want tools: go", in.Languages) + } + // Through the indexer: the merge's tree of the module's directory and the lock hashes of that directory, kept in modules.avahi. + ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) + f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "avahitree", "modules/other": "x"}), + map[string]string{"modules/avahi": catalogueManifest(t, "avahi")}, map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}}) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + m, ok := moduleOf(t, mods, "avahi") + if !ok || !reflect.DeepEqual(m.Inputs2, want) || m.Manifest != delivery.ManifestDigest([]byte(catalogueManifest(t, "avahi"))) { + t.Errorf("module = %+v (%v), want inputs %+v and the manifest digest", m, ok, want) + } +} + +func TestAnArtifactThatResolvesPackagesAndHasNoLockFileHasAnEmptyHash(t *testing.T) { // 1450/AC2 + in := inputsOf("avahi", catalogueManifest(t, "avahi"), "avahitree", nil, noBases) + if in.Kept == nil || !reflect.DeepEqual(in.Kept.Locks, map[string]string{"tools": ""}) { + t.Fatalf("kept = %+v, want Locks {tools: \"\"}", in.Kept) + } + // The fingerprint of such inputs is none, "no lock" (mesh-sdk SourceFingerprint2). + if fp, none := delivery.SourceFingerprint2(in.Kept.With("avahi", nil)); fp != "" || none != "no lock" { + t.Errorf("fingerprint = %q none %q, want none 'no lock'", fp, none) + } + // A TypeScript bundle's lock file is package-lock.json, not go.sum. + ts := inputsOf("audit-logger", catalogueManifest(t, "audit-logger"), "t", map[string]string{"go.sum": "wrong", "package-lock.json": avahiGoSum}, noBases) + if ts.Kept == nil || !reflect.DeepEqual(ts.Kept.Locks, map[string]string{"code": avahiGoSum}) || !reflect.DeepEqual(ts.Languages, map[string]string{"code": "typescript"}) { + t.Errorf("typescript: kept %+v languages %v", ts.Kept, ts.Languages) + } + // netcheck has a Go and a TypeScript bundle: each takes its own language's lock file. + both := inputsOf("netcheck", catalogueManifest(t, "netcheck"), "t", map[string]string{"go.sum": "g", "package-lock.json": "p"}, noBases) + if both.Kept == nil || !reflect.DeepEqual(both.Kept.Locks, map[string]string{"tools-go": "g", "tools-typescript": "p"}) { + t.Errorf("netcheck: kept %+v", both.Kept) + } +} + +// aModuleWithABase is avahi's manifest (mesh-catalog a67f0196) with the one build.on entry that names a module as the +// controller's own test has it (mesh-controller internal/builder/standing_on_test.go:22): no manifest of the sources names +// a module as a base, only vendors' images. +func aModuleWithABase(t *testing.T) string { + t.Helper() + raw := catalogueManifest(t, "avahi") + with := strings.Replace(raw, `"build": {`, `"build": { + "on": [{"arg": "RUNTIME_BASE", "module": "mesh-tools", "artifact": "runtime"}],`, 1) + if with == raw { + t.Fatal("avahi's manifest has no build") + } + return with +} + +func TestABaseEntersByTheFingerprintOfItsModuleOrNamesItsAbsence(t *testing.T) { // 1450/AC3 + fp := aBase(t) + has := func(module string) (string, string) { + if module != "mesh-tools" { + t.Errorf("asked for the base %q", module) + } + return fp, "" + } + in := inputsOf("avahi", aModuleWithABase(t), "t", map[string]string{"go.sum": avahiGoSum}, has) + if in.Kept == nil || !reflect.DeepEqual(in.Kept.Bases, []string{fp}) { + t.Errorf("bases = %+v, want [%s]", in.Kept, fp) + } + none := inputsOf("avahi", aModuleWithABase(t), "t", map[string]string{"go.sum": avahiGoSum}, noBases) + if none.Kept == nil || len(none.Kept.Bases) != 0 || !strings.Contains(none.None2, "mesh-tools") || !strings.Contains(none.None2, "base") { + t.Errorf("kept %+v none2 %q, want no bases and none2 naming the base mesh-tools", none.Kept, none.None2) + } + // Through the indexer: modules.mesh-tools holds a fingerprint of the new form (a record written by a later release). + ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) + mods.put("mesh-tools", mustEncode(t, delivery.Module{Repository: catalogueRepo, Commit: merge, None: "x", FP2: fp, Manifest: strings.Repeat("a", 64)})) + f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "t"}), map[string]string{"modules/avahi": aModuleWithABase(t)}, nil) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + if m, _ := moduleOf(t, mods, "avahi"); m.Inputs2 == nil || !reflect.DeepEqual(m.Inputs2.Bases, []string{fp}) { + t.Errorf("module = %+v, want the base's fingerprint", m) + } +} + +func mustEncode(t *testing.T, m delivery.Module) []byte { + t.Helper() + b, err := delivery.EncodeModule(m) + if err != nil { + t.Fatal(err) + } + return b +} + +func TestAnArtifactOfNoToolchainHasNoLanguageAndNoLock(t *testing.T) { // 1450/AC4 + in := inputsOf("ca-trust", catalogueManifest(t, "ca-trust"), "t", map[string]string{"go.sum": avahiGoSum}, noBases) + if in.Kept == nil || len(in.Kept.Locks) != 0 || len(in.Languages) != 0 { + t.Errorf("kept %+v languages %v, want neither locks nor languages for an archive", in.Kept, in.Languages) + } +} + +func TestAManifestTheParserRefusesLeavesNoInputsAndSaysWhy(t *testing.T) { // 1450/AC5 + for name, raw := range map[string]string{ + "not json": "not json", + "unknown field": strings.Replace(catalogueManifest(t, "avahi"), `"version": "1",`, `"version": "1", "scope": "everything",`, 1), + } { + in := inputsOf("avahi", raw, "t", nil, noBases) + if in.Kept != nil || !strings.HasPrefix(in.None2, "the manifest cannot be read: ") || len(in.None2) <= len("the manifest cannot be read: ") { + t.Errorf("%s: kept %+v none2 %q, want nil and 'the manifest cannot be read: '", name, in.Kept, in.None2) + } + } + if in := inputsOf("someone-else", catalogueManifest(t, "avahi"), "t", nil, noBases); in.Kept != nil || !strings.Contains(in.None2, "avahi") { + t.Errorf("a manifest of another module: kept %+v none2 %q", in.Kept, in.None2) + } +} + +func TestAContextIsNotPinned(t *testing.T) { // 1450 (the rule 1444 keeps: context not pinned, as today's form) + // build-agent's manifest from mesh-controller at f2d7bc94 has an image artifact built from a context in another repository. + in := inputsOf("build-agent", fixture(t, "controller-f2d7bc94-build-agent"), "t", nil, noBases) + if in.Kept == nil || in.None2 != None2Context { + t.Errorf("kept %+v none2 %q, want none2 %q", in.Kept, in.None2, None2Context) + } +} -- 2.54.0 From c9f742d5f20ea8d2e7489d1a5917802434c8e02e Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 19:06:03 +0000 Subject: [PATCH 9/9] Read a merge's manifests and lock files at most eight at once, so a catalogue merge stays inside the bridge's bound (issues 1448, 1458, design 47 R4c-2) --- .../mesh-delivery-forge-bridge/main_test.go | 4 + .../internal/bridge/bridge.go | 82 ++++++++--- .../internal/bridge/files_test.go | 139 ++++++++++++++++++ 3 files changed, 208 insertions(+), 17 deletions(-) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go index b74d8da..c4ab010 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go @@ -8,6 +8,7 @@ import ( "log" "slices" "strings" + "sync" "testing" "time" @@ -139,8 +140,11 @@ func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/ m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.merged", data: mergedEvent()}, &fakePub{} var keys []string var args []any + var mu sync.Mutex // the files are asked for concurrently (issue 1458) call := func(_ context.Context, key string, body any) (json.RawMessage, error) { + mu.Lock() keys, args = append(keys, key), append(args, body) + mu.Unlock() if key == "gitea.gitea_get_file" { // the manifests, 1448; the trees are what this test is about return json.RawMessage(`{"file":{"path":"p","sha":"0123","size":2,"content":"{}"}}`), nil } diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go index b1c74f8..a6cbc7d 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go @@ -7,10 +7,12 @@ import ( "encoding/json" "errors" "fmt" + "path" "regexp" "slices" "strconv" "strings" + "sync" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -135,38 +137,84 @@ func Convert(subject string, data []byte, trees Trees, files Files) (Message, er return Message{Event: event, Data: body}, nil } +// MaxReadsInFlight is how many files of the forge inputsOf asks for at once. A merge into a repository of many +// modules (131 in the catalogue) reads three files for each, one after another inside the bound of a handled event +// (30 s) that leaves about 76 ms a call; eight at once keeps the forge's load small and the read well inside the bound +// (issue 1458). +const MaxReadsInFlight = 8 + +// read is one file asked for: the path in the directory dir, and whether it is the module's manifest. +type read struct { + dir, path string + manifest bool + raw []byte + err error +} + // inputsOf reads, at ref, the module.json of each directory (the root spelled "") and the lock files beside it // (LockFiles): the manifests as the forge answered them and the lock files' hex sha256 by name. A manifest the forge // does not have is an error, since the directory was said to hold one; a lock file it does not have is left out; any other // failure is an error (issue 1448). +// +// The files are asked for with at most MaxReadsInFlight calls at once (issue 1458). The result does not depend on the +// order the calls finish in, and the error is that of the first failing file in the order of dirs and LockFiles. After +// a failure no further file is asked for, and inputsOf returns only when every call it started has returned. func inputsOf(owner, repo, ref string, dirs []string, files Files) (manifests map[string]string, locks map[string]map[string]string, err error) { - manifests = map[string]string{} - locks = map[string]map[string]string{} at := func(dir, name string) string { if dir == "" { return name } return dir + "/" + name } + var reads []*read for _, dir := range dirs { - raw, err := files(owner, repo, at(dir, "module.json"), ref) - if err != nil { - return nil, nil, fmt.Errorf("%s: %w", at(dir, "module.json"), err) - } - manifests[dir] = string(raw) + reads = append(reads, &read{dir: dir, path: at(dir, "module.json"), manifest: true}) for _, name := range LockFiles { - raw, err := files(owner, repo, at(dir, name), ref) - if errors.Is(err, ErrNotFound) { - continue + reads = append(reads, &read{dir: dir, path: at(dir, name)}) + } + } + var ( + wg sync.WaitGroup + mu sync.Mutex + failed bool + slots = make(chan struct{}, MaxReadsInFlight) + ) + for _, r := range reads { + slots <- struct{}{} + mu.Lock() + stop := failed + mu.Unlock() + if stop { + <-slots + break + } + wg.Add(1) + go func(r *read) { + defer func() { <-slots; wg.Done() }() + r.raw, r.err = files(owner, repo, r.path, ref) + if r.err != nil && (r.manifest || !errors.Is(r.err, ErrNotFound)) { + mu.Lock() + failed = true + mu.Unlock() } - if err != nil { - return nil, nil, fmt.Errorf("%s: %w", at(dir, name), err) + }(r) + } + wg.Wait() + manifests = map[string]string{} + locks = map[string]map[string]string{} + for _, r := range reads { + switch { + case r.err != nil && (r.manifest || !errors.Is(r.err, ErrNotFound)): + return nil, nil, fmt.Errorf("%s: %w", r.path, r.err) + case r.err != nil: // a lock file the forge does not have + case r.manifest: + manifests[r.dir] = string(r.raw) + default: + sum := sha256.Sum256(r.raw) + if locks[r.dir] == nil { + locks[r.dir] = map[string]string{} } - sum := sha256.Sum256(raw) - if locks[dir] == nil { - locks[dir] = map[string]string{} - } - locks[dir][name] = hex.EncodeToString(sum[:]) + locks[r.dir][path.Base(r.path)] = hex.EncodeToString(sum[:]) } } if len(manifests) == 0 { diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go index 67e33b1..a5617ec 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go @@ -4,10 +4,14 @@ import ( "crypto/sha256" "encoding/hex" "errors" + "fmt" "os" "reflect" + "sort" "strings" + "sync" "testing" + "time" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -32,11 +36,14 @@ type fileCall struct{ owner, repo, path, ref string } type forgeFiles struct { have map[string][]byte errs map[string]error + mu sync.Mutex calls []fileCall } func (f *forgeFiles) get(owner, repo, path, ref string) ([]byte, error) { + f.mu.Lock() f.calls = append(f.calls, fileCall{owner, repo, path, ref}) + f.mu.Unlock() if err, ok := f.errs[path]; ok { return nil, err } @@ -136,7 +143,139 @@ func TestTheFilesAreAskedAtTheMergeCommitByOwnerRepoPathAndRef(t *testing.T) { / } want := []fileCall{{"novox", "mesh-delivery", "modules/a/module.json", merge}, {"novox", "mesh-delivery", "modules/a/go.sum", merge}, {"novox", "mesh-delivery", "modules/a/package-lock.json", merge}} + sort.Slice(forge.calls, func(i, j int) bool { return forge.calls[i].path < forge.calls[j].path }) // they are asked for concurrently + sort.Slice(want, func(i, j int) bool { return want[i].path < want[j].path }) if !reflect.DeepEqual(forge.calls, want) { t.Errorf("calls = %v, want %v", forge.calls, want) } } + +// slowForge answers like a forge that takes d a call: a module.json of the directory's name everywhere, a go.sum in +// the directories whose number is even and in the root, and counts the calls in flight. +type slowForge struct { + d time.Duration + mu sync.Mutex + inFlight, most, started, n int + failOn string + finished int +} + +func (f *slowForge) get(_, _, path, _ string) ([]byte, error) { + f.mu.Lock() + f.inFlight++ + f.started++ + if f.inFlight > f.most { + f.most = f.inFlight + } + f.mu.Unlock() + time.Sleep(f.d) + defer func() { f.mu.Lock(); f.inFlight--; f.finished++; f.mu.Unlock() }() + switch { + case path == f.failOn: + return nil, errors.New("Gitea API " + path + ": 502 bad gateway") + case strings.HasSuffix(path, "module.json"): + return []byte(`{"module":"` + path + `"}`), nil + case strings.HasSuffix(path, "go.sum") && (path == "go.sum" || path[len(path)-8]%2 == 0): + return []byte("sum of " + path), nil + } + return nil, ErrNotFound +} + +func manyDirs(n int) []string { + dirs := []string{""} + for i := 1; i < n; i++ { + dirs = append(dirs, fmt.Sprintf("modules/m%02d", i)) + } + return dirs +} + +func TestFortyDirectoriesAreReadEightAtATimeWithTheResultOfOneAtATime(t *testing.T) { // 1458/AC1 + dirs := manyDirs(40) + slow := &slowForge{d: 10 * time.Millisecond} + manifests, locks, err := inputsOf("novox", "mesh-catalog", merge, dirs, slow.get) + if err != nil { + t.Fatal(err) + } + if slow.most > MaxReadsInFlight || slow.most < 2 { + t.Errorf("at most %d calls were in flight, want more than one and at most %d", slow.most, MaxReadsInFlight) + } + if slow.started != 40*3 { + t.Errorf("%d calls, want 120", slow.started) + } + // The same read one file at a time. + wantM, wantL := map[string]string{}, map[string]map[string]string{} + one := &slowForge{} + for _, dir := range dirs { + m, l, err := inputsOfOne(dir, one.get) + if err != nil { + t.Fatal(err) + } + wantM[dir] = m + if l != nil { + wantL[dir] = l + } + } + if !reflect.DeepEqual(manifests, wantM) || !reflect.DeepEqual(locks, wantL) { + t.Errorf("the concurrent read differs from a one-at-a-time read: %d manifests %d locks, want %d and %d", len(manifests), len(locks), len(wantM), len(wantL)) + } + if len(locks) == 0 || len(locks) == 40 { + t.Errorf("%d directories with locks, the fixture should have some and not all", len(locks)) + } +} + +// inputsOfOne reads one directory with a single call at a time, as the bridge did before issue 1458. +func inputsOfOne(dir string, files Files) (string, map[string]string, error) { + at := func(name string) string { + if dir == "" { + return name + } + return dir + "/" + name + } + raw, err := files("o", "r", at("module.json"), "x") + if err != nil { + return "", nil, err + } + var locks map[string]string + for _, name := range LockFiles { + b, err := files("o", "r", at(name), "x") + if errors.Is(err, ErrNotFound) { + continue + } + if err != nil { + return "", nil, err + } + if locks == nil { + locks = map[string]string{} + } + s := sha256.Sum256(b) + locks[name] = hex.EncodeToString(s[:]) + } + return string(raw), locks, nil +} + +func TestOneFailingFileIsRetryAndEveryCallHasReturnedBeforeTheReadDoes(t *testing.T) { // 1458/AC2 + dirs := manyDirs(40) + slow := &slowForge{d: 10 * time.Millisecond, failOn: "modules/m05/go.sum"} + _, _, err := inputsOf("novox", "mesh-catalog", merge, dirs, slow.get) + if err == nil || !strings.Contains(err.Error(), "modules/m05/go.sum") || !strings.Contains(err.Error(), "502") { + t.Fatalf("err = %v, want the failing file named", err) + } + slow.mu.Lock() + defer slow.mu.Unlock() + if slow.inFlight != 0 || slow.finished != slow.started { + t.Errorf("%d calls in flight, %d of %d finished, when the read returned", slow.inFlight, slow.finished, slow.started) + } + if slow.started >= 40*3 { + t.Errorf("%d calls were made, want the read to stop asking after the failure", slow.started) + } + // Through Convert it is ErrRetry. + var calls []call + dirsMap := map[string]string{} + for _, d := range dirs[1:] { + dirsMap[d] = "t" + } + slow2 := &slowForge{failOn: "modules/m05/module.json"} + if _, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", dirsMap, nil), slow2.get); !errors.Is(err, ErrRetry) { + t.Errorf("err = %v, want ErrRetry", err) + } +} -- 2.54.0