From 82da252cf027912aeb7ba4f351f7292192cd517f Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 11:23:21 +0100 Subject: [PATCH 1/4] Indexer: index the manifests of modules registered from their own repositories, so the grants part has no gap for them --- .../cmd/mesh-delivery-indexer/main.go | 27 +++- .../cmd/mesh-delivery-indexer/named.go | 77 +++++++++++ .../cmd/mesh-delivery-indexer/named_test.go | 26 ++++ .../internal/index/builds.go | 49 +++++++ .../internal/index/candidates.go | 4 + .../internal/index/modules.go | 9 +- .../internal/index/registered.go | 126 ++++++++++++++++++ .../internal/index/registered_test.go | 97 ++++++++++++++ .../mesh-delivery-indexer/manifest_test.go | 4 +- modules/mesh-delivery-indexer/module.json | 3 +- 10 files changed, 416 insertions(+), 6 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/registered.go create mode 100644 modules/mesh-delivery-indexer/internal/index/registered_test.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index a4a515f..8081a2f 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -266,11 +266,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } } errs := make(chan error, 1) + var buildsWatch jetstream.KeyWatcher if builds != nil { if w, err := settle(ctx, builds, each); err != nil { process.NotWatched(logger.Printf, delivery.BuildsBucket, err, "a module whose key names no directory is not given one until the indexer starts again") } else { - go func() { errs <- follow(ctx, w, each) }() + buildsWatch = w } } @@ -285,7 +286,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } defer calls.Close() ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf, - Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call)} + Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call)} // The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key // main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does // not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate. @@ -336,6 +337,28 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := ix.RecomputeAtStart(ctx); err != nil { logger.Printf("computing the new-form fingerprints at start: %v", err) } + // A module registered in the catalogue from its own repository gets its manifest indexed when the controller's records + // name it and the bucket holds none (registered.go); the records that follow are heard below. + if err := ix.IndexRegisteredAll(ctx); err != nil { + logger.Printf("indexing the registered modules at start: %v", err) + } + if buildsWatch != nil { + go func() { + errs <- follow(ctx, buildsWatch, func(e jetstream.KeyValueEntry) { + each(e) + if !opOf(e) { + return + } + var r delivery.BuildRecord + if json.Unmarshal(e.Value(), &r) != nil || r.Module == "" { + return + } + if err := ix.IndexRegisteredModule(ctx, r.Module); err != nil { + logger.Printf("the registered module %s is not indexed: %v", r.Module, err) + } + }) + }() + } // The modules the trunk no longer holds go from the bucket (issue 1900); a merge does the same from its own message. if err := ix.Reconcile(ctx); err != nil { logger.Printf("dropping the modules the trunk does not hold at start: %v", err) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go index 562c6a9..0678740 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go @@ -2,11 +2,16 @@ package main import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" + "path" "strings" "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-indexer/internal/index" ) // commitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea): with `paths`, by module directory @@ -93,3 +98,75 @@ func containsVia(call toolCaller) func(ctx context.Context, repository, commit, return *a.Contains, nil } } + +// getFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea gitea_get_file): arguments owner, +// repo, path and ref, answer {file: {content}}; a path it does not have fails with a 404 in its words. +const getFileTool = "gitea.gitea_get_file" + +// lockFiles are read beside a manifest, as the forge bridge reads them (bridge.LockFiles). +var lockFiles = []string{"go.sum", "package-lock.json"} + +// readVia reads a repository at a commit as the forge bridge does for a merged message: the module directories by +// gitea_commit_trees, then each one's module.json and lock files by gitea_get_file. A manifest the forge does not have is +// an error; a lock file it does not have is left out. +func readVia(call toolCaller) func(ctx context.Context, repository, sha string) (index.RepoRead, error) { + return func(_ context.Context, repository, sha string) (index.RepoRead, error) { + owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/") + if !ok { + return index.RepoRead{}, fmt.Errorf("%q is not /", repository) + } + raw, err := call(commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha}) + if err != nil { + return index.RepoRead{}, err + } + var a struct { + Root string `json:"root"` + Trees map[string]string `json:"trees"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return index.RepoRead{}, fmt.Errorf("the answer of %s is not {root, trees}: %w", commitTreesTool, err) + } + out := index.RepoRead{Root: a.Root, Trees: a.Trees, Manifests: map[string]string{}, Locks: map[string]map[string]string{}} + file := func(dir, name string) (string, error) { + p := name + if dir != "" { + p = dir + "/" + name + } + raw, err := call(getFileTool, map[string]string{"owner": owner, "repo": repo, "path": p, "ref": sha}) + if err != nil { + return "", err + } + var f struct { + File *struct { + Content *string `json:"content"` + } `json:"file"` + } + if err := json.Unmarshal(raw, &f); err != nil || f.File == nil || f.File.Content == nil { + return "", fmt.Errorf("the answer of %s is not {file: {content}}: %v", getFileTool, err) + } + return *f.File.Content, nil + } + for dir := range a.Trees { + m, err := file(dir, "module.json") + if err != nil { + return index.RepoRead{}, fmt.Errorf("%s: %w", path.Join(dir, "module.json"), err) + } + out.Manifests[dir] = m + for _, name := range lockFiles { + c, err := file(dir, name) + if err != nil { + if strings.Contains(err.Error(), ": 404 ") { + continue + } + return index.RepoRead{}, fmt.Errorf("%s: %w", path.Join(dir, name), err) + } + sum := sha256.Sum256([]byte(c)) + if out.Locks[dir] == nil { + out.Locks[dir] = map[string]string{} + } + out.Locks[dir][name] = hex.EncodeToString(sum[:]) + } + } + return out, nil + } +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go index 71ace76..2815aae 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "errors" "slices" "strings" "testing" @@ -50,3 +51,28 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3 t.Error("the trunk asked for paths") } } + +func TestReadViaReadsManifestsAndLocks(t *testing.T) { + call := func(key string, body any) (json.RawMessage, error) { + switch key { + case commitTreesTool: + return json.RawMessage(`{"root":"r0","trees":{"":"t1"}}`), nil + case getFileTool: + switch body.(map[string]string)["path"] { + case "module.json": + return json.RawMessage(`{"file":{"content":"{\"module\":\"snake\"}"}}`), nil + case "go.sum": + return json.RawMessage(`{"file":{"content":"x"}}`), nil + } + return nil, errors.New("Gitea API /x: 404 not found") + } + return nil, errors.New("unexpected " + key) + } + got, err := readVia(call)(context.Background(), "http://novox.internal:20000/jschoubben/snake.git", "abc") + if err != nil { + t.Fatal(err) + } + if got.Root != "r0" || got.Manifests[""] != `{"module":"snake"}` || len(got.Locks[""]) != 1 || got.Locks[""]["go.sum"] == "" { + t.Fatalf("read = %+v", got) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/builds.go b/modules/mesh-delivery-indexer/internal/index/builds.go index 5dfd681..38376a6 100644 --- a/modules/mesh-delivery-indexer/internal/index/builds.go +++ b/modules/mesh-delivery-indexer/internal/index/builds.go @@ -2,6 +2,8 @@ package index import ( "encoding/json" + "maps" + "slices" "strings" "sync" @@ -88,3 +90,50 @@ func (b *Builds) PathOf(module, repository string) (string, bool) { } return pathOf(strings.Trim(best.Path, "/")), true } + +// Registration is a module registered in the catalogue from its own repository, as the controller's newest outcome record +// of it names it: the repository, the directory of its manifest there ("" or "." for the root) and the commit it was +// built at. +type Registration struct { + Module, Repository, Path, Commit string +} + +// registered is whether the record is an outcome of the trunk (or of no named branch) that did not fail: what registers a +// module. Unlike IsTrunkBuild it asks for no recorded source inputs, which no build records since ADR 0530 clause 1. +func registered(r delivery.BuildRecord) bool { + return r.State == "" && r.Failed == "" && (r.Ref == "" || r.Ref == "main") +} + +// RegistrationOf is the registration of a module by its newest registering record; false when there is none. +func (b *Builds) RegistrationOf(module string) (Registration, bool) { + var best *delivery.BuildRecord + b.mu.Lock() + defer b.mu.Unlock() + for _, r := range b.records { + if r.Module == module && registered(r) && (best == nil || newer(r, *best)) { + c := r + best = &c + } + } + if best == nil { + return Registration{}, false + } + return Registration{Module: best.Module, Repository: best.Repository, Path: best.Path, Commit: best.Commit}, true +} + +// Registered is the registration of every module a record names, by module name. +func (b *Builds) Registered() []Registration { + b.mu.Lock() + names := map[string]bool{} + for _, r := range b.records { + names[r.Module] = true + } + b.mu.Unlock() + var out []Registration + for _, name := range slices.Sorted(maps.Keys(names)) { + if reg, ok := b.RegistrationOf(name); ok { + out = append(out, reg) + } + } + return out +} diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 7a14378..2cc52d0 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -53,6 +53,9 @@ type Indexer struct { // has none. Named func(ctx context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error) Trunk func(ctx context.Context, repository, sha string) ([]string, error) + // Read asks the forge for a repository's module directories at a commit, with their manifests and lock-file hashes, as + // the forge bridge reads them for a merged message (registered.go). Nil indexes no registered module. + Read func(ctx context.Context, repository, sha string) (RepoRead, error) // Contains says whether a commit is the second or an ancestor of it (gitea.gitea_contains). Contains func(ctx context.Context, repository, commit, in string) (bool, error) Logf func(format string, args ...any) @@ -61,6 +64,7 @@ type Indexer struct { source sourceNow previews map[string]delivery.Preview // by previews key (previews.go) toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go) + tried map[string]string // registered module to the commit its manifest was last tried at (registered.go) recompute sync.Mutex // one Recompute at a time } diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 5ff6464..5bd2670 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -25,6 +25,13 @@ import ( // A value the module already holds but for its commit is not written again: its commit stays the one its fingerprint first // appeared at (writeOverlay), so a merge writes only the modules it changes. func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error { + return ix.applyManifests(ctx, f, true) +} + +// applyManifests is applyMerged; drop says whether the modules of the repository the message holds no manifest for are +// deleted. A message that names only some of a repository's modules (a registered module read on its own, registered.go) +// drops none. +func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bool) error { digests, err := ix.keepManifests(ctx, f) if err != nil { return err @@ -84,7 +91,7 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error { } // A module of this repository the merge commit holds no module.json for is gone from the trunk, and so is its key // (issue 1900). A message that carries no manifests (an older writer) or one that cannot be read whole says nothing. - if len(f.Manifests) > 0 && every { + if drop && len(f.Manifests) > 0 && every { if err := ix.dropRemoved(ctx, f.Repository, held, func(key string, _ delivery.Module) bool { return named[key] }); err != nil { return err } diff --git a/modules/mesh-delivery-indexer/internal/index/registered.go b/modules/mesh-delivery-indexer/internal/index/registered.go new file mode 100644 index 0000000..09ddd50 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/registered.go @@ -0,0 +1,126 @@ +package index + +import ( + "context" + "fmt" + "strings" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// A module registered in the catalogue from its own repository (the operator registers it with the repository and the +// path of its manifest) had no manifest here: the indexer learns a repository only from a merged message, and no merge +// followed the registration. The controller's outcome records name each registered module's repository, directory and +// commit (builds.go Registered), so the indexer reads that module's manifest at the repository's trunk head and writes the +// same keys a merge writes. A later merge to the repository's trunk reaches the module through the forge bridge's merged +// message as for every other repository. + +// RepoRead is a repository at a commit as the forge bridge reads it for a merged message: the root tree, the tree of each +// directory holding a module.json (the root spelled ""), their manifests and the hex sha256 of the lock files beside them. +type RepoRead struct { + Root string + Trees map[string]string + Manifests map[string]string + Locks map[string]map[string]string +} + +// IndexRegistered writes the module of a registration as a merge of its repository would, when the modules bucket holds +// none of that name. Its commit is the head the gate's source names for the repository, else the registration's. The +// repository's other modules are neither written nor dropped: they are the merges'. A manifest that is not at the path or +// names another module is said and left. +func (ix *Indexer) IndexRegistered(ctx context.Context, r Registration) error { + if ix.Modules == nil || ix.Read == nil { + return nil + } + key, err := delivery.ModulesKey(r.Module) + if err != nil { + ix.logf("%s: the registered module has no usable key, skipped: %v", r.Module, err) + return nil + } + if _, _, found, err := ix.Modules.Get(ctx, key); err != nil { + return fmt.Errorf("modules.%s: %w", key, err) + } else if found { + return nil + } + repository := delivery.RepositoryName(r.Repository) + commit := r.Commit + ix.mu.Lock() + for repo, head := range ix.source.heads { + if delivery.SameRepository(repo, r.Repository) && head != "" { + commit = head + } + } + ix.mu.Unlock() + if repository == "" || commit == "" { + ix.logf("%s: registered with no repository or commit, so its manifest is not read", r.Module) + return nil + } + // One try for each commit of a registration: a repository with no manifest where it was registered is said once, not at + // every record of the controller's builds. + ix.mu.Lock() + if ix.tried[r.Module] == commit { + ix.mu.Unlock() + return nil + } + if ix.tried == nil { + ix.tried = map[string]string{} + } + ix.tried[r.Module] = commit + ix.mu.Unlock() + read, err := ix.Read(ctx, repository, commit) + if err != nil { + ix.mu.Lock() + delete(ix.tried, r.Module) // a forge that could not say is asked again + ix.mu.Unlock() + return fmt.Errorf("%s: reading %s at %s: %w", r.Module, repository, commit, err) + } + dir := strings.Trim(r.Path, "/") + if dir == delivery.RootPath { + dir = "" + } + raw, ok := read.Manifests[dir] + if !ok || read.Trees[dir] == "" { + ix.logf("%s: %s at %s holds no module.json at %q, so the registered module is not indexed", r.Module, repository, commit, dir) + return nil + } + if name := manifestModule(raw); name != r.Module { + ix.logf("%s: the manifest at %q of %s at %s names the module %q, so the registered module is not indexed", r.Module, dir, repository, commit, name) + return nil + } + f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repository, Merge: commit, Root: read.Root, + Trees: map[string]string{dir: read.Trees[dir]}, + Manifests: map[string]string{dir: raw}} + if l := read.Locks[dir]; len(l) > 0 { + f.Locks = map[string]map[string]string{dir: l} + } + return ix.applyManifests(ctx, f, false) +} + +// IndexRegisteredModule indexes the registration the controller's records name for a module, when there is one. +func (ix *Indexer) IndexRegisteredModule(ctx context.Context, module string) error { + if ix.Builds == nil { + return nil + } + if r, ok := ix.Builds.RegistrationOf(module); ok { + return ix.IndexRegistered(ctx, r) + } + return nil +} + +// IndexRegisteredAll indexes every registered module the modules bucket does not hold. A module that cannot be read is +// said and the others go on; the first error is answered. +func (ix *Indexer) IndexRegisteredAll(ctx context.Context) error { + if ix.Builds == nil { + return nil + } + var first error + for _, r := range ix.Builds.Registered() { + if err := ix.IndexRegistered(ctx, r); err != nil { + ix.logf("the registered module %s is not indexed: %v", r.Module, err) + if first == nil { + first = err + } + } + } + return first +} diff --git a/modules/mesh-delivery-indexer/internal/index/registered_test.go b/modules/mesh-delivery-indexer/internal/index/registered_test.go new file mode 100644 index 0000000..b5bfae8 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/registered_test.go @@ -0,0 +1,97 @@ +package index + +import ( + "context" + "errors" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +func TestARegisteredModuleIsIndexedFromItsRepository(t *testing.T) { // the grants part's gaps: modules registered from their own repositories had no manifest here + const forge = "http://novox.internal:20000/jschoubben/snake.git" + snake := `{"module":"snake","version":"1"}` + for _, tc := range []struct { + name string + path string + read RepoRead + err error + held bool // modules.snake exists already + other bool // modules.other of the same repository exists + want bool // modules.snake is written + }{ + {name: "a module at the root of its repository", path: "", want: true, + read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}}, + {name: "a root spelled as a dot", path: ".", want: true, + read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}}, + {name: "a module in a directory", path: "modules/snake", want: true, + read: RepoRead{Root: "r0", Trees: map[string]string{"modules/snake": "t1"}, Manifests: map[string]string{"modules/snake": snake}}}, + {name: "another module of the repository is neither written nor dropped", path: "", want: true, other: true, + read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}}, + {name: "a module the bucket holds is left", path: "", held: true, + read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}}, + {name: "no manifest at the path", path: "modules/x", + read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}}, + {name: "a manifest of another module", path: "", + read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": `{"module":"other","version":"1"}`}}}, + {name: "a forge that cannot say", path: "", err: errors.New("down")}, + } { + t.Run(tc.name, func(t *testing.T) { + ix, mods := modulesIndexer(t, rec(t, "build-1", "snake", forge, tc.path, "", "", "2026-10-19T09:00:00Z", "")) + if tc.held { + if err := mods.Create(context.Background(), "snake", mustModule(t, delivery.Module{Repository: "jschoubben/snake", Commit: h1, Path: ".", None2: "x"})); err != nil { + t.Fatal(err) + } + } + if tc.other { + if err := mods.Create(context.Background(), "other", mustModule(t, delivery.Module{Repository: "jschoubben/snake", Commit: h1, Path: "modules/other", None2: "x"})); err != nil { + t.Fatal(err) + } + } + ix.Manifests = newMem() + var asked struct{ repo, sha string } + ix.Read = func(_ context.Context, repo, sha string) (RepoRead, error) { + asked.repo, asked.sha = repo, sha + return tc.read, tc.err + } + err := ix.IndexRegisteredAll(context.Background()) + if (err != nil) != (tc.err != nil) { + t.Fatalf("error = %v", err) + } + m, ok := moduleOf(t, mods, "snake") + if tc.want { + if !ok || m.Manifest == "" || m.Repository != "jschoubben/snake" || m.Commit != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" || m.Path != pathOf(dirOf(tc.path)) { + t.Fatalf("modules.snake = %+v, found %v", m, ok) + } + if len(ix.Manifests.(*memStore).vals) != 1 { + t.Errorf("manifests written = %d, want 1", len(ix.Manifests.(*memStore).vals)) + } + if asked.repo != "jschoubben/snake" { + t.Errorf("the forge was asked for %q", asked.repo) + } + } else if !tc.held && ok { + t.Fatalf("modules.snake written: %+v", m) + } + if _, ok := moduleOf(t, mods, "other"); ok != tc.other { + t.Errorf("modules.other held = %v, want %v", ok, tc.other) + } + }) + } +} + + +func mustModule(t *testing.T, m delivery.Module) []byte { + t.Helper() + b, err := delivery.EncodeModule(withV(m)) + if err != nil { + t.Fatal(err) + } + return b +} + +func dirOf(p string) string { + if p == "." { + return "" + } + return p +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index b8c3fc5..5822f79 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -95,10 +95,10 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build if m.SecretsOwner != "mesh-delivery-indexer" { t.Errorf("secrets-owner = %q", m.SecretsOwner) } - // Two invokes: the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths: + // Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths: // ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a // commit is in another (issue 1900). - if want := []string{"gitea.gitea_commit_trees", "gitea.gitea_contains"}; !reflect.DeepEqual(m.Invokes, want) { + if want := []string{"gitea.gitea_commit_trees", "gitea.gitea_contains", "gitea.gitea_get_file"}; !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } // No emits, no streams, no bus, no seat, no tool (ADR 0437 §4). diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index bd5a90e..6abdf0e 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -23,7 +23,8 @@ ], "invokes": [ "gitea.gitea_commit_trees", - "gitea.gitea_contains" + "gitea.gitea_contains", + "gitea.gitea_get_file" ], "own-secrets": { "broker": "${dir:state}/broker" -- 2.54.0 From c44b9f91cd81bb274e612137778ffac02ad381ba Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 11:23:27 +0100 Subject: [PATCH 2/4] Indexer: gofmt --- modules/mesh-delivery-indexer/internal/index/registered_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/mesh-delivery-indexer/internal/index/registered_test.go b/modules/mesh-delivery-indexer/internal/index/registered_test.go index b5bfae8..70e9779 100644 --- a/modules/mesh-delivery-indexer/internal/index/registered_test.go +++ b/modules/mesh-delivery-indexer/internal/index/registered_test.go @@ -79,7 +79,6 @@ func TestARegisteredModuleIsIndexedFromItsRepository(t *testing.T) { // the gran } } - func mustModule(t *testing.T, m delivery.Module) []byte { t.Helper() b, err := delivery.EncodeModule(withV(m)) -- 2.54.0 From d9bfba1e6253d4f9cdb9d5ef3e934b3450ae948c Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 11:36:22 +0100 Subject: [PATCH 3/4] Indexer: read one directory of a registered module's repository, leave a repository the source names to its merges, accept master as a trunk --- .../cmd/mesh-delivery-indexer/named.go | 12 ++-- .../cmd/mesh-delivery-indexer/named_test.go | 9 ++- .../internal/index/builds.go | 4 +- .../internal/index/candidates.go | 8 +-- .../internal/index/registered.go | 39 +++++++----- .../internal/index/registered_test.go | 62 +++++++++++++------ 6 files changed, 86 insertions(+), 48 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go index 0678740..796cfba 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go @@ -106,11 +106,11 @@ const getFileTool = "gitea.gitea_get_file" // lockFiles are read beside a manifest, as the forge bridge reads them (bridge.LockFiles). var lockFiles = []string{"go.sum", "package-lock.json"} -// readVia reads a repository at a commit as the forge bridge does for a merged message: the module directories by -// gitea_commit_trees, then each one's module.json and lock files by gitea_get_file. A manifest the forge does not have is -// an error; a lock file it does not have is left out. -func readVia(call toolCaller) func(ctx context.Context, repository, sha string) (index.RepoRead, error) { - return func(_ context.Context, repository, sha string) (index.RepoRead, error) { +// readVia reads a repository at a commit as the forge bridge does for a merged message: the trees by gitea_commit_trees, +// then the module.json and lock files of the one directory dir by gitea_get_file (a repository can hold hundreds). A manifest the +// forge does not have is an error; a directory with no module.json in the trees is answered with no manifest; a lock file it does not have is left out. +func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir string) (index.RepoRead, error) { + return func(_ context.Context, repository, sha, dir string) (index.RepoRead, error) { owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/") if !ok { return index.RepoRead{}, fmt.Errorf("%q is not /", repository) @@ -146,7 +146,7 @@ func readVia(call toolCaller) func(ctx context.Context, repository, sha string) } return *f.File.Content, nil } - for dir := range a.Trees { + if _, held := a.Trees[dir]; held { m, err := file(dir, "module.json") if err != nil { return index.RepoRead{}, fmt.Errorf("%s: %w", path.Join(dir, "module.json"), err) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go index 2815aae..b3040ed 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go @@ -53,11 +53,13 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3 } func TestReadViaReadsManifestsAndLocks(t *testing.T) { + var files []string call := func(key string, body any) (json.RawMessage, error) { switch key { case commitTreesTool: - return json.RawMessage(`{"root":"r0","trees":{"":"t1"}}`), nil + return json.RawMessage(`{"root":"r0","trees":{"":"t1","modules/other":"t2"}}`), nil case getFileTool: + files = append(files, body.(map[string]string)["path"]) switch body.(map[string]string)["path"] { case "module.json": return json.RawMessage(`{"file":{"content":"{\"module\":\"snake\"}"}}`), nil @@ -68,10 +70,13 @@ func TestReadViaReadsManifestsAndLocks(t *testing.T) { } return nil, errors.New("unexpected " + key) } - got, err := readVia(call)(context.Background(), "http://novox.internal:20000/jschoubben/snake.git", "abc") + got, err := readVia(call)(context.Background(), "http://novox.internal:20000/jschoubben/snake.git", "abc", "") if err != nil { t.Fatal(err) } + if len(got.Manifests) != 1 || len(files) != 3 { // the one directory asked: its manifest and two lock files, never the others + t.Fatalf("manifests = %v, files asked = %v", got.Manifests, files) + } if got.Root != "r0" || got.Manifests[""] != `{"module":"snake"}` || len(got.Locks[""]) != 1 || got.Locks[""]["go.sum"] == "" { t.Fatalf("read = %+v", got) } diff --git a/modules/mesh-delivery-indexer/internal/index/builds.go b/modules/mesh-delivery-indexer/internal/index/builds.go index 38376a6..8656664 100644 --- a/modules/mesh-delivery-indexer/internal/index/builds.go +++ b/modules/mesh-delivery-indexer/internal/index/builds.go @@ -98,10 +98,10 @@ type Registration struct { Module, Repository, Path, Commit string } -// registered is whether the record is an outcome of the trunk (or of no named branch) that did not fail: what registers a +// registered is whether the record is an outcome of the trunk (no named branch, or main, master or trunk, as a repository's default branch is named) that did not fail: what registers a // module. Unlike IsTrunkBuild it asks for no recorded source inputs, which no build records since ADR 0530 clause 1. func registered(r delivery.BuildRecord) bool { - return r.State == "" && r.Failed == "" && (r.Ref == "" || r.Ref == "main") + return r.State == "" && r.Failed == "" && trunkLike[r.Ref] } // RegistrationOf is the registration of a module by its newest registering record; false when there is none. diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 2cc52d0..a40eeb1 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -53,9 +53,9 @@ type Indexer struct { // has none. Named func(ctx context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error) Trunk func(ctx context.Context, repository, sha string) ([]string, error) - // Read asks the forge for a repository's module directories at a commit, with their manifests and lock-file hashes, as - // the forge bridge reads them for a merged message (registered.go). Nil indexes no registered module. - Read func(ctx context.Context, repository, sha string) (RepoRead, error) + // Read asks the forge for a repository's trees at a commit and the manifest and lock-file hashes of the one module directory dir (the + // root spelled ""), as the forge bridge reads them for a merged message (registered.go). Nil indexes no registered module. + Read func(ctx context.Context, repository, sha, dir string) (RepoRead, error) // Contains says whether a commit is the second or an ancestor of it (gitea.gitea_contains). Contains func(ctx context.Context, repository, commit, in string) (bool, error) Logf func(format string, args ...any) @@ -64,7 +64,7 @@ type Indexer struct { source sourceNow previews map[string]delivery.Preview // by previews key (previews.go) toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go) - tried map[string]string // registered module to the commit its manifest was last tried at (registered.go) + tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go) recompute sync.Mutex // one Recompute at a time } diff --git a/modules/mesh-delivery-indexer/internal/index/registered.go b/modules/mesh-delivery-indexer/internal/index/registered.go index 09ddd50..248bc81 100644 --- a/modules/mesh-delivery-indexer/internal/index/registered.go +++ b/modules/mesh-delivery-indexer/internal/index/registered.go @@ -11,7 +11,7 @@ import ( // A module registered in the catalogue from its own repository (the operator registers it with the repository and the // path of its manifest) had no manifest here: the indexer learns a repository only from a merged message, and no merge // followed the registration. The controller's outcome records name each registered module's repository, directory and -// commit (builds.go Registered), so the indexer reads that module's manifest at the repository's trunk head and writes the +// commit (builds.go Registered), so the indexer reads that module's manifest writes the // same keys a merge writes. A later merge to the repository's trunk reaches the module through the forge bridge's merged // message as for every other repository. @@ -25,7 +25,7 @@ type RepoRead struct { } // IndexRegistered writes the module of a registration as a merge of its repository would, when the modules bucket holds -// none of that name. Its commit is the head the gate's source names for the repository, else the registration's. The +// none of that name. Its commit is the registration's; a repository the gate's source names is left to the merges. The // repository's other modules are neither written nor dropped: they are the merges'. A manifest that is not at the path or // names another module is said and left. func (ix *Indexer) IndexRegistered(ctx context.Context, r Registration) error { @@ -44,40 +44,47 @@ func (ix *Indexer) IndexRegistered(ctx context.Context, r Registration) error { } repository := delivery.RepositoryName(r.Repository) commit := r.Commit + dir := strings.Trim(r.Path, "/") + if dir == delivery.RootPath { + dir = "" + } + // A repository the gate's source names is the merges': an unheld module there was removed by a merge, or its record is + // stale, and reading the repository for it would cost a forge call for each such module at every start. ix.mu.Lock() - for repo, head := range ix.source.heads { - if delivery.SameRepository(repo, r.Repository) && head != "" { - commit = head + owned := false + for repo := range ix.source.heads { + if delivery.SameRepository(repo, r.Repository) { + owned = true } } ix.mu.Unlock() + if owned { + return nil + } if repository == "" || commit == "" { ix.logf("%s: registered with no repository or commit, so its manifest is not read", r.Module) return nil } - // One try for each commit of a registration: a repository with no manifest where it was registered is said once, not at - // every record of the controller's builds. + // One try for each repository, commit and directory: a repository with no manifest where it was registered is said once, + // not at every record of the controller's builds. + tried := repository + "@" + commit + ":" + dir ix.mu.Lock() - if ix.tried[r.Module] == commit { + if ix.tried[tried] { ix.mu.Unlock() return nil } if ix.tried == nil { - ix.tried = map[string]string{} + ix.tried = map[string]bool{} } - ix.tried[r.Module] = commit + ix.tried[tried] = true ix.mu.Unlock() - read, err := ix.Read(ctx, repository, commit) + read, err := ix.Read(ctx, repository, commit, dir) if err != nil { ix.mu.Lock() - delete(ix.tried, r.Module) // a forge that could not say is asked again + delete(ix.tried, tried) // a forge that could not say is asked again ix.mu.Unlock() return fmt.Errorf("%s: reading %s at %s: %w", r.Module, repository, commit, err) } - dir := strings.Trim(r.Path, "/") - if dir == delivery.RootPath { - dir = "" - } raw, ok := read.Manifests[dir] if !ok || read.Trees[dir] == "" { ix.logf("%s: %s at %s holds no module.json at %q, so the registered module is not indexed", r.Module, repository, commit, dir) diff --git a/modules/mesh-delivery-indexer/internal/index/registered_test.go b/modules/mesh-delivery-indexer/internal/index/registered_test.go index 70e9779..a6bca70 100644 --- a/modules/mesh-delivery-indexer/internal/index/registered_test.go +++ b/modules/mesh-delivery-indexer/internal/index/registered_test.go @@ -12,32 +12,41 @@ func TestARegisteredModuleIsIndexedFromItsRepository(t *testing.T) { // the gran const forge = "http://novox.internal:20000/jschoubben/snake.git" snake := `{"module":"snake","version":"1"}` for _, tc := range []struct { - name string - path string - read RepoRead - err error - held bool // modules.snake exists already - other bool // modules.other of the same repository exists - want bool // modules.snake is written + name string + path string + read RepoRead + err error + ref string // the build record's ref + heads map[string]string // the gate source's heads + wantRead bool // the forge is read + held bool // modules.snake exists already + other bool // modules.other of the same repository exists + want bool // modules.snake is written }{ - {name: "a module at the root of its repository", path: "", want: true, + {wantRead: true, name: "a module at the root of its repository", path: "", want: true, read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}}, - {name: "a root spelled as a dot", path: ".", want: true, + {wantRead: true, name: "a root spelled as a dot", path: ".", want: true, read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}}, - {name: "a module in a directory", path: "modules/snake", want: true, + {wantRead: true, name: "a module in a directory", path: "modules/snake", want: true, read: RepoRead{Root: "r0", Trees: map[string]string{"modules/snake": "t1"}, Manifests: map[string]string{"modules/snake": snake}}}, - {name: "another module of the repository is neither written nor dropped", path: "", want: true, other: true, + {wantRead: true, name: "another module of the repository is neither written nor dropped", path: "", want: true, other: true, read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}}, {name: "a module the bucket holds is left", path: "", held: true, read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}}, - {name: "no manifest at the path", path: "modules/x", + {wantRead: true, name: "no manifest at the path", path: "modules/x", read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}}, - {name: "a manifest of another module", path: "", + {wantRead: true, name: "a manifest of another module", path: "", read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": `{"module":"other","version":"1"}`}}}, - {name: "a forge that cannot say", path: "", err: errors.New("down")}, + {wantRead: true, name: "a repository whose default branch is master", path: "", ref: "master", want: true, + read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}}, + {name: "a record of another branch", path: "", ref: "feat/x", + read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}}, + {name: "a repository the gate's source names is the merges'", path: "", heads: map[string]string{"jschoubben/snake": h2}, + read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}}, + {wantRead: true, name: "a forge that cannot say", path: "", err: errors.New("down")}, } { t.Run(tc.name, func(t *testing.T) { - ix, mods := modulesIndexer(t, rec(t, "build-1", "snake", forge, tc.path, "", "", "2026-10-19T09:00:00Z", "")) + ix, mods := modulesIndexer(t, rec(t, "build-1", "snake", forge, tc.path, tc.ref, "", "2026-10-19T09:00:00Z", "")) if tc.held { if err := mods.Create(context.Background(), "snake", mustModule(t, delivery.Module{Repository: "jschoubben/snake", Commit: h1, Path: ".", None2: "x"})); err != nil { t.Fatal(err) @@ -49,15 +58,32 @@ func TestARegisteredModuleIsIndexedFromItsRepository(t *testing.T) { // the gran } } ix.Manifests = newMem() - var asked struct{ repo, sha string } - ix.Read = func(_ context.Context, repo, sha string) (RepoRead, error) { - asked.repo, asked.sha = repo, sha + var asked struct { + repo, sha, dir string + n int + } + ix.Read = func(_ context.Context, repo, sha, dir string) (RepoRead, error) { + asked.repo, asked.sha, asked.dir = repo, sha, dir + asked.n++ return tc.read, tc.err } + if tc.heads != nil { + ix.source = sourceNow{rev: 1, heads: tc.heads} + } err := ix.IndexRegisteredAll(context.Background()) if (err != nil) != (tc.err != nil) { t.Fatalf("error = %v", err) } + wantReads := 1 + if !tc.wantRead { + wantReads = 0 + } + if tc.wantRead && asked.dir != dirOf(tc.path) { + t.Errorf("the forge was asked for directory %q", asked.dir) + } + if asked.n != wantReads { + t.Errorf("forge reads = %d, want %d", asked.n, wantReads) + } m, ok := moduleOf(t, mods, "snake") if tc.want { if !ok || m.Manifest == "" || m.Repository != "jschoubben/snake" || m.Commit != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" || m.Path != pathOf(dirOf(tc.path)) { -- 2.54.0 From 646f83642a01f65fe9890ff37fd8bd37038d39c1 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 11:41:46 +0100 Subject: [PATCH 4/4] Indexer: read a registered module at the source's head when it names one, else at the registered commit --- .../internal/index/registered.go | 20 ++++++++----------- .../internal/index/registered_test.go | 11 ++++++++-- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/index/registered.go b/modules/mesh-delivery-indexer/internal/index/registered.go index 248bc81..2ed1f30 100644 --- a/modules/mesh-delivery-indexer/internal/index/registered.go +++ b/modules/mesh-delivery-indexer/internal/index/registered.go @@ -11,8 +11,8 @@ import ( // A module registered in the catalogue from its own repository (the operator registers it with the repository and the // path of its manifest) had no manifest here: the indexer learns a repository only from a merged message, and no merge // followed the registration. The controller's outcome records name each registered module's repository, directory and -// commit (builds.go Registered), so the indexer reads that module's manifest writes the -// same keys a merge writes. A later merge to the repository's trunk reaches the module through the forge bridge's merged +// commit (builds.go Registered), so the indexer reads that module's manifest at the repository's head (the gate's source names it) or, with none named, at the +// registered commit, and writes the same keys a merge writes. A later merge to the repository's trunk reaches the module through the forge bridge's merged // message as for every other repository. // RepoRead is a repository at a commit as the forge bridge reads it for a merged message: the root tree, the tree of each @@ -25,7 +25,7 @@ type RepoRead struct { } // IndexRegistered writes the module of a registration as a merge of its repository would, when the modules bucket holds -// none of that name. Its commit is the registration's; a repository the gate's source names is left to the merges. The +// none of that name. Its commit is the head the gate's source names for the repository, else the registration's. The // repository's other modules are neither written nor dropped: they are the merges'. A manifest that is not at the path or // names another module is said and left. func (ix *Indexer) IndexRegistered(ctx context.Context, r Registration) error { @@ -48,19 +48,15 @@ func (ix *Indexer) IndexRegistered(ctx context.Context, r Registration) error { if dir == delivery.RootPath { dir = "" } - // A repository the gate's source names is the merges': an unheld module there was removed by a merge, or its record is - // stale, and reading the repository for it would cost a forge call for each such module at every start. + // The head the gate's source names for the repository, when it names one, else the registration's commit. A module a + // merge removed is absent at the head and is said and left. ix.mu.Lock() - owned := false - for repo := range ix.source.heads { - if delivery.SameRepository(repo, r.Repository) { - owned = true + for repo, head := range ix.source.heads { + if delivery.SameRepository(repo, r.Repository) && head != "" { + commit = head } } ix.mu.Unlock() - if owned { - return nil - } if repository == "" || commit == "" { ix.logf("%s: registered with no repository or commit, so its manifest is not read", r.Module) return nil diff --git a/modules/mesh-delivery-indexer/internal/index/registered_test.go b/modules/mesh-delivery-indexer/internal/index/registered_test.go index a6bca70..9098816 100644 --- a/modules/mesh-delivery-indexer/internal/index/registered_test.go +++ b/modules/mesh-delivery-indexer/internal/index/registered_test.go @@ -41,7 +41,7 @@ func TestARegisteredModuleIsIndexedFromItsRepository(t *testing.T) { // the gran read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}}, {name: "a record of another branch", path: "", ref: "feat/x", read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}}, - {name: "a repository the gate's source names is the merges'", path: "", heads: map[string]string{"jschoubben/snake": h2}, + {wantRead: true, name: "a repository the source names, the module unheld and present at the head", path: "", want: true, heads: map[string]string{"jschoubben/snake": h2}, read: RepoRead{Root: "r0", Trees: map[string]string{"": "t1"}, Manifests: map[string]string{"": snake}}}, {wantRead: true, name: "a forge that cannot say", path: "", err: errors.New("down")}, } { @@ -74,6 +74,13 @@ func TestARegisteredModuleIsIndexedFromItsRepository(t *testing.T) { // the gran if (err != nil) != (tc.err != nil) { t.Fatalf("error = %v", err) } + wantCommit := "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" // the record's + if tc.heads != nil { + wantCommit = h2 // the source's head + } + if tc.wantRead && asked.sha != wantCommit { + t.Errorf("the forge was asked at %s, want %s", asked.sha, wantCommit) + } wantReads := 1 if !tc.wantRead { wantReads = 0 @@ -86,7 +93,7 @@ func TestARegisteredModuleIsIndexedFromItsRepository(t *testing.T) { // the gran } m, ok := moduleOf(t, mods, "snake") if tc.want { - if !ok || m.Manifest == "" || m.Repository != "jschoubben/snake" || m.Commit != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" || m.Path != pathOf(dirOf(tc.path)) { + if !ok || m.Manifest == "" || m.Repository != "jschoubben/snake" || m.Commit != wantCommit || m.Path != pathOf(dirOf(tc.path)) { t.Fatalf("modules.snake = %+v, found %v", m, ok) } if len(ix.Manifests.(*memStore).vals) != 1 { -- 2.54.0