diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 54df569..d921599 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -25,6 +25,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-indexer/internal/index" ) @@ -283,7 +284,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro errs := make(chan error, 1) go func() { errs <- follow(ctx, w, each) }() - ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf} + // The indexer asks the forge for the objects of a manifest's named paths as its own account (ADR 0499 clause 3). + account, err := process.ReadAccount(path) + if err != nil { + return err + } + calls, err := process.Connect(account, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true)) + if err != nil { + return err + } + defer calls.Close() + ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf, + Named: namedVia(calls.Call)} // The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key // main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does // not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate. diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go new file mode 100644 index 0000000..f8fd619 --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go @@ -0,0 +1,48 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// commitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea): with `paths`, by module directory +// (the root spelled ""), the paths relative to it, it answers {root, trees, paths} with paths by directory then path, each +// path's git object id; a path not at the commit fails the call naming it (the interface agreed in R4e; mesh-sdk +// delivery.Forge.Paths; novox/hq ADR 0499 clause 3). +const commitTreesTool = "gitea.gitea_commit_trees" + +// toolCaller calls a module's tool, `.`, as the indexer's own account (mesh-sdk process.Conn.Call). +type toolCaller func(key string, body any) (json.RawMessage, error) + +// namedVia asks the forge for the objects of the named paths through call. An answer without the object of every path +// asked is an error naming it. +func namedVia(call toolCaller) func(ctx context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error) { + return func(_ context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error) { + owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/") + if !ok { + return nil, fmt.Errorf("%q is not /", repository) + } + raw, err := call(commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha, "paths": paths}) + if err != nil { + return nil, err + } + var a struct { + Paths map[string]map[string]string `json:"paths"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the answer of %s is not {root, trees, paths}: %w", commitTreesTool, err) + } + for dir, want := range paths { + for _, p := range want { + if a.Paths[dir][p] == "" { + return nil, fmt.Errorf("the answer of %s names no object for %q of %q", commitTreesTool, p, dir) + } + } + } + return a.Paths, nil + } +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go new file mode 100644 index 0000000..9691911 --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go @@ -0,0 +1,31 @@ +package main + +import ( + "context" + "encoding/json" + "testing" +) + +func TestNamedViaAsksTheTreesToolWithThePathsAndTakesTheirObjects(t *testing.T) { // I6, ADR 0499 clause 3 + var asked map[string]any + call := func(key string, body any) (json.RawMessage, error) { + if key != commitTreesTool { + t.Errorf("called %s", key) + } + asked = body.(map[string]any) + return json.RawMessage(`{"root":"r0","trees":{"":"r0"},"paths":{"":{"src":"1111111111111111111111111111111111111111","package.json":"3333333333333333333333333333333333333333"}}}`), nil + } + got, err := namedVia(call)(context.Background(), "novox/mesh-sdk", "s", map[string][]string{"": {"package.json", "src"}}) + if err != nil || got[""]["src"] != "1111111111111111111111111111111111111111" || got[""]["package.json"] == "" { + t.Fatalf("objects = %v, %v", got, err) + } + if asked["owner"] != "novox" || asked["repo"] != "mesh-sdk" || asked["sha"] != "s" || asked["paths"] == nil { + t.Errorf("asked %v, want the owner, repo, commit and paths", asked) + } + call = func(string, any) (json.RawMessage, error) { + return json.RawMessage(`{"root":"r0","trees":{},"paths":{"":{"src":"1111111111111111111111111111111111111111"}}}`), nil + } + if _, err := namedVia(call)(context.Background(), "novox/mesh-sdk", "s", map[string][]string{"": {"package.json", "src"}}); err == nil { + t.Error("an answer missing a path asked was taken") + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index d0baf34..89a1ac9 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -47,7 +47,12 @@ type Indexer struct { // Manifests is the manifests bucket; nil writes no manifests (issue 1449). Manifests Store Builds *Builds - Logf func(format string, args ...any) + // Named asks the forge for the git object of each named path of the module directories given, at a commit + // (gitea.gitea_commit_trees with paths; novox/hq ADR 0499 clauses 2 and 3): the indexer is the one reader of a + // manifest's paths (ADR 0495 clause 1, the lead's decision L8 of R4e). Nil asks nothing, and a module naming paths + // has none. + Named func(ctx context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error) + Logf func(format string, args ...any) mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach source sourceNow diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 137b035..7749238 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -5,6 +5,7 @@ import ( "maps" "slices" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -29,6 +30,7 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error { if err != nil { return err } + objects, namedErr := ix.namedObjects(ctx, f) overlay := map[string]delivery.Module{} dirOf := map[string]string{} for _, dir := range slices.Sorted(maps.Keys(f.Manifests)) { @@ -60,8 +62,11 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error { if next.Manifest == "" { next.None2 = "the manifest cannot be read: it cannot be kept as a value" } else { - in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], f.Paths[dir], ix.baseFP(ctx)) + in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], objects[dir], ix.baseFP(ctx)) next.Inputs2, next.None2, next.ArtifactPaths = in.Kept, in.None2, in.ArtifactPaths + if in.Kept == nil && namedErr[dir] != "" { + next.None2 = in.None2 + ": the forge could not say them: " + namedErr[dir] + } } overlay[key] = next } @@ -105,3 +110,42 @@ func pathOf(dir string) string { } func withV(m delivery.Module) delivery.Module { m.V = delivery.Version; return m } + +// namedObjects are, by module directory, the git objects of the paths each manifest of the merge names for every artifact +// (catalogue Manifest.NamedPaths): those the message carries, else asked of the forge at the merge commit in one call +// (Named). A forge that cannot say gives, for each directory asked, its words, so the module's reason names them. +func (ix *Indexer) namedObjects(ctx context.Context, f delivery.Forge) (map[string]map[string]string, map[string]string) { + objects := map[string]map[string]string{} + need := map[string][]string{} + for dir, raw := range f.Manifests { + m, err := catalogue.ParseManifestForInputs([]byte(raw)) + if err != nil { + continue // inputsOfNamed says why + } + paths, every := m.NamedPaths() + if !every { + continue + } + if carried := f.Paths[dir]; len(carried) > 0 { + objects[dir] = carried + continue + } + need[dir] = paths + } + if len(need) == 0 || ix.Named == nil { + return objects, nil + } + got, err := ix.Named(ctx, f.Repository, f.Merge, need) + if err != nil { + words := map[string]string{} + for dir := range need { + words[dir] = err.Error() + } + ix.logf("%s at %s: the objects of the named paths of %v: %v", f.Repository, f.Merge, slices.Sorted(maps.Keys(need)), err) + return objects, words + } + for dir := range need { + objects[dir] = got[dir] + } + return objects, nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/named_test.go b/modules/mesh-delivery-indexer/internal/index/named_test.go index 91bec02..13f99c5 100644 --- a/modules/mesh-delivery-indexer/internal/index/named_test.go +++ b/modules/mesh-delivery-indexer/internal/index/named_test.go @@ -2,6 +2,7 @@ package index import ( "context" + "errors" "reflect" "strings" "testing" @@ -89,3 +90,44 @@ func TestACandidatesModuleTakesTheNamedPathsObjectsFromThePreview(t *testing.T) t.Errorf("with a fingerprint: %+v, want the named paths' objects carried for the builder's fetch", got) } } + +func TestTheIndexerAsksTheForgeForTheNamedPathsObjectsOfAMerge(t *testing.T) { // I6 as the lead has it: the indexer is the one reader of paths + ix, mods, _ := manifestsIndexer(t) + type ask struct { + repository, sha string + paths map[string][]string + } + var asked []ask + ix.Named = func(_ context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error) { + asked = append(asked, ask{repository, sha, paths}) + return map[string]map[string]string{"": sdkObjects}, nil + } + raw := fixture(t, "sdk-9cb4d59-mesh-sdk") + f := withManifests(mergedMsg("novox/mesh-sdk", map[string]string{"": "roottree", "modules/a": "t"}), map[string]string{"": raw, "modules/a": `{"module":"a","version":"1"}`}, nil) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + if len(asked) != 1 || asked[0].repository != "novox/mesh-sdk" || asked[0].sha != merge || len(asked[0].paths) != 1 || + strings.Join(asked[0].paths[""], ",") != "package-lock.json,package.json,src,test,tsconfig.json" { + t.Fatalf("asked %+v, want the root's named paths of the merge commit, once, and nothing for a module naming none", asked) + } + m, _ := moduleOf(t, mods, "mesh-sdk") + want, _ := delivery.NamedTree(delivery.ManifestDigest([]byte(raw)), sdkObjects) + if m.Inputs2 == nil || m.Inputs2.Tree != want { + t.Errorf("module = %+v, want the named tree %s", m, want) + } +} + +func TestAForgeThatCannotSayTheObjectsGivesTheModuleNoneWithItsWords(t *testing.T) { // I6 + ix, mods, _ := manifestsIndexer(t) + ix.Named = func(context.Context, string, string, map[string][]string) (map[string]map[string]string, error) { + return nil, errors.New("gitea: the path tsconfig.json of \"\" is not at the commit") + } + raw := fixture(t, "sdk-9cb4d59-mesh-sdk") + if err := ix.Apply(context.Background(), withManifests(mergedMsg("novox/mesh-sdk", map[string]string{"": "roottree"}), map[string]string{"": raw}, nil)); err != nil { + t.Fatal(err) + } + if m, _ := moduleOf(t, mods, "mesh-sdk"); m.FP2 != "" || !strings.Contains(m.None2, "is not at the commit") { + t.Errorf("module = %+v, want none with the forge's words", m) + } +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 055782f..ce9784a 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -38,6 +38,7 @@ type manifest struct { Consumes []string `json:"consumes"` State []state `json:"state"` Reads []string `json:"reads"` + Invokes []string `json:"invokes"` OwnSecrets map[string]string `json:"own-secrets"` SecretsOwner string `json:"secrets-owner"` Resources []map[string]any `json:"resources"` @@ -93,8 +94,13 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build if m.SecretsOwner != "mesh-delivery-indexer" { t.Errorf("secrets-owner = %q", m.SecretsOwner) } - // No invokes, no emits, no streams, no bus, no seat, no tool (ADR 0437 §4). - for _, k := range []string{"invokes", "streams", "logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats", "bus"} { + // One invoke: the forge's trees with the named paths' objects, since the indexer is the one reader of a manifest's paths + // (ADR 0499 clauses 2 and 3, ADR 0495 clause 1, the lead's decision L8 of R4e). + if want := []string{"gitea.gitea_commit_trees"}; !reflect.DeepEqual(m.Invokes, want) { + t.Errorf("invokes = %v, want %v", m.Invokes, want) + } + // No emits, no streams, no bus, no seat, no tool (ADR 0437 §4). + for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats", "bus"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the indexer has none", k) } diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index cb58b8b..706ee06 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -20,6 +20,9 @@ "mesh-delivery-checker.previews", "mesh-delivery-gate.source" ], + "invokes": [ + "gitea.gitea_commit_trees" + ], "own-secrets": { "broker": "${dir:state}/broker" },