Ask the forge for the objects of a merge's named paths in the indexer, the one reader of a manifest's paths, and narrow the module's tree from the answer (issue 1758, I6 reworked; ADR 0499 §2 §3, ADR 0495 §1, L8)

The manifest invokes gitea.gitea_commit_trees. A forge that cannot say gives the module none with its words.

modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go

@@ -25,6 +25,7 @@ import (
 	"github.com/nats-io/nats.go/jetstream"
 
 	"git.novox.be/novox/mesh-sdk/go/delivery"
+	"git.novox.be/novox/mesh-sdk/go/process"
 
 	"mesh-delivery-indexer/internal/index"
 )
@@ -283,7 +284,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
 	errs := make(chan error, 1)
 	go func() { errs <- follow(ctx, w, each) }()
 
-	ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf}
+	// The indexer asks the forge for the objects of a manifest's named paths as its own account (ADR 0499 clause 3).
+	account, err := process.ReadAccount(path)
+	if err != nil {
+		return err
+	}
+	calls, err := process.Connect(account, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
+	if err != nil {
+		return err
+	}
+	defer calls.Close()
+	ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf,
+		Named: namedVia(calls.Call)}
 	// The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key
 	// main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does
 	// not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate.

modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+// commitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea): with `paths`, by module directory
+// (the root spelled ""), the paths relative to it, it answers {root, trees, paths} with paths by directory then path, each
+// path's git object id; a path not at the commit fails the call naming it (the interface agreed in R4e; mesh-sdk
+// delivery.Forge.Paths; novox/hq ADR 0499 clause 3).
+const commitTreesTool = "gitea.gitea_commit_trees"
+
+// toolCaller calls a module's tool, `<module>.<tool>`, as the indexer's own account (mesh-sdk process.Conn.Call).
+type toolCaller func(key string, body any) (json.RawMessage, error)
+
+// namedVia asks the forge for the objects of the named paths through call. An answer without the object of every path
+// asked is an error naming it.
+func namedVia(call toolCaller) func(ctx context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error) {
+	return func(_ context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error) {
+		owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/")
+		if !ok {
+			return nil, fmt.Errorf("%q is not <owner>/<name>", repository)
+		}
+		raw, err := call(commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha, "paths": paths})
+		if err != nil {
+			return nil, err
+		}
+		var a struct {
+			Paths map[string]map[string]string `json:"paths"`
+		}
+		if err := json.Unmarshal(raw, &a); err != nil {
+			return nil, fmt.Errorf("the answer of %s is not {root, trees, paths}: %w", commitTreesTool, err)
+		}
+		for dir, want := range paths {
+			for _, p := range want {
+				if a.Paths[dir][p] == "" {
+					return nil, fmt.Errorf("the answer of %s names no object for %q of %q", commitTreesTool, p, dir)
+				}
+			}
+		}
+		return a.Paths, nil
+	}
+}

modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go

@@ -0,0 +1,31 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+)
+
+func TestNamedViaAsksTheTreesToolWithThePathsAndTakesTheirObjects(t *testing.T) { // I6, ADR 0499 clause 3
+	var asked map[string]any
+	call := func(key string, body any) (json.RawMessage, error) {
+		if key != commitTreesTool {
+			t.Errorf("called %s", key)
+		}
+		asked = body.(map[string]any)
+		return json.RawMessage(`{"root":"r0","trees":{"":"r0"},"paths":{"":{"src":"1111111111111111111111111111111111111111","package.json":"3333333333333333333333333333333333333333"}}}`), nil
+	}
+	got, err := namedVia(call)(context.Background(), "novox/mesh-sdk", "s", map[string][]string{"": {"package.json", "src"}})
+	if err != nil || got[""]["src"] != "1111111111111111111111111111111111111111" || got[""]["package.json"] == "" {
+		t.Fatalf("objects = %v, %v", got, err)
+	}
+	if asked["owner"] != "novox" || asked["repo"] != "mesh-sdk" || asked["sha"] != "s" || asked["paths"] == nil {
+		t.Errorf("asked %v, want the owner, repo, commit and paths", asked)
+	}
+	call = func(string, any) (json.RawMessage, error) {
+		return json.RawMessage(`{"root":"r0","trees":{},"paths":{"":{"src":"1111111111111111111111111111111111111111"}}}`), nil
+	}
+	if _, err := namedVia(call)(context.Background(), "novox/mesh-sdk", "s", map[string][]string{"": {"package.json", "src"}}); err == nil {
+		t.Error("an answer missing a path asked was taken")
+	}
+}

modules/mesh-delivery-indexer/internal/index/candidates.go

@@ -47,7 +47,12 @@ type Indexer struct {
 	// Manifests is the manifests bucket; nil writes no manifests (issue 1449).
 	Manifests Store
 	Builds    *Builds
-	Logf      func(format string, args ...any)
+	// Named asks the forge for the git object of each named path of the module directories given, at a commit
+	// (gitea.gitea_commit_trees with paths; novox/hq ADR 0499 clauses 2 and 3): the indexer is the one reader of a
+	// manifest's paths (ADR 0495 clause 1, the lead's decision L8 of R4e). Nil asks nothing, and a module naming paths
+	// has none.
+	Named func(ctx context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error)
+	Logf  func(format string, args ...any)
 
 	mu         sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach
 	source     sourceNow

modules/mesh-delivery-indexer/internal/index/modules.go

@@ -5,6 +5,7 @@ import (
 	"maps"
 	"slices"
 
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
 	"git.novox.be/novox/mesh-sdk/go/delivery"
 )
 
@@ -29,6 +30,7 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
 	if err != nil {
 		return err
 	}
+	objects, namedErr := ix.namedObjects(ctx, f)
 	overlay := map[string]delivery.Module{}
 	dirOf := map[string]string{}
 	for _, dir := range slices.Sorted(maps.Keys(f.Manifests)) {
@@ -60,8 +62,11 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
 		if next.Manifest == "" {
 			next.None2 = "the manifest cannot be read: it cannot be kept as a value"
 		} else {
-			in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], f.Paths[dir], ix.baseFP(ctx))
+			in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], objects[dir], ix.baseFP(ctx))
 			next.Inputs2, next.None2, next.ArtifactPaths = in.Kept, in.None2, in.ArtifactPaths
+			if in.Kept == nil && namedErr[dir] != "" {
+				next.None2 = in.None2 + ": the forge could not say them: " + namedErr[dir]
+			}
 		}
 		overlay[key] = next
 	}
@@ -105,3 +110,42 @@ func pathOf(dir string) string {
 }
 
 func withV(m delivery.Module) delivery.Module { m.V = delivery.Version; return m }
+
+// namedObjects are, by module directory, the git objects of the paths each manifest of the merge names for every artifact
+// (catalogue Manifest.NamedPaths): those the message carries, else asked of the forge at the merge commit in one call
+// (Named). A forge that cannot say gives, for each directory asked, its words, so the module's reason names them.
+func (ix *Indexer) namedObjects(ctx context.Context, f delivery.Forge) (map[string]map[string]string, map[string]string) {
+	objects := map[string]map[string]string{}
+	need := map[string][]string{}
+	for dir, raw := range f.Manifests {
+		m, err := catalogue.ParseManifestForInputs([]byte(raw))
+		if err != nil {
+			continue // inputsOfNamed says why
+		}
+		paths, every := m.NamedPaths()
+		if !every {
+			continue
+		}
+		if carried := f.Paths[dir]; len(carried) > 0 {
+			objects[dir] = carried
+			continue
+		}
+		need[dir] = paths
+	}
+	if len(need) == 0 || ix.Named == nil {
+		return objects, nil
+	}
+	got, err := ix.Named(ctx, f.Repository, f.Merge, need)
+	if err != nil {
+		words := map[string]string{}
+		for dir := range need {
+			words[dir] = err.Error()
+		}
+		ix.logf("%s at %s: the objects of the named paths of %v: %v", f.Repository, f.Merge, slices.Sorted(maps.Keys(need)), err)
+		return objects, words
+	}
+	for dir := range need {
+		objects[dir] = got[dir]
+	}
+	return objects, nil
+}

modules/mesh-delivery-indexer/internal/index/named_test.go

@@ -2,6 +2,7 @@ package index
 
 import (
 	"context"
+	"errors"
 	"reflect"
 	"strings"
 	"testing"
@@ -89,3 +90,44 @@ func TestACandidatesModuleTakesTheNamedPathsObjectsFromThePreview(t *testing.T)
 		t.Errorf("with a fingerprint: %+v, want the named paths' objects carried for the builder's fetch", got)
 	}
 }
+
+func TestTheIndexerAsksTheForgeForTheNamedPathsObjectsOfAMerge(t *testing.T) { // I6 as the lead has it: the indexer is the one reader of paths
+	ix, mods, _ := manifestsIndexer(t)
+	type ask struct {
+		repository, sha string
+		paths           map[string][]string
+	}
+	var asked []ask
+	ix.Named = func(_ context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error) {
+		asked = append(asked, ask{repository, sha, paths})
+		return map[string]map[string]string{"": sdkObjects}, nil
+	}
+	raw := fixture(t, "sdk-9cb4d59-mesh-sdk")
+	f := withManifests(mergedMsg("novox/mesh-sdk", map[string]string{"": "roottree", "modules/a": "t"}), map[string]string{"": raw, "modules/a": `{"module":"a","version":"1"}`}, nil)
+	if err := ix.Apply(context.Background(), f); err != nil {
+		t.Fatal(err)
+	}
+	if len(asked) != 1 || asked[0].repository != "novox/mesh-sdk" || asked[0].sha != merge || len(asked[0].paths) != 1 ||
+		strings.Join(asked[0].paths[""], ",") != "package-lock.json,package.json,src,test,tsconfig.json" {
+		t.Fatalf("asked %+v, want the root's named paths of the merge commit, once, and nothing for a module naming none", asked)
+	}
+	m, _ := moduleOf(t, mods, "mesh-sdk")
+	want, _ := delivery.NamedTree(delivery.ManifestDigest([]byte(raw)), sdkObjects)
+	if m.Inputs2 == nil || m.Inputs2.Tree != want {
+		t.Errorf("module = %+v, want the named tree %s", m, want)
+	}
+}
+
+func TestAForgeThatCannotSayTheObjectsGivesTheModuleNoneWithItsWords(t *testing.T) { // I6
+	ix, mods, _ := manifestsIndexer(t)
+	ix.Named = func(context.Context, string, string, map[string][]string) (map[string]map[string]string, error) {
+		return nil, errors.New("gitea: the path tsconfig.json of \"\" is not at the commit")
+	}
+	raw := fixture(t, "sdk-9cb4d59-mesh-sdk")
+	if err := ix.Apply(context.Background(), withManifests(mergedMsg("novox/mesh-sdk", map[string]string{"": "roottree"}), map[string]string{"": raw}, nil)); err != nil {
+		t.Fatal(err)
+	}
+	if m, _ := moduleOf(t, mods, "mesh-sdk"); m.FP2 != "" || !strings.Contains(m.None2, "is not at the commit") {
+		t.Errorf("module = %+v, want none with the forge's words", m)
+	}
+}

modules/mesh-delivery-indexer/manifest_test.go

@@ -38,6 +38,7 @@ type manifest struct {
 	Consumes     []string          `json:"consumes"`
 	State        []state           `json:"state"`
 	Reads        []string          `json:"reads"`
+	Invokes      []string          `json:"invokes"`
 	OwnSecrets   map[string]string `json:"own-secrets"`
 	SecretsOwner string            `json:"secrets-owner"`
 	Resources    []map[string]any  `json:"resources"`
@@ -93,8 +94,13 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
 	if m.SecretsOwner != "mesh-delivery-indexer" {
 		t.Errorf("secrets-owner = %q", m.SecretsOwner)
 	}
-	// No invokes, no emits, no streams, no bus, no seat, no tool (ADR 0437 §4).
-	for _, k := range []string{"invokes", "streams", "logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats", "bus"} {
+	// One invoke: the forge's trees with the named paths' objects, since the indexer is the one reader of a manifest's paths
+	// (ADR 0499 clauses 2 and 3, ADR 0495 clause 1, the lead's decision L8 of R4e).
+	if want := []string{"gitea.gitea_commit_trees"}; !reflect.DeepEqual(m.Invokes, want) {
+		t.Errorf("invokes = %v, want %v", m.Invokes, want)
+	}
+	// No emits, no streams, no bus, no seat, no tool (ADR 0437 §4).
+	for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats", "bus"} {
 		if _, ok := top[k]; ok {
 			t.Errorf("the manifest declares %q: the indexer has none", k)
 		}

modules/mesh-delivery-indexer/module.json

@@ -20,6 +20,9 @@
     "mesh-delivery-checker.previews",
     "mesh-delivery-gate.source"
   ],
+  "invokes": [
+    "gitea.gitea_commit_trees"
+  ],
   "own-secrets": {
     "broker": "${dir:state}/broker"
   },