Test the kept manifests, the new-form reasons and the candidate's manifests against real files (issues 1449, 1444, 1451, design 47 R4c-2)

modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go

@@ -145,6 +145,10 @@ func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue)) (
 	if err != nil {
 		t.Fatal(err)
 	}
+	// The bucket of the manifests, made by the controller from the manifest's state entry manifests (history 1, issue 1449).
+	if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ManifestsBucket, History: delivery.ManifestsHistory}); err != nil {
+		t.Fatal(err)
+	}
 	builds, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.BuildsBucket, History: 5})
 	if err != nil {
 		t.Fatal(err)
@@ -203,3 +207,61 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9
 		t.Errorf("module = %+v within 5 s, want fp %s", m, want)
 	}
 }
+
+// TestIndexerKeepsManifestsAtAMerge: a merged message of mesh-catalog carrying the real manifest of the module avahi and the
+// hash of its go.sum (the bridge's Manifests and Locks, issue 1448) makes, within 5 s, modules.avahi with its manifest digest,
+// its inputs and the reason it has no fingerprint of the new form, and manifests.<digest> with the manifest byte for byte.
+func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4
+	// The manifest is avahi's module.json from mesh-catalog at a67f01963f0d9ef0913a63720b89e573ca7a16ef (copied into the
+	// index package's testdata); the lock hash is the sha256 of modules/avahi/go.sum there.
+	manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"
+	const inputs = `{"Module":"avahi","Tree":"oldtree","Bases":null,"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}`
+	js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) {
+		rec := `{"id":"build-1","repository":"novox/mesh-catalog","ref":"main","module":"avahi","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/avahi","at":"2026-10-19T09:38:00Z","source-inputs":` + inputs + `}`
+		if _, err := builds.Put(ctx, delivery.BuildKey("avahi", "build-1"), []byte(rec)); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := builds.Put(ctx, delivery.NewestKey("avahi"), []byte("build-1")); err != nil {
+			t.Fatal(err)
+		}
+	})
+	data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301,
+		Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0",
+		Trees:     map[string]string{"modules/avahi": "newtree"},
+		Manifests: map[string]string{"modules/avahi": string(manifest)},
+		Locks:     map[string]map[string]string{"modules/avahi": {"go.sum": goSum}}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.merged", data); err != nil {
+		t.Fatal(err)
+	}
+	var m delivery.Module
+	for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
+		if e, err := modules.Get(ctx, "avahi"); err == nil {
+			if m, err = delivery.DecodeModule(e.Value()); err == nil {
+				break
+			}
+		}
+	}
+	digest := delivery.ManifestDigest(manifest)
+	if m.Manifest != digest || m.None2 != "toolchain not yet reported" || m.FP2 != "" || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" ||
+		m.Inputs2.Locks["tools"] != goSum {
+		t.Errorf("module = %+v within 5 s, want its manifest digest %s, its inputs and none2 'toolchain not yet reported'", m, digest)
+	}
+	kv, err := js.KeyValue(ctx, delivery.ManifestsBucket)
+	if err != nil {
+		t.Fatal(err)
+	}
+	e, err := kv.Get(ctx, digest)
+	if err != nil {
+		t.Fatalf("manifests.%s: %v", digest, err)
+	}
+	if got, err := delivery.DecodeManifest(e.Value()); err != nil || got.Module != "avahi" || got.JSON != string(manifest) {
+		t.Errorf("manifest = %+v err %v, want avahi and the file's bytes", got, err)
+	}
+}

modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go

@@ -0,0 +1,105 @@
+package index
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+func manifestPull(t *testing.T, number int, head, manifest string) delivery.Forge {
+	f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: number, Head: head,
+		Action: delivery.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true}
+	if manifest != "" {
+		f.Manifests = map[string]string{"modules/avahi": manifest}
+	}
+	return f
+}
+
+func candidateOf(t *testing.T, st *memStore) delivery.Candidate {
+	t.Helper()
+	key, err := delivery.CandidatesKey("feat/x")
+	if err != nil {
+		t.Fatal(err)
+	}
+	raw, ok := st.vals[key]
+	if !ok {
+		t.Fatalf("candidates holds %d keys, none %s", len(st.vals), key)
+	}
+	c, err := delivery.DecodeCandidate(raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return c
+}
+
+func TestACandidateGivesEachTouchedModuleItsManifestAndTheReasonItHasNoPreview(t *testing.T) { // 1451/AC1
+	avahi := catalogueManifest(t, "avahi")
+	ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
+	if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, avahi)); err != nil {
+		t.Fatal(err)
+	}
+	c := candidateOf(t, ix.Store.(*memStore)) // DecodeCandidate has run Candidate.check
+	want := delivery.CandidateModule{Manifest: delivery.ManifestDigest([]byte(avahi)), None2: "no merge preview is computed"}
+	if len(c.Pulls) != 1 || len(c.Pulls[0].Fps) != 1 || c.Pulls[0].Fps["avahi"] != want {
+		t.Fatalf("pulls = %+v, want Fps[avahi] = %+v", c.Pulls, want)
+	}
+	if c.FPSet == "" || c.FPSet != delivery.FingerprintSet(c.Pulls) {
+		t.Errorf("fpset = %q, want %q", c.FPSet, delivery.FingerprintSet(c.Pulls))
+	}
+	// The same pull request heard again without its directories (the head unchanged) keeps what it had.
+	again := manifestPull(t, 301, h1, "")
+	again.Dirs, again.DirsKnown = nil, false
+	if err := ix.Apply(context.Background(), again); err != nil {
+		t.Fatal(err)
+	}
+	if c2 := candidateOf(t, ix.Store.(*memStore)); c2.Pulls[0].Fps["avahi"] != want || c2.FPSet != c.FPSet {
+		t.Errorf("after a message without directories: %+v fpset %q", c2.Pulls[0].Fps, c2.FPSet)
+	}
+}
+
+func TestAPullWithoutManifestsWritesTheCandidateAsToday(t *testing.T) { // 1451/AC2
+	ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
+	if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, "")); err != nil {
+		t.Fatal(err)
+	}
+	c := candidateOf(t, ix.Store.(*memStore))
+	if len(c.Pulls) != 1 || c.Pulls[0].Fps != nil || c.FPSet != "" || len(c.Pulls[0].Modules) != 1 || c.Pulls[0].Modules[0] != "avahi" {
+		t.Errorf("candidate = %+v, want the module avahi and no Fps and no fpset", c)
+	}
+}
+
+func TestTwoPullsGivingOneModuleDifferentManifestsAreRefusedNamingBoth(t *testing.T) { // 1451/AC3
+	avahi := catalogueManifest(t, "avahi")
+	// The second pull request carries another real manifest (ca-trust's, mesh-catalog a67f0196) for the same directory.
+	other := catalogueManifest(t, "ca-trust")
+	ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
+	if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, avahi)); err != nil {
+		t.Fatal(err)
+	}
+	before := candidateOf(t, ix.Store.(*memStore))
+	writes := ix.Store.(*memStore).writes
+	err := ix.Apply(context.Background(), manifestPull(t, 302, h2, other))
+	if !errors.Is(err, ErrUnreadable) {
+		t.Fatalf("err = %v, want a refusal", err)
+	}
+	for _, w := range []string{catalogueRepo + "#301", catalogueRepo + "#302", `"avahi"`} {
+		if !strings.Contains(err.Error(), w) {
+			t.Errorf("the refusal %q does not name %s", err, w)
+		}
+	}
+	// Candidate.check refuses the group too, which is what the refusal stands for.
+	if _, encErr := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat.x", HeadsHash: before.HeadsHash, Pulls: []delivery.CandidatePull{before.Pulls[0], {
+		Head: delivery.Head{Repository: catalogueRepo, Number: 302, Head: h2}, Modules: []string{"avahi"},
+		Fps: map[string]delivery.CandidateModule{"avahi": {Manifest: delivery.ManifestDigest([]byte(other)), None2: NoneNoPreview}}}}}); encErr == nil {
+		t.Error("the candidate check accepts two manifests for one module")
+	}
+	if ix.Store.(*memStore).writes != writes {
+		t.Error("a partial write was made")
+	}
+	if after := candidateOf(t, ix.Store.(*memStore)); len(after.Pulls) != 1 || after.FPSet != before.FPSet {
+		t.Errorf("the candidate changed: %+v", after)
+	}
+}

modules/mesh-delivery-indexer/internal/index/manifests_test.go

@@ -0,0 +1,102 @@
+package index
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+// The manifests these tests carry are real files, copied unchanged into testdata: the module.json of modules of
+// mesh-catalog at its main commit a67f01963f0d9ef0913a63720b89e573ca7a16ef (testdata/catalogue-a67f0196-<module>.module.json)
+// and of build-agent from mesh-controller at its main commit f2d7bc94 (testdata/controller-f2d7bc94-build-agent.module.json).
+func fixture(t *testing.T, name string) string {
+	t.Helper()
+	b, err := os.ReadFile("testdata/" + name + ".module.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(b)
+}
+
+func catalogueManifest(t *testing.T, module string) string {
+	return fixture(t, "catalogue-a67f0196-"+module)
+}
+
+// avahiGoSum is the sha256 of modules/avahi/go.sum in mesh-catalog at a67f0196 (sha256sum of that file).
+const avahiGoSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"
+
+const catalogueRepo = "novox/mesh-catalog"
+
+func manifestsIndexer(t *testing.T, recs ...[2]any) (*Indexer, *memStore, *memStore) {
+	t.Helper()
+	ix, mods := modulesIndexer(t, recs...)
+	manifests := newMem()
+	ix.Manifests = manifests
+	return ix, mods, manifests
+}
+
+func withManifests(f delivery.Forge, manifests map[string]string, locks map[string]map[string]string) delivery.Forge {
+	f.Manifests, f.Locks = manifests, locks
+	return f
+}
+
+func TestAMergedManifestIsKeptByItsDigestAndNamedByTheModule(t *testing.T) { // 1449/AC1
+	avahi := catalogueManifest(t, "avahi")
+	ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
+	f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, nil)
+	if err := ix.Apply(context.Background(), f); err != nil {
+		t.Fatal(err)
+	}
+	digest := delivery.ManifestDigest([]byte(avahi))
+	raw, ok := manifests.vals[digest]
+	if !ok || len(manifests.vals) != 1 {
+		t.Fatalf("manifests holds %d keys, want only %s", len(manifests.vals), digest)
+	}
+	m, err := delivery.DecodeManifest(raw)
+	if err != nil || m.Module != "avahi" || m.JSON != avahi {
+		t.Errorf("manifest = %+v err %v, want avahi and the bytes of the file", m, err)
+	}
+	if mod, ok := moduleOf(t, mods, "avahi"); !ok || mod.Manifest != digest {
+		t.Errorf("module = %+v (%v), want the manifest digest %s", mod, ok, digest)
+	}
+}
+
+func TestTheSameManifestAppliedAgainIsWrittenOnce(t *testing.T) { // 1449/AC2
+	avahi := catalogueManifest(t, "avahi")
+	ix, _, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
+	f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, nil)
+	if err := ix.Apply(context.Background(), f); err != nil {
+		t.Fatal(err)
+	}
+	// A second merge, and a pull message carrying it, the digest being the same.
+	f.Merge = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+	if err := ix.Apply(context.Background(), f); err != nil {
+		t.Errorf("a second merge: %v", err)
+	}
+	pull := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: 301, Head: h1,
+		Action: delivery.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true, Manifests: map[string]string{"modules/avahi": avahi}}
+	if err := ix.Apply(context.Background(), pull); err != nil {
+		t.Errorf("a pull message: %v", err)
+	}
+	if manifests.writes != 1 || len(manifests.vals) != 1 {
+		t.Errorf("manifests written %d times into %d keys, want once into one", manifests.writes, len(manifests.vals))
+	}
+}
+
+func TestAManifestThatCannotBeAValueIsLeftOutAndSaid(t *testing.T) { // 1449 (a manifest that is not JSON cannot be kept byte for byte)
+	var logs []string
+	ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
+	ix.Logf = func(format string, args ...any) { logs = append(logs, format) }
+	f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": "not json"}, nil)
+	if err := ix.Apply(context.Background(), f); err != nil {
+		t.Fatal(err)
+	}
+	if len(manifests.vals) != 0 || len(logs) == 0 {
+		t.Errorf("manifests %d keys, log %v", len(manifests.vals), logs)
+	}
+	if mod, ok := moduleOf(t, mods, "avahi"); !ok || mod.Manifest != "" {
+		t.Errorf("module = %+v (%v), want no manifest digest", mod, ok)
+	}
+}

modules/mesh-delivery-indexer/internal/index/newform_test.go

@@ -0,0 +1,85 @@
+package index
+
+import (
+	"context"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+// wantAvahiFP is today's fingerprint, at the tree "newtree", of a module with the recorded inputs aInputs (modules_test.go).
+func wantAvahiFP(t *testing.T) string {
+	t.Helper()
+	fp, none := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"},
+		Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}})
+	if none != "" {
+		t.Fatal(none)
+	}
+	return fp
+}
+
+func TestAGoModuleAtAMergeHasItsManifestItsInputsAndNoFingerprintOfTheNewForm(t *testing.T) { // 1444/AC1
+	avahi := catalogueManifest(t, "avahi")
+	ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z",
+		`{"Module":"avahi","Tree":"oldtree","Bases":["registry.example/base@sha256:aaa"],"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}`))
+	f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi},
+		map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}})
+	if err := ix.Apply(context.Background(), f); err != nil {
+		t.Fatal(err)
+	}
+	m, ok := moduleOf(t, mods, "avahi")
+	if !ok {
+		t.Fatal("no module avahi")
+	}
+	todays, _ := delivery.SourceFingerprint(delivery.SourceInputs{Module: "avahi", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"},
+		Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}})
+	if m.Manifest != delivery.ManifestDigest([]byte(avahi)) || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" || m.Inputs2.Locks["tools"] != avahiGoSum {
+		t.Errorf("module = %+v, want its manifest digest and inputs", m)
+	}
+	if m.None2 != "toolchain not yet reported" || m.FP2 != "" {
+		t.Errorf("none2 = %q fp2 = %q, want none2 'toolchain not yet reported'", m.None2, m.FP2)
+	}
+	if m.FP != todays || m.None != "" {
+		t.Errorf("fp = %q none = %q, want today's fingerprint %s unchanged", m.FP, m.None, todays)
+	}
+}
+
+func TestAModuleWithNoToolchainArtifactNamesTheBuildersFingerprint(t *testing.T) { // 1444/AC2
+	ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "ca-trust", catalogueRepo, "modules/ca-trust", "main", "", "2026-10-19T09:00:00Z", aInputs))
+	f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/ca-trust": "newtree"}), map[string]string{"modules/ca-trust": catalogueManifest(t, "ca-trust")}, nil)
+	if err := ix.Apply(context.Background(), f); err != nil {
+		t.Fatal(err)
+	}
+	if m, ok := moduleOf(t, mods, "ca-trust"); !ok || m.None2 != "builder's fingerprint not yet known" || m.Inputs2 == nil {
+		t.Errorf("module = %+v (%v), want none2 naming the builder's fingerprint", m, ok)
+	}
+}
+
+func TestAMergedMessageWithoutManifestsIsWrittenAsTodayWithItsReason(t *testing.T) { // 1444/AC3
+	ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
+	if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil {
+		t.Fatal(err)
+	}
+	m, ok := moduleOf(t, mods, "a")
+	if !ok || m.FP != wantAvahiFP(t) || m.None2 != "the message carries no manifest" || m.Inputs2 != nil || m.Manifest != "" || m.FP2 != "" {
+		t.Errorf("module = %+v (%v), want today's fingerprint, none2 'the message carries no manifest' and no inputs", m, ok)
+	}
+	if len(manifests.vals) != 0 {
+		t.Errorf("manifests holds %d keys", len(manifests.vals))
+	}
+}
+
+func TestTheSameMergeAppliedAgainWritesNothing(t *testing.T) { // 1444 (a value the module already holds is not written again, with inputs kept)
+	avahi := catalogueManifest(t, "avahi")
+	ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
+	f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi},
+		map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}})
+	for i := 0; i < 2; i++ {
+		if err := ix.Apply(context.Background(), f); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if mods.writes != 1 {
+		t.Errorf("modules written %d times, want once", mods.writes)
+	}
+}