diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index bf98098..dc88ca3 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -145,6 +145,10 @@ func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue)) ( if err != nil { t.Fatal(err) } + // The bucket of the manifests, made by the controller from the manifest's state entry manifests (history 1, issue 1449). + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ManifestsBucket, History: delivery.ManifestsHistory}); err != nil { + t.Fatal(err) + } builds, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.BuildsBucket, History: 5}) if err != nil { t.Fatal(err) @@ -203,3 +207,61 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9 t.Errorf("module = %+v within 5 s, want fp %s", m, want) } } + +// TestIndexerKeepsManifestsAtAMerge: a merged message of mesh-catalog carrying the real manifest of the module avahi and the +// hash of its go.sum (the bridge's Manifests and Locks, issue 1448) makes, within 5 s, modules.avahi with its manifest digest, +// its inputs and the reason it has no fingerprint of the new form, and manifests. with the manifest byte for byte. +func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4 + // The manifest is avahi's module.json from mesh-catalog at a67f01963f0d9ef0913a63720b89e573ca7a16ef (copied into the + // index package's testdata); the lock hash is the sha256 of modules/avahi/go.sum there. + manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json") + if err != nil { + t.Fatal(err) + } + const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381" + const inputs = `{"Module":"avahi","Tree":"oldtree","Bases":null,"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}` + js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) { + rec := `{"id":"build-1","repository":"novox/mesh-catalog","ref":"main","module":"avahi","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/avahi","at":"2026-10-19T09:38:00Z","source-inputs":` + inputs + `}` + if _, err := builds.Put(ctx, delivery.BuildKey("avahi", "build-1"), []byte(rec)); err != nil { + t.Fatal(err) + } + if _, err := builds.Put(ctx, delivery.NewestKey("avahi"), []byte("build-1")); err != nil { + t.Fatal(err) + } + }) + data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, + Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0", + Trees: map[string]string{"modules/avahi": "newtree"}, + Manifests: map[string]string{"modules/avahi": string(manifest)}, + Locks: map[string]map[string]string{"modules/avahi": {"go.sum": goSum}}}) + if err != nil { + t.Fatal(err) + } + if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.merged", data); err != nil { + t.Fatal(err) + } + var m delivery.Module + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + if e, err := modules.Get(ctx, "avahi"); err == nil { + if m, err = delivery.DecodeModule(e.Value()); err == nil { + break + } + } + } + digest := delivery.ManifestDigest(manifest) + if m.Manifest != digest || m.None2 != "toolchain not yet reported" || m.FP2 != "" || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" || + m.Inputs2.Locks["tools"] != goSum { + t.Errorf("module = %+v within 5 s, want its manifest digest %s, its inputs and none2 'toolchain not yet reported'", m, digest) + } + kv, err := js.KeyValue(ctx, delivery.ManifestsBucket) + if err != nil { + t.Fatal(err) + } + e, err := kv.Get(ctx, digest) + if err != nil { + t.Fatalf("manifests.%s: %v", digest, err) + } + if got, err := delivery.DecodeManifest(e.Value()); err != nil || got.Module != "avahi" || got.JSON != string(manifest) { + t.Errorf("manifest = %+v err %v, want avahi and the file's bytes", got, err) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go b/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go new file mode 100644 index 0000000..bdadf94 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go @@ -0,0 +1,105 @@ +package index + +import ( + "context" + "errors" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +func manifestPull(t *testing.T, number int, head, manifest string) delivery.Forge { + f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: number, Head: head, + Action: delivery.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true} + if manifest != "" { + f.Manifests = map[string]string{"modules/avahi": manifest} + } + return f +} + +func candidateOf(t *testing.T, st *memStore) delivery.Candidate { + t.Helper() + key, err := delivery.CandidatesKey("feat/x") + if err != nil { + t.Fatal(err) + } + raw, ok := st.vals[key] + if !ok { + t.Fatalf("candidates holds %d keys, none %s", len(st.vals), key) + } + c, err := delivery.DecodeCandidate(raw) + if err != nil { + t.Fatal(err) + } + return c +} + +func TestACandidateGivesEachTouchedModuleItsManifestAndTheReasonItHasNoPreview(t *testing.T) { // 1451/AC1 + avahi := catalogueManifest(t, "avahi") + ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) + if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, avahi)); err != nil { + t.Fatal(err) + } + c := candidateOf(t, ix.Store.(*memStore)) // DecodeCandidate has run Candidate.check + want := delivery.CandidateModule{Manifest: delivery.ManifestDigest([]byte(avahi)), None2: "no merge preview is computed"} + if len(c.Pulls) != 1 || len(c.Pulls[0].Fps) != 1 || c.Pulls[0].Fps["avahi"] != want { + t.Fatalf("pulls = %+v, want Fps[avahi] = %+v", c.Pulls, want) + } + if c.FPSet == "" || c.FPSet != delivery.FingerprintSet(c.Pulls) { + t.Errorf("fpset = %q, want %q", c.FPSet, delivery.FingerprintSet(c.Pulls)) + } + // The same pull request heard again without its directories (the head unchanged) keeps what it had. + again := manifestPull(t, 301, h1, "") + again.Dirs, again.DirsKnown = nil, false + if err := ix.Apply(context.Background(), again); err != nil { + t.Fatal(err) + } + if c2 := candidateOf(t, ix.Store.(*memStore)); c2.Pulls[0].Fps["avahi"] != want || c2.FPSet != c.FPSet { + t.Errorf("after a message without directories: %+v fpset %q", c2.Pulls[0].Fps, c2.FPSet) + } +} + +func TestAPullWithoutManifestsWritesTheCandidateAsToday(t *testing.T) { // 1451/AC2 + ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) + if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, "")); err != nil { + t.Fatal(err) + } + c := candidateOf(t, ix.Store.(*memStore)) + if len(c.Pulls) != 1 || c.Pulls[0].Fps != nil || c.FPSet != "" || len(c.Pulls[0].Modules) != 1 || c.Pulls[0].Modules[0] != "avahi" { + t.Errorf("candidate = %+v, want the module avahi and no Fps and no fpset", c) + } +} + +func TestTwoPullsGivingOneModuleDifferentManifestsAreRefusedNamingBoth(t *testing.T) { // 1451/AC3 + avahi := catalogueManifest(t, "avahi") + // The second pull request carries another real manifest (ca-trust's, mesh-catalog a67f0196) for the same directory. + other := catalogueManifest(t, "ca-trust") + ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) + if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, avahi)); err != nil { + t.Fatal(err) + } + before := candidateOf(t, ix.Store.(*memStore)) + writes := ix.Store.(*memStore).writes + err := ix.Apply(context.Background(), manifestPull(t, 302, h2, other)) + if !errors.Is(err, ErrUnreadable) { + t.Fatalf("err = %v, want a refusal", err) + } + for _, w := range []string{catalogueRepo + "#301", catalogueRepo + "#302", `"avahi"`} { + if !strings.Contains(err.Error(), w) { + t.Errorf("the refusal %q does not name %s", err, w) + } + } + // Candidate.check refuses the group too, which is what the refusal stands for. + if _, encErr := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat.x", HeadsHash: before.HeadsHash, Pulls: []delivery.CandidatePull{before.Pulls[0], { + Head: delivery.Head{Repository: catalogueRepo, Number: 302, Head: h2}, Modules: []string{"avahi"}, + Fps: map[string]delivery.CandidateModule{"avahi": {Manifest: delivery.ManifestDigest([]byte(other)), None2: NoneNoPreview}}}}}); encErr == nil { + t.Error("the candidate check accepts two manifests for one module") + } + if ix.Store.(*memStore).writes != writes { + t.Error("a partial write was made") + } + if after := candidateOf(t, ix.Store.(*memStore)); len(after.Pulls) != 1 || after.FPSet != before.FPSet { + t.Errorf("the candidate changed: %+v", after) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/manifests_test.go b/modules/mesh-delivery-indexer/internal/index/manifests_test.go new file mode 100644 index 0000000..8c26287 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/manifests_test.go @@ -0,0 +1,102 @@ +package index + +import ( + "context" + "os" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The manifests these tests carry are real files, copied unchanged into testdata: the module.json of modules of +// mesh-catalog at its main commit a67f01963f0d9ef0913a63720b89e573ca7a16ef (testdata/catalogue-a67f0196-.module.json) +// and of build-agent from mesh-controller at its main commit f2d7bc94 (testdata/controller-f2d7bc94-build-agent.module.json). +func fixture(t *testing.T, name string) string { + t.Helper() + b, err := os.ReadFile("testdata/" + name + ".module.json") + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func catalogueManifest(t *testing.T, module string) string { + return fixture(t, "catalogue-a67f0196-"+module) +} + +// avahiGoSum is the sha256 of modules/avahi/go.sum in mesh-catalog at a67f0196 (sha256sum of that file). +const avahiGoSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381" + +const catalogueRepo = "novox/mesh-catalog" + +func manifestsIndexer(t *testing.T, recs ...[2]any) (*Indexer, *memStore, *memStore) { + t.Helper() + ix, mods := modulesIndexer(t, recs...) + manifests := newMem() + ix.Manifests = manifests + return ix, mods, manifests +} + +func withManifests(f delivery.Forge, manifests map[string]string, locks map[string]map[string]string) delivery.Forge { + f.Manifests, f.Locks = manifests, locks + return f +} + +func TestAMergedManifestIsKeptByItsDigestAndNamedByTheModule(t *testing.T) { // 1449/AC1 + avahi := catalogueManifest(t, "avahi") + ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) + f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, nil) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + digest := delivery.ManifestDigest([]byte(avahi)) + raw, ok := manifests.vals[digest] + if !ok || len(manifests.vals) != 1 { + t.Fatalf("manifests holds %d keys, want only %s", len(manifests.vals), digest) + } + m, err := delivery.DecodeManifest(raw) + if err != nil || m.Module != "avahi" || m.JSON != avahi { + t.Errorf("manifest = %+v err %v, want avahi and the bytes of the file", m, err) + } + if mod, ok := moduleOf(t, mods, "avahi"); !ok || mod.Manifest != digest { + t.Errorf("module = %+v (%v), want the manifest digest %s", mod, ok, digest) + } +} + +func TestTheSameManifestAppliedAgainIsWrittenOnce(t *testing.T) { // 1449/AC2 + avahi := catalogueManifest(t, "avahi") + ix, _, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) + f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, nil) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + // A second merge, and a pull message carrying it, the digest being the same. + f.Merge = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + if err := ix.Apply(context.Background(), f); err != nil { + t.Errorf("a second merge: %v", err) + } + pull := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: 301, Head: h1, + Action: delivery.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true, Manifests: map[string]string{"modules/avahi": avahi}} + if err := ix.Apply(context.Background(), pull); err != nil { + t.Errorf("a pull message: %v", err) + } + if manifests.writes != 1 || len(manifests.vals) != 1 { + t.Errorf("manifests written %d times into %d keys, want once into one", manifests.writes, len(manifests.vals)) + } +} + +func TestAManifestThatCannotBeAValueIsLeftOutAndSaid(t *testing.T) { // 1449 (a manifest that is not JSON cannot be kept byte for byte) + var logs []string + ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) + ix.Logf = func(format string, args ...any) { logs = append(logs, format) } + f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": "not json"}, nil) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + if len(manifests.vals) != 0 || len(logs) == 0 { + t.Errorf("manifests %d keys, log %v", len(manifests.vals), logs) + } + if mod, ok := moduleOf(t, mods, "avahi"); !ok || mod.Manifest != "" { + t.Errorf("module = %+v (%v), want no manifest digest", mod, ok) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/newform_test.go b/modules/mesh-delivery-indexer/internal/index/newform_test.go new file mode 100644 index 0000000..a610a9c --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/newform_test.go @@ -0,0 +1,85 @@ +package index + +import ( + "context" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// wantAvahiFP is today's fingerprint, at the tree "newtree", of a module with the recorded inputs aInputs (modules_test.go). +func wantAvahiFP(t *testing.T) string { + t.Helper() + fp, none := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"}, + Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}}) + if none != "" { + t.Fatal(none) + } + return fp +} + +func TestAGoModuleAtAMergeHasItsManifestItsInputsAndNoFingerprintOfTheNewForm(t *testing.T) { // 1444/AC1 + avahi := catalogueManifest(t, "avahi") + ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", + `{"Module":"avahi","Tree":"oldtree","Bases":["registry.example/base@sha256:aaa"],"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}`)) + f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, + map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}}) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + m, ok := moduleOf(t, mods, "avahi") + if !ok { + t.Fatal("no module avahi") + } + todays, _ := delivery.SourceFingerprint(delivery.SourceInputs{Module: "avahi", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"}, + Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}}) + if m.Manifest != delivery.ManifestDigest([]byte(avahi)) || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" || m.Inputs2.Locks["tools"] != avahiGoSum { + t.Errorf("module = %+v, want its manifest digest and inputs", m) + } + if m.None2 != "toolchain not yet reported" || m.FP2 != "" { + t.Errorf("none2 = %q fp2 = %q, want none2 'toolchain not yet reported'", m.None2, m.FP2) + } + if m.FP != todays || m.None != "" { + t.Errorf("fp = %q none = %q, want today's fingerprint %s unchanged", m.FP, m.None, todays) + } +} + +func TestAModuleWithNoToolchainArtifactNamesTheBuildersFingerprint(t *testing.T) { // 1444/AC2 + ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "ca-trust", catalogueRepo, "modules/ca-trust", "main", "", "2026-10-19T09:00:00Z", aInputs)) + f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/ca-trust": "newtree"}), map[string]string{"modules/ca-trust": catalogueManifest(t, "ca-trust")}, nil) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + if m, ok := moduleOf(t, mods, "ca-trust"); !ok || m.None2 != "builder's fingerprint not yet known" || m.Inputs2 == nil { + t.Errorf("module = %+v (%v), want none2 naming the builder's fingerprint", m, ok) + } +} + +func TestAMergedMessageWithoutManifestsIsWrittenAsTodayWithItsReason(t *testing.T) { // 1444/AC3 + ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs)) + if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil { + t.Fatal(err) + } + m, ok := moduleOf(t, mods, "a") + if !ok || m.FP != wantAvahiFP(t) || m.None2 != "the message carries no manifest" || m.Inputs2 != nil || m.Manifest != "" || m.FP2 != "" { + t.Errorf("module = %+v (%v), want today's fingerprint, none2 'the message carries no manifest' and no inputs", m, ok) + } + if len(manifests.vals) != 0 { + t.Errorf("manifests holds %d keys", len(manifests.vals)) + } +} + +func TestTheSameMergeAppliedAgainWritesNothing(t *testing.T) { // 1444 (a value the module already holds is not written again, with inputs kept) + avahi := catalogueManifest(t, "avahi") + ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) + f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, + map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}}) + for i := 0; i < 2; i++ { + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + } + if mods.writes != 1 { + t.Errorf("modules written %d times, want once", mods.writes) + } +}