From 9f7e082688a14d6163d20bfd598ee940bf6c1a09 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:08:48 +0200 Subject: [PATCH] =?UTF-8?q?List=20a=20repository's=20modules=20from=20the?= =?UTF-8?q?=20merged=20message's=20manifests=20and=20the=20indexer's=20own?= =?UTF-8?q?=20bucket,=20not=20the=20controller's=20build=20records,=20so?= =?UTF-8?q?=20a=20new=20module=20gets=20a=20value=20and=20a=20key=20with?= =?UTF-8?q?=20no=20build=20asked=20(issue=201851,=20I1=20I2=20I4,=20ADR=20?= =?UTF-8?q?0501=20=C2=A76,=20ADR=200453=20=C2=A76)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I1: a merge writes each module its manifests name, at its directory. I2: a pull request's changed directory, a preview's touched module and a source revision's modules come from the modules bucket (a module a pull request adds is found in the preview by its manifest). I4: the value of today's form (fp, none) is no longer written, since it was computed from the build records; mesh-sdk keeps it readable until R4g. One commit: the three share applyMerged and its tests. Pins mesh-sdk feat/r4e-sdk 9cb4d59. --- .../mesh-delivery-indexer/integration_test.go | 24 +- modules/mesh-delivery-indexer/go.mod | 2 +- modules/mesh-delivery-indexer/go.sum | 2 + .../index/candidate_manifests_test.go | 4 +- .../internal/index/candidates.go | 12 +- .../internal/index/index_test.go | 32 ++- .../internal/index/live_builds_test.go | 33 --- .../internal/index/manifests.go | 17 +- .../internal/index/manifests_test.go | 5 +- .../internal/index/modules.go | 105 ++++---- .../internal/index/modules_test.go | 224 +++++++----------- .../internal/index/newform_test.go | 31 +-- .../internal/index/own.go | 81 +++++++ .../internal/index/previews.go | 27 ++- .../internal/index/source.go | 56 +++-- .../internal/index/source_test.go | 9 +- 16 files changed, 325 insertions(+), 339 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/own.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index a885d9d..2d49b96 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -183,30 +183,18 @@ func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue), b // TestIndexerWritesModulesAtAMerge: a module's record and its newest pointer are in mesh-controller_builds; a merged // message on FORGE makes modules. within 5 s, the fingerprint of the merge's tree and the recorded inputs. -func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9 - // Inputs and record in the controller's layout: SourceInputs under its Go field names (mesh-sdk - // go/delivery/testdata/build-record-build-agent.json, captured from the controller's encoder). - const inputs = `{"Module":"a","Tree":"oldtree","Bases":["registry.example/base@sha256:aaa"],"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}` - js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) { - rec := `{"id":"build-1","repository":"novox/mesh-delivery","ref":"main","module":"a","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/a","at":"2026-10-19T09:38:00Z","source-inputs":` + inputs + `}` - if _, err := builds.Put(ctx, delivery.BuildKey("a", "build-1"), []byte(rec)); err != nil { - t.Fatal(err) - } - if _, err := builds.Put(ctx, delivery.NewestKey("a"), []byte("build-1")); err != nil { - t.Fatal(err) - } - }) +func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1: the modules are the merged message's, no build record names them (issue 1851) + js, ctx, _, modules := startIndexer(t, func(context.Context, jetstream.KeyValue) {}) const merge = "89abcdef0123456789abcdef0123456789abcdef" data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, - Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: "r0", Trees: map[string]string{"modules/a": "newtree"}}) + Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: "r0", Trees: map[string]string{"modules/a": "newtree"}, + Manifests: map[string]string{"modules/a": `{"module":"a","version":"1"}`}}) if err != nil { t.Fatal(err) } if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.merged", data); err != nil { t.Fatal(err) } - want, _ := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"}, - Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}}) var m delivery.Module for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { if e, err := modules.Get(ctx, "a"); err == nil { @@ -215,8 +203,8 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9 } } } - if m.FP != want || m.Repository != "novox/mesh-delivery" || m.Commit != merge || m.Path != "modules/a" { // 1606/AC2 - t.Errorf("module = %+v within 5 s, want fp %s and path modules/a", m, want) + if m.Repository != "novox/mesh-delivery" || m.Commit != merge || m.Path != "modules/a" || m.Manifest == "" || m.FP != "" || m.None != "" { // 1606/AC2 + t.Errorf("module = %+v within 5 s, want path modules/a, its manifest, and no value of today's form", m) } } diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index a97df59..1f77135 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-indexer go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-indexer/go.sum b/modules/mesh-delivery-indexer/go.sum index 3515fb8..bb52af2 100644 --- a/modules/mesh-delivery-indexer/go.sum +++ b/modules/mesh-delivery-indexer/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7 h1:wK9O1CgNk7mo3IdJqW/CCx/hebg+1F0rkvW8CvJfnRM= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go b/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go index bdadf94..41c2f1b 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go +++ b/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go @@ -73,8 +73,8 @@ func TestAPullWithoutManifestsWritesTheCandidateAsToday(t *testing.T) { // 1451/ func TestTwoPullsGivingOneModuleDifferentManifestsAreRefusedNamingBoth(t *testing.T) { // 1451/AC3 avahi := catalogueManifest(t, "avahi") - // The second pull request carries another real manifest (ca-trust's, mesh-catalog a67f0196) for the same directory. - other := catalogueManifest(t, "ca-trust") + // The second pull request carries avahi's manifest changed (a line added at its end) for the same directory. + other := avahi + "\n" ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, avahi)); err != nil { t.Fatal(err) diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 2f1bf70..c322283 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -78,11 +78,15 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error { } } var digests map[string]string + var held map[string]delivery.Module if f.Kind == delivery.ForgePull { var err error if digests, err = ix.keepManifests(ctx, f); err != nil { return err } + if held, err = ix.heldModules(ctx); err != nil { + return err + } } if trunkLike[f.Branch] { ix.logf("%s#%d: its branch %q is no delivery group, so no candidate", f.Repository, f.Number, f.Branch) @@ -101,7 +105,7 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error { } cur = &c } - next := ix.next(cur, f, digests) + next := ix.next(cur, f, digests, held) if next != nil { ix.withPreviews(ctx, next) if err := ix.refuseDisagreement(*next); err != nil { @@ -207,8 +211,8 @@ func write(ctx context.Context, st Store, key string, compute func(value []byte, // A pull message that carries manifests (issue 1448) gives each module it touches, in the pull request's Fps, the digest of // its manifest and why it has no fingerprint of the new form (NoneNoPreview), and the candidate its FPSet (issue 1451); a // pull request whose head is held again without its directories keeps the Fps it had. digests are the manifests' digests by -// directory. -func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[string]string) *delivery.Candidate { +// directory; held are the modules the modules bucket holds, which name the module of a changed directory (moduleOfDir). +func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[string]string, held map[string]delivery.Module) *delivery.Candidate { var pulls []delivery.CandidatePull var base uint64 if cur != nil { @@ -225,7 +229,7 @@ func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[s switch { case f.DirsKnown: for _, dir := range f.Dirs { - m := ix.Builds.ModuleOf(f.Repository, dir) + m := moduleOfDir(held, f, dir) if !slices.Contains(entry.Modules, m) { entry.Modules = append(entry.Modules, m) } diff --git a/modules/mesh-delivery-indexer/internal/index/index_test.go b/modules/mesh-delivery-indexer/internal/index/index_test.go index 8fa182b..74bcc57 100644 --- a/modules/mesh-delivery-indexer/internal/index/index_test.go +++ b/modules/mesh-delivery-indexer/internal/index/index_test.go @@ -148,27 +148,35 @@ func TestTwoRepositoriesFormOneGroupByBranch(t *testing.T) { // 1238/AC2 (the gr } } -func TestModuleOfADirectory(t *testing.T) { // 1238/AC3 +func TestModuleOfADirectory(t *testing.T) { // 1238/AC3, as R4e has it: the module of a directory is the indexer's own (I2, ADR 0501 clause 6) ix, st, _ := newIndexer(t) - // modules/odd is named by a build record of the repository (module odd-name); modules/plain is named by none. - if err := ix.Apply(context.Background(), pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/odd", "modules/plain", "modules/odd")); err != nil { + mods := newMem() + ix.Modules = mods + ctx := context.Background() + // The trunk holds modules/odd as the module odd-name (its manifest names it so); modules/plain is held by none. + merged := mergedMsg("novox/mesh-delivery", map[string]string{"modules/odd": "t"}) + merged.Manifests["modules/odd"] = `{"module":"odd-name","version":"1"}` + if err := ix.Apply(ctx, merged); err != nil { + t.Fatal(err) + } + if err := ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/odd", "modules/plain", "modules/odd")); err != nil { t.Fatal(err) } c, _ := read(t, st, "feat/x") if strings.Join(c.Pulls[0].Modules, ",") != "odd-name,plain" { - t.Errorf("modules = %v, want the record's module for modules/odd, the last element for modules/plain, no repeats", c.Pulls[0].Modules) + t.Errorf("modules = %v, want the held module for modules/odd, the last element for modules/plain, no repeats", c.Pulls[0].Modules) } - // A record of another repository does not name this repository's directory. - if m, ok := ix.Builds.Module("novox/other", "modules/a"); ok { - t.Errorf("modules/a of novox/other is %q", m) + held := map[string]delivery.Module{"x": {Repository: "novox/other", Path: "modules/a"}} + if name, ok := moduleAt(held, "novox/mesh-delivery", "modules/a"); ok { + t.Errorf("modules/a of novox/mesh-delivery is %q, held only of novox/other", name) } - // The newest record of a directory wins. - ix.Builds.Put(delivery.BuildKey("renamed", "build-9"), buildRecord(t, "novox/mesh-delivery", "renamed", "modules/a", "2026-10-19T10:00:00Z")) - if m := ix.Builds.ModuleOf("novox/mesh-delivery", "modules/a"); m != "renamed" { - t.Errorf("modules/a = %q, want the newest record's module", m) + // A manifest the pull message carries names its module, whatever directory it is in. + f := pull("novox/mesh-delivery", 13, "feat/y", h2, "modules/odd") + f.Manifests = map[string]string{"modules/odd": `{"module":"renamed","version":"1"}`} + if name := moduleOfDir(held, f, "modules/odd"); name != "renamed" { + t.Errorf("modules/odd = %q, want the module its manifest names", name) } } - func TestDirsNotKnownKeepTheModulesOfTheSameHead(t *testing.T) { // 1238/AC4 ix, st, _ := newIndexer(t) ctx := context.Background() diff --git a/modules/mesh-delivery-indexer/internal/index/live_builds_test.go b/modules/mesh-delivery-indexer/internal/index/live_builds_test.go index 4794ab6..a196a54 100644 --- a/modules/mesh-delivery-indexer/internal/index/live_builds_test.go +++ b/modules/mesh-delivery-indexer/internal/index/live_builds_test.go @@ -1,7 +1,6 @@ package index import ( - "context" "testing" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -51,38 +50,6 @@ func liveIndexer(t *testing.T) (*Indexer, *memStore) { return ix, mods } -func TestMergeWritesEveryModuleTheLiveRecordsNameInTheRepository(t *testing.T) { // 1281/AC3 - ix, mods := liveIndexer(t) - msg := mergedMsg("novox/mesh-catalog", map[string]string{ - "modules/zsh": liveTree, "modules/mesh-delivery": "newtree", "modules/mailu": "newtree", "modules/sonarr": "newtree", - }) - if err := ix.Apply(context.Background(), msg); err != nil { - t.Fatal(err) - } - for _, name := range []string{"zsh", "mesh-delivery", "mailu"} { - m, ok := moduleOf(t, mods, name) - if !ok { - t.Errorf("%s: no module key written", name) - continue - } - if (m.FP == "") == (m.None == "") || m.Repository != "novox/mesh-catalog" || m.Commit != merge { - t.Errorf("%s: module = %+v, want an fp or a none, repository novox/mesh-catalog, commit %s", name, m, merge) - } - } - // the tree of zsh at the merge is the tree its inputs record, so its fingerprint is the one those inputs give - var in delivery.SourceInputs - in.Module, in.Tree = "zsh", liveTree - in.Bases = []string{"sha256:25f3cfeaceca14921366af4d1240b56457ef46273bdb508c7b0e8f469f6fd228"} - in.Toolchains = map[string]string{"code": "typescript sha256:4b81682df5665c1f461d3f720a3a0fd7593f4aa71e4fc272dbfdc4bbd235b489 20125bd590041690"} - want, _ := delivery.SourceFingerprint(in) - if m, _ := moduleOf(t, mods, "zsh"); m.FP != want { - t.Errorf("zsh fp = %q, want %q", m.FP, want) - } - if _, ok := mods.vals["sonarr"]; ok { - t.Error("the key of a module of mesh-media-catalog was written at a merge of mesh-catalog") - } -} - func TestChangedDirectoryMapsToTheModuleItsLiveRecordNamesByPath(t *testing.T) { // 1281/AC4 ix, _ := liveIndexer(t) for dir, want := range map[string]string{"modules/zsh": "zsh", "modules/mesh-delivery": "mesh-delivery", "modules/mailu/": "mailu"} { diff --git a/modules/mesh-delivery-indexer/internal/index/manifests.go b/modules/mesh-delivery-indexer/internal/index/manifests.go index d1883b1..740e9ce 100644 --- a/modules/mesh-delivery-indexer/internal/index/manifests.go +++ b/modules/mesh-delivery-indexer/internal/index/manifests.go @@ -2,27 +2,12 @@ package index import ( "context" - "encoding/json" "errors" "fmt" "git.novox.be/novox/mesh-sdk/go/delivery" ) -// moduleNameOf is the module a manifest is of: the name the manifest itself gives (read here only to key the manifests -// value, so one digest always has one module; the inputs are read by the catalogue's parser, inputs.go), else the -// module a build record names the directory the path of. -func (ix *Indexer) moduleNameOf(repository, dir string, raw string) string { - var m struct { - Module string `json:"module"` - } - if json.Unmarshal([]byte(raw), &m) == nil && m.Module != "" { - return m.Module - } - name, _ := ix.Builds.Module(repository, dir) - return name -} - // keepManifest writes manifests. for a manifest as the forge answered it, create-once (issue 1449): a key that // exists with the same value is left and is no error; one that holds another value is an error (it cannot happen for // one digest, ManifestDigest being a hash of the bytes, so it says the bucket was written by hand). It answers the @@ -66,7 +51,7 @@ func (ix *Indexer) keepManifest(ctx context.Context, module, raw string) (string func (ix *Indexer) keepManifests(ctx context.Context, f delivery.Forge) (map[string]string, error) { digests := make(map[string]string, len(f.Manifests)) for dir, raw := range f.Manifests { - module := ix.moduleNameOf(f.Repository, dir, raw) + module := manifestModule(raw) // the name the manifest gives itself: one digest always has one module digest, err := ix.keepManifest(ctx, module, raw) switch { case err == nil: diff --git a/modules/mesh-delivery-indexer/internal/index/manifests_test.go b/modules/mesh-delivery-indexer/internal/index/manifests_test.go index 8c26287..08455bb 100644 --- a/modules/mesh-delivery-indexer/internal/index/manifests_test.go +++ b/modules/mesh-delivery-indexer/internal/index/manifests_test.go @@ -96,7 +96,8 @@ func TestAManifestThatCannotBeAValueIsLeftOutAndSaid(t *testing.T) { // 1449 (a if len(manifests.vals) != 0 || len(logs) == 0 { t.Errorf("manifests %d keys, log %v", len(manifests.vals), logs) } - if mod, ok := moduleOf(t, mods, "avahi"); !ok || mod.Manifest != "" { - t.Errorf("module = %+v (%v), want no manifest digest", mod, ok) + // A manifest that is not JSON names no module (I1): nothing is written for its directory. + if mod, ok := moduleOf(t, mods, "avahi"); ok { + t.Errorf("module = %+v, want none written for a manifest naming no module", mod) } } diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 7e55e43..1503df6 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -2,78 +2,65 @@ package index import ( "context" - "sort" - "strings" + "maps" + "slices" "git.novox.be/novox/mesh-sdk/go/delivery" ) -// The reasons a module has no fingerprint (ADR 0437 ยง5; issue 1209's decision). -const ( - // NoneNarrow is for a module whose image compiles a Go program from a context: the context is the hash of the build - // source the builder handed it, not a git tree. - NoneNarrow = "an artifact compiles a Go program from a build source that git trees do not pin: no fingerprint before release 4" - // NoneContext is for a context that is a repository's root tree. - NoneContext = "an artifact reads the root tree of a repository the indexer does not know the tree of before release 4" - // NoneNoBuild is for a module no successful trunk build recorded the inputs of. - NoneNoBuild = "no successful trunk build recorded its inputs" -) - -// fingerprintOf is the module's fingerprint at the merge, or none and why: its directory's tree from the merged -// message and the bases, contexts and toolchains its newest successful trunk build recorded. -func fingerprintOf(k Known, trees map[string]string) (fp, none string) { - if k.Trunk == nil { - return "", NoneNoBuild - } - in := *k.Trunk.SourceInputs - artifacts := make([]string, 0, len(in.Contexts)) - for a := range in.Contexts { - artifacts = append(artifacts, a) - } - sort.Strings(artifacts) - for _, a := range artifacts { - if strings.HasPrefix(in.Contexts[a], "narrow:") { - return "", NoneNarrow - } - } - if len(artifacts) > 0 { - return "", NoneContext - } - in.Tree = trees[k.Path] // the directory "" is the repository's root - return delivery.SourceFingerprint(in) -} - -// applyMerged writes, for each module built from the merged repository, its fingerprint at the merge commit (or none and -// why), with the source revision that names the merge commit as the repository's head when the indexer has seen it -// (ApplySource), and Src 0 when it has not. A module of another repository is not written. A value the module already -// holds is not written again. From the manifests and lock hashes the message carries (the forge bridge, issue 1448) it -// also writes the module's manifest digest, its new-form inputs and why it has no new-form fingerprint (issues 1444, 1449, -// 1450); the fingerprint itself is computed by Recompute (toolchains.go) once the merge is written, and again when a build -// module reports its toolchain. +// applyMerged writes, for each module whose manifest the merged message carries (each directory of the merge commit that +// holds a module.json, the module its manifest names), its value at the merge commit: its directory, its manifest digest, +// its new-form inputs and why it has no new-form fingerprint yet, with the source revision that names the merge commit as +// the repository's head when the indexer has seen it (ApplySource), and Src 0 when it has not. The fingerprint itself is +// computed by Recompute (toolchains.go) once the merge is written, and again when a build module reports its toolchain. +// +// The modules are listed from the message, never from the controller's build records (novox/hq ADR 0501 clause 6, issue +// 1851), so a module added by the merge gets a value, and a fingerprint, with no build anywhere. The value of today's +// form (Module.FP, None) is no longer written: it was computed from those records (ADR 0453 clause 6). A module the +// bucket holds of another repository is left as it is and said: a move between repositories is the operator's (ADR 0420 +// clause 7). Two directories naming one module write the one the bucket holds, or the first by name, and say the other. +// A value the module already holds is not written again. func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error { digests, err := ix.keepManifests(ctx, f) if err != nil { return err } + held, err := ix.heldModules(ctx) + if err != nil { + return err + } overlay := map[string]delivery.Module{} - for _, k := range ix.Builds.Modules(f.Repository) { - key, err := delivery.ModulesKey(k.Module) - if err != nil { - ix.logf("%s: the module %q has no usable key, skipped: %v", f.Repository, k.Module, err) + dirOf := map[string]string{} + for _, dir := range slices.Sorted(maps.Keys(f.Manifests)) { + raw := f.Manifests[dir] + name := manifestModule(raw) + if name == "" { + ix.logf("%s at %s: the manifest in %q names no module, so no module is written for it", f.Repository, f.Merge, dir) continue } - fp, none := fingerprintOf(k, f.Trees) - next := delivery.Module{Repository: f.Repository, Commit: f.Merge, FP: fp, None: none, Src: ix.srcOf(f.Repository, f.Merge), Manifest: digests[k.Path], Path: pathOf(k.Path)} - // The new form (issue 1444, R4c-2): no module has a fingerprint of it yet, so each has the reason it has none, which - // names what is missing, from its manifest and lock hashes (inputs.go). - raw, has := f.Manifests[k.Path] - switch { - case !has: - next.None2 = None2NoManifest - case next.Manifest == "": + key, err := delivery.ModulesKey(name) + if err != nil { + ix.logf("%s: the module %q has no usable key, skipped: %v", f.Repository, name, err) + continue + } + cur, has := held[key] + if has && cur.Repository != "" && !delivery.SameRepository(cur.Repository, f.Repository) { + ix.logf("%s at %s: %q names the module %s, which the indexer holds of %s; left as it is (a move between repositories is the operator's)", f.Repository, f.Merge, dir, name, cur.Repository) + continue + } + if other, dup := dirOf[name]; dup { + if !has || cur.Path != pathOf(dir) { + ix.logf("%s at %s: %q and %q both name the module %s; %q is written", f.Repository, f.Merge, other, dir, name, other) + continue + } + ix.logf("%s at %s: %q and %q both name the module %s; %q, which the indexer holds, is written", f.Repository, f.Merge, other, dir, name, dir) + } + dirOf[name] = dir + next := delivery.Module{Repository: f.Repository, Commit: f.Merge, Src: ix.srcOf(f.Repository, f.Merge), Manifest: digests[dir], Path: pathOf(dir)} + if next.Manifest == "" { next.None2 = "the manifest cannot be read: it cannot be kept as a value" - default: - in := inputsOf(k.Module, raw, f.Trees[k.Path], f.Locks[k.Path], ix.baseFP(ctx)) + } else { + in := inputsOf(name, raw, f.Trees[dir], f.Locks[dir], ix.baseFP(ctx)) next.Inputs2, next.None2 = in.Kept, in.None2 } overlay[key] = next diff --git a/modules/mesh-delivery-indexer/internal/index/modules_test.go b/modules/mesh-delivery-indexer/internal/index/modules_test.go index 36eee4f..ca0cf36 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules_test.go +++ b/modules/mesh-delivery-indexer/internal/index/modules_test.go @@ -3,6 +3,7 @@ package index import ( "context" "encoding/json" + "path" "strings" "testing" @@ -51,8 +52,22 @@ func rec(t *testing.T, id, module, repo, path, ref, state, at, inputs string) [2 return [2]any{k, v} } +// mergedMsg is a merged message of the repository with these module directories' trees, each with a manifest naming its +// module, the directory's last element ("r" for the root), as the forge bridge sends it (one manifest per directory that +// holds a module.json). func mergedMsg(repo string, trees map[string]string) delivery.Forge { - return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repo, Branch: "feat/x", Number: 12, Head: h1, Merge: merge, Root: "r0", Trees: trees} + manifests := map[string]string{} + for dir := range trees { + manifests[dir] = `{"module":"` + nameOfDir(dir) + `","version":"1"}` + } + return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repo, Branch: "feat/x", Number: 12, Head: h1, Merge: merge, Root: "r0", Trees: trees, Manifests: manifests} +} + +func nameOfDir(dir string) string { + if dir == "" { + return "r" + } + return path.Base(dir) } func moduleOf(t *testing.T, st *memStore, key string) (delivery.Module, bool) { @@ -70,126 +85,6 @@ func moduleOf(t *testing.T, st *memStore, key string) (delivery.Module, bool) { const aInputs = `{"Module":"a","Tree":"oldtree","Bases":["registry.example/base@sha256:aaa"],"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}` -func TestModuleFingerprintFromTheMergeTreeAndTheRecordedInputs(t *testing.T) { // 1239/AC1 - ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs)) - if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree", "modules/b": "other"})); err != nil { - t.Fatal(err) - } - want, none := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"}, - Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}}) - if none != "" { - t.Fatal(none) - } - m, ok := moduleOf(t, mods, "a") - if !ok || m.FP != want || m.None != "" || m.Repository != "novox/mesh-delivery" || m.Commit != merge || m.Src != 0 { - t.Errorf("module = %+v (%v), want fp %s", m, ok, want) - } - if _, ok := mods.vals["b"]; ok { - t.Error("a key was written for a directory no build names") - } -} - -func TestMergeOfTheRecordedTreeReproducesTheRecordedFingerprint(t *testing.T) { // 1239/AC2 - // mailu, build-1792391134620131370: mesh-catalog at b4527d9091e972f28cb365623d9280045e7741a3, dir modules/mailu. Its - // inputs and recorded fingerprint are the live ones mesh-sdk go/delivery/fingerprint_test.go TestFingerprintLiveBuilds - // captured (its tree is `git rev-parse :modules/mailu`, its bases from the build's [bases] log line). - const tree = "cfc6fa8ff5fc1e220132e51ade653b9d3f387b3e" - const recorded = "src1:539f2e64000d436c572ef53b40e524012455945f8c5a1c6bb173ee0707e88ef4" - inputs := `{"Module":"mailu","Tree":"` + tree + `","Bases":["sha256:25f3cfeaceca14921366af4d1240b56457ef46273bdb508c7b0e8f469f6fd228"],"Contexts":null,` + - `"Toolchains":{"code":"typescript sha256:4b81682df5665c1f461d3f720a3a0fd7593f4aa71e4fc272dbfdc4bbd235b489 20125bd590041690"},"Unpinned":""}` - ix, mods := modulesIndexer(t, rec(t, "build-1792391134620131370", "mailu", "novox/mesh-catalog", "modules/mailu", "main", "", "2026-10-19T09:00:00Z", inputs)) - if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-catalog", map[string]string{"modules/mailu": tree})); err != nil { - t.Fatal(err) - } - if m, ok := moduleOf(t, mods, "mailu"); !ok || m.FP != recorded { - t.Errorf("module = %+v (%v), want fp %s", m, ok, recorded) - } -} - -func TestANarrowContextHasNoFingerprint(t *testing.T) { // 1239/AC3, 1209/AC1 - // A Go program's build source as the builder records it (build-agent's record, mesh-sdk go/delivery/testdata): - // the context "server" is "narrow:" and a hash. - inputs := `{"Module":"build-agent","Tree":"11c0b4f90cae0d6e4ef9cc48cbfdf9f6dbe52bca","Bases":["sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"],"Contexts":{"server":"narrow:d51ca744fd11af59d616c858f76c27b3e58e16e11fbdc0c720eb9d7d19d55fac"},"Toolchains":null,"Unpinned":""}` - ix, mods := modulesIndexer(t, rec(t, "build-1", "build-agent", "novox/mesh-controller", "modules/build-agent", "main", "", "2026-10-19T09:00:00Z", inputs)) - if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-controller", map[string]string{"modules/build-agent": "newtree"})); err != nil { - t.Fatal(err) - } - m, ok := moduleOf(t, mods, "build-agent") - if !ok || m.FP != "" || !strings.Contains(m.None, "Go program") || !strings.Contains(m.None, "release 4") { - t.Errorf("module = %+v (%v), want none naming the Go program's build source and release 4", m, ok) - } -} - -func TestUnpinnedBuildHasNoFingerprint(t *testing.T) { // 1239/AC4 - inputs := `{"Module":"a","Tree":"t","Bases":null,"Contexts":null,"Toolchains":null,"Unpinned":"the build resolves packages when built"}` - ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", inputs)) - _ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})) - if m, ok := moduleOf(t, mods, "a"); !ok || m.FP != "" || m.None != "the build resolves packages when built" { - t.Errorf("module = %+v (%v)", m, ok) - } -} - -func TestARootTreeContextHasNoFingerprint(t *testing.T) { // 1239/AC5 - inputs := `{"Module":"a","Tree":"t","Bases":null,"Contexts":{"web":"f00dcafef00dcafef00dcafef00dcafef00dcafe"},"Toolchains":null,"Unpinned":""}` - ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", inputs)) - _ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})) - m, ok := moduleOf(t, mods, "a") - if !ok || m.FP != "" || !strings.Contains(m.None, "root tree") || !strings.Contains(m.None, "release 4") { - t.Errorf("module = %+v (%v)", m, ok) - } -} - -func TestDirectoryGoneFromTheMergeHasNoTree(t *testing.T) { // 1239/AC6 - ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs)) - _ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/other": "t"})) - if m, ok := moduleOf(t, mods, "a"); !ok || m.FP != "" || m.None != "no tree" { - t.Errorf("module = %+v (%v)", m, ok) - } -} - -func TestAModuleWithNoSuccessfulTrunkBuildHasNone(t *testing.T) { // 1239/AC7 - ix, mods := modulesIndexer(t, - rec(t, "build-1", "failed-only", "novox/mesh-delivery", "modules/f", "main", "failed", "2026-10-19T09:00:00Z", aInputs), - rec(t, "build-2", "branch-only", "novox/mesh-delivery", "modules/g", "feat/x", "", "2026-10-19T09:00:00Z", aInputs), - rec(t, "build-3", "no-inputs", "novox/mesh-delivery", "modules/h", "main", "", "2026-10-19T09:00:00Z", ""), - ) - _ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/f": "t", "modules/g": "t", "modules/h": "t"})) - for _, name := range []string{"failed-only", "branch-only", "no-inputs"} { - if m, ok := moduleOf(t, mods, name); !ok || m.FP != "" || !strings.Contains(m.None, "no successful trunk build") { - t.Errorf("%s: module = %+v (%v)", name, m, ok) - } - } -} - -func TestTheNewestSuccessfulTrunkBuildCounts(t *testing.T) { // 1239 (the newest key is only a hint) - older := strings.Replace(aInputs, `"Bases":["registry.example/base@sha256:aaa"]`, `"Bases":["registry.example/base@sha256:old"]`, 1) - ix, mods := modulesIndexer(t, - rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", older), - rec(t, "build-2", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T10:00:00Z", aInputs), - rec(t, "build-3", "a", "novox/mesh-delivery", "modules/a", "main", "failed", "2026-10-19T11:00:00Z", older), - rec(t, "build-4", "a", "novox/mesh-delivery", "modules/a", "feat/x", "", "2026-10-19T12:00:00Z", older), - ) - _ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})) - want, _ := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"}, - Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}}) - if m, _ := moduleOf(t, mods, "a"); m.FP != want { - t.Errorf("fp = %s, want the 10:00 trunk build's %s", m.FP, want) - } -} - -func TestModulesOfOtherRepositoriesAreNotWritten(t *testing.T) { // 1239/AC8 - ix, mods := modulesIndexer(t, - rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs), - rec(t, "build-2", "c", "novox/mesh-catalog", "modules/c", "main", "", "2026-10-19T09:00:00Z", strings.Replace(aInputs, `"Module":"a"`, `"Module":"c"`, 1))) - _ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t", "modules/c": "t"})) - if _, ok := mods.vals["c"]; ok { - t.Error("the key of a module of another repository was written") - } - if _, ok := mods.vals["a"]; !ok { - t.Error("the key of the merged repository's module was not written") - } -} - func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value writes nothing) ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs)) m := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"}) @@ -211,21 +106,6 @@ func TestMergedMessageRemovesThePullFromItsGroupToo(t *testing.T) { // 1238/AC5 } } -func TestARecordAsTheControllerWritesItGivesAFingerprint(t *testing.T) { // 1281/AC8 - // The layout of mesh-controller internal/chain/records.go Build (commit 1f708e2d8d72): an outcome has no state key, a build - // asked by a plan after a merge no ref, and the repository is the clone URL. - raw := `{"id":"build-1","repository":"http://novox.internal:20000/novox/mesh-delivery.git","module":"a","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20",` + - `"path":"modules/a","source-fingerprint":"src1:ignored","source-inputs":` + aInputs + `,"at":"2026-10-19T09:00:00Z"}` - ix, mods := modulesIndexer(t, [2]any{delivery.BuildKey("a", "build-1"), []byte(raw)}) - if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil { - t.Fatal(err) - } - m, ok := moduleOf(t, mods, "a") - if !ok || m.FP == "" || m.None != "" { - t.Errorf("module = %+v (%v), want a fingerprint, not none", m, ok) - } -} - func TestAMergeWritesTheModulesDirectoryAndKeepsItThroughSourceAndRecompute(t *testing.T) { // 1606/AC1, AC2 ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs), @@ -263,3 +143,75 @@ func TestARootModuleHasTheDirectoryDot(t *testing.T) { // 1615/AC3: empty means t.Errorf("module = %+v (%v), want path %q", m, ok, delivery.RootPath) } } + +func TestAMergeAddingAModuleDirectoryWritesItsValueWithNoBuildRecordAnywhere(t *testing.T) { // I1: issue 1851, ADR 0501 clause 6 + ix, mods, _ := manifestsIndexer(t) // no build record of any module + avahi := catalogueManifest(t, "avahi") + f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, + map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}}) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + m, ok := moduleOf(t, mods, "avahi") + if !ok || m.Path != "modules/avahi" || m.Repository != catalogueRepo || m.Commit != merge || m.Manifest != delivery.ManifestDigest([]byte(avahi)) || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" { + t.Fatalf("module = %+v (%v), want its directory, merge, manifest and inputs from the message alone", m, ok) + } + if m.FP != "" || m.None != "" { + t.Errorf("module = %+v, want no value of today's form (I4: it came from the build records)", m) + } + if m.FP2 == "" && m.None2 == "" { + t.Errorf("module = %+v, want a new-form fingerprint or its reason", m) + } +} + +func TestAMergeWritesOnlyTheModulesItsManifestsName(t *testing.T) { // I1 + ix, mods := modulesIndexer(t, rec(t, "build-2", "c", "novox/mesh-catalog", "modules/c", "main", "", "2026-10-19T09:00:00Z", aInputs)) + f := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t", "modules/b": "t"}) + f.Trees["modules/c"] = "t" // a directory whose manifest the message does not carry + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + for name, want := range map[string]bool{"a": true, "b": true, "c": false} { + if _, ok := mods.vals[name]; ok != want { + t.Errorf("modules.%s written %v, want %v", name, ok, want) + } + } +} + +func TestAModuleHeldOfAnotherRepositoryIsLeftAndSaid(t *testing.T) { // I1: a move between repositories is the operator's (ADR 0420 clause 7) + ix, mods := modulesIndexer(t) + var logs []string + ix.Logf = func(format string, args ...any) { logs = append(logs, format) } + ctx := context.Background() + if err := ix.Apply(ctx, mergedMsg("novox/mesh-catalog", map[string]string{"modules/a": "t"})); err != nil { + t.Fatal(err) + } + before := string(mods.vals["a"]) + if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t2"})); err != nil { + t.Fatal(err) + } + if string(mods.vals["a"]) != before || len(logs) == 0 { + t.Errorf("modules.a = %s (was %s), logs %v: want it left, and said", mods.vals["a"], before, logs) + } +} + +func TestTwoDirectoriesNamingOneModuleWriteOneAndSayTheOther(t *testing.T) { // I1 + ix, mods := modulesIndexer(t) + var logs []string + ix.Logf = func(format string, args ...any) { logs = append(logs, format) } + f := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t1", "other/a": "t2"}) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + if m, ok := moduleOf(t, mods, "a"); !ok || m.Path != "modules/a" || len(logs) == 0 { + t.Errorf("module = %+v (%v), logs %v: want the first directory by name, and the other said", m, ok, logs) + } + // Once held, the directory the bucket holds stays the module's. + f = mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t1", "aaa/a": "t3"}) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + if m, _ := moduleOf(t, mods, "a"); m.Path != "modules/a" { + t.Errorf("module = %+v, want the held directory kept", m) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/newform_test.go b/modules/mesh-delivery-indexer/internal/index/newform_test.go index a610a9c..31247a6 100644 --- a/modules/mesh-delivery-indexer/internal/index/newform_test.go +++ b/modules/mesh-delivery-indexer/internal/index/newform_test.go @@ -7,17 +7,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" ) -// wantAvahiFP is today's fingerprint, at the tree "newtree", of a module with the recorded inputs aInputs (modules_test.go). -func wantAvahiFP(t *testing.T) string { - t.Helper() - fp, none := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"}, - Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}}) - if none != "" { - t.Fatal(none) - } - return fp -} - func TestAGoModuleAtAMergeHasItsManifestItsInputsAndNoFingerprintOfTheNewForm(t *testing.T) { // 1444/AC1 avahi := catalogueManifest(t, "avahi") ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", @@ -31,16 +20,14 @@ func TestAGoModuleAtAMergeHasItsManifestItsInputsAndNoFingerprintOfTheNewForm(t if !ok { t.Fatal("no module avahi") } - todays, _ := delivery.SourceFingerprint(delivery.SourceInputs{Module: "avahi", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"}, - Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}}) if m.Manifest != delivery.ManifestDigest([]byte(avahi)) || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" || m.Inputs2.Locks["tools"] != avahiGoSum { t.Errorf("module = %+v, want its manifest digest and inputs", m) } if m.None2 != "toolchain not yet reported" || m.FP2 != "" { t.Errorf("none2 = %q fp2 = %q, want none2 'toolchain not yet reported'", m.None2, m.FP2) } - if m.FP != todays || m.None != "" { - t.Errorf("fp = %q none = %q, want today's fingerprint %s unchanged", m.FP, m.None, todays) + if m.FP != "" || m.None != "" { + t.Errorf("fp = %q none = %q, want no value of today's form (I4)", m.FP, m.None) } } @@ -55,17 +42,15 @@ func TestAModuleWithNoToolchainArtifactNamesTheBuildersFingerprint(t *testing.T) } } -func TestAMergedMessageWithoutManifestsIsWrittenAsTodayWithItsReason(t *testing.T) { // 1444/AC3 +func TestAMergedMessageWithoutManifestsWritesNoModule(t *testing.T) { // 1444/AC3 as R4e has it: the modules are the manifests' (I1) ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs)) - if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil { + f := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"}) + f.Manifests = nil + if err := ix.Apply(context.Background(), f); err != nil { t.Fatal(err) } - m, ok := moduleOf(t, mods, "a") - if !ok || m.FP != wantAvahiFP(t) || m.None2 != "the message carries no manifest" || m.Inputs2 != nil || m.Manifest != "" || m.FP2 != "" { - t.Errorf("module = %+v (%v), want today's fingerprint, none2 'the message carries no manifest' and no inputs", m, ok) - } - if len(manifests.vals) != 0 { - t.Errorf("manifests holds %d keys", len(manifests.vals)) + if len(mods.vals) != 0 || len(manifests.vals) != 0 { + t.Errorf("modules %d keys, manifests %d keys, want none: a message naming no manifest names no module", len(mods.vals), len(manifests.vals)) } } diff --git a/modules/mesh-delivery-indexer/internal/index/own.go b/modules/mesh-delivery-indexer/internal/index/own.go new file mode 100644 index 0000000..bcb0f81 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/own.go @@ -0,0 +1,81 @@ +package index + +import ( + "context" + "encoding/json" + "fmt" + "path" + "strings" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// From R4e the indexer lists a repository's modules from what it holds itself (novox/hq ADR 0501 clause 6, issue 1851): +// a merged message's manifests name the modules of the merge, and the modules bucket names the modules it wrote before, +// with their repository and directory. The controller's build records name none of them (ADR 0453 clause 5 keeps only +// the builder's own input from them until R4g, builds.go). + +// manifestModule is the module a manifest names itself (its `module`), or "" when it names none or is not JSON. Only the +// name is read here; the manifest is read whole by the catalogue's parser (inputs.go). +func manifestModule(raw string) string { + var m struct { + Module string `json:"module"` + } + if json.Unmarshal([]byte(raw), &m) != nil { + return "" + } + return m.Module +} + +// heldModules are the values the modules bucket holds, by key (the module's name). A value that is not a module is left +// out and said. +func (ix *Indexer) heldModules(ctx context.Context) (map[string]delivery.Module, error) { + out := map[string]delivery.Module{} + if ix.Modules == nil { + return out, nil + } + keys, err := ix.Modules.Keys(ctx) + if err != nil { + return nil, fmt.Errorf("listing the modules: %w", err) + } + for _, key := range keys { + value, _, found, err := ix.Modules.Get(ctx, key) + if err != nil { + return nil, fmt.Errorf("modules.%s: %w", key, err) + } + if !found { + continue + } + m, err := delivery.DecodeModule(value) + if err != nil { + ix.logf("modules.%s holds what is not a module, left: %v", key, err) + continue + } + out[key] = m + } + return out, nil +} + +// moduleAt is the module whose directory in the repository is dir, as the modules bucket holds it. +func moduleAt(held map[string]delivery.Module, repository, dir string) (string, bool) { + want := pathOf(strings.Trim(dir, "/")) + for name, m := range held { + if m.Path == want && delivery.SameRepository(m.Repository, repository) { + return name, true + } + } + return "", false +} + +// moduleOfDir is the module of a changed directory of a pull request: the module its manifest in the message names, else +// the one the modules bucket holds at that directory of the repository, else the directory's last element (the +// catalogue's rule for nearly every manifest). +func moduleOfDir(held map[string]delivery.Module, f delivery.Forge, dir string) string { + if name := manifestModule(f.Manifests[dir]); name != "" { + return name + } + if name, ok := moduleAt(held, f.Repository, dir); ok { + return name + } + return path.Base(strings.Trim(dir, "/")) +} diff --git a/modules/mesh-delivery-indexer/internal/index/previews.go b/modules/mesh-delivery-indexer/internal/index/previews.go index ddd0a51..1f804c4 100644 --- a/modules/mesh-delivery-indexer/internal/index/previews.go +++ b/modules/mesh-delivery-indexer/internal/index/previews.go @@ -26,7 +26,7 @@ const ( NonePreviewNoModule = "the merge preview names no manifest for the module's directory" // NonePreviewNoManifest is for a module whose manifest at the merge the indexer does not hold. NonePreviewNoManifest = "the manifest of the merge is not kept" - // NoneNoDirectory is for a module whose directory in its repository the builds do not tell. + // NoneNoDirectory is for a module whose directory in its repository neither the modules bucket nor the preview tells. NoneNoDirectory = "the module's directory in its repository is not known" ) @@ -143,9 +143,27 @@ func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool if !found && !stale { continue } + // A touched module's directory: the one the modules bucket holds of the repository, else the preview's directory + // whose manifest is the one the pull request gave the module (a module the pull request adds; ADR 0501 clause 6). dirs := map[string]string{} - for _, k := range ix.Builds.Modules(p.Repository) { - dirs[k.Module] = k.Path + if held, err := ix.heldModules(ctx); err == nil { + for name, m := range held { + if delivery.SameRepository(m.Repository, p.Repository) && m.Path != "" { + dirs[name] = m.Path + } + } + } else { + ix.logf("the modules cannot be listed, so no touched module's directory is known: %v", err) + } + for _, m := range p.Modules { + if _, known := dirs[m]; known || p.Fps[m].Manifest == "" { + continue + } + for dir, pm := range pv.Modules { + if pm.Manifest == p.Fps[m].Manifest { + dirs[m] = dir + } + } } fps := maps.Clone(p.Fps) for _, m := range p.Modules { @@ -189,6 +207,9 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive if !ok && dir == "" { pm, ok = pv.Modules[delivery.RootPath] } + if !ok && dir == delivery.RootPath { + pm, ok = pv.Modules[""] + } if !ok { return delivery.CandidateModule{None2: NonePreviewNoModule} } diff --git a/modules/mesh-delivery-indexer/internal/index/source.go b/modules/mesh-delivery-indexer/internal/index/source.go index 2235218..0f2c781 100644 --- a/modules/mesh-delivery-indexer/internal/index/source.go +++ b/modules/mesh-delivery-indexer/internal/index/source.go @@ -34,7 +34,8 @@ func (ix *Indexer) srcOf(repository, merge string) uint64 { // ApplySource takes a revision of the gate's source.main in. An older revision than the latest seen is left. For each // repository the source names, each module built from it whose fingerprint was computed at the head the source names is // written again with Src the revision; a module computed at another commit is left (a merge heard later is stamped -// when it is written, applyMerged), and a repository none of the indexer's modules is built from writes nothing. +// when it is written, applyMerged), and a repository none of the indexer's modules is built from writes nothing. The +// modules of a repository are those the modules bucket holds of it (ADR 0501 clause 6). func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Source) error { ix.mu.Lock() if rev < ix.source.rev { @@ -46,34 +47,39 @@ func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Sourc if ix.Modules == nil { return ix.RecomputeCandidates(ctx) } - repositories := slices.Sorted(maps.Keys(s.Heads)) - for _, repository := range repositories { + held, err := ix.heldModules(ctx) + if err != nil { + return err + } + for _, key := range slices.Sorted(maps.Keys(held)) { + repository := "" + for r := range s.Heads { + if delivery.SameRepository(r, held[key].Repository) { + repository = r + } + } + if repository == "" { + continue + } head := s.Heads[repository] - for _, k := range ix.Builds.Modules(repository) { - key, err := delivery.ModulesKey(k.Module) - if err != nil { - ix.logf("%s: the module %q has no usable key, skipped: %v", repository, k.Module, err) - continue + err = write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) { + if !found { + return nil, false, nil } - err = write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) { - if !found { - return nil, false, nil - } - cur, err := delivery.DecodeModule(value) - if err != nil { - ix.logf("modules.%s holds what is not a module, left: %v", key, err) - return nil, false, nil - } - if cur.Repository != repository || cur.Commit != head || cur.Src == rev { - return nil, false, nil - } - cur.Src = rev - data, err := delivery.EncodeModule(cur) - return data, false, err - }) + cur, err := delivery.DecodeModule(value) if err != nil { - return fmt.Errorf("modules.%s: %w", key, err) + ix.logf("modules.%s holds what is not a module, left: %v", key, err) + return nil, false, nil } + if cur.Repository != repository || cur.Commit != head || cur.Src == rev { + return nil, false, nil + } + cur.Src = rev + data, err := delivery.EncodeModule(cur) + return data, false, err + }) + if err != nil { + return fmt.Errorf("modules.%s: %w", key, err) } } // A trunk that moved makes the previews made onto the old one out of date. diff --git a/modules/mesh-delivery-indexer/internal/index/source_test.go b/modules/mesh-delivery-indexer/internal/index/source_test.go index 634e216..d7b16a5 100644 --- a/modules/mesh-delivery-indexer/internal/index/source_test.go +++ b/modules/mesh-delivery-indexer/internal/index/source_test.go @@ -15,8 +15,7 @@ func sourceOf(repo, head string) delivery.Source { } func TestASourceRevisionStampsTheModulesOfTheRepositoriesHeadItNames(t *testing.T) { // 1344/AC1 - ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs), - rec(t, "build-2", "c", "novox/mesh-catalog", "modules/c", "main", "", "2026-10-19T09:00:00Z", aInputs)) + ix, mods := modulesIndexer(t) ctx := context.Background() if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil { t.Fatal(err) @@ -63,7 +62,7 @@ func TestAMergeHeardAfterTheSourceNamesItsHeadIsStampedWhenWritten(t *testing.T) if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil { t.Fatal(err) } - if m, ok := moduleOf(t, mods, "a"); !ok || m.Src != 7 || m.Commit != merge || m.FP == "" { + if m, ok := moduleOf(t, mods, "a"); !ok || m.Src != 7 || m.Commit != merge || m.Path != "modules/a" { t.Errorf("module a = %+v (%v), want the merge's fingerprint with Src 7", m, ok) } } @@ -75,8 +74,8 @@ func TestAMergeThatNoSourceRevisionNamesIsWrittenAsToday(t *testing.T) { // 1344 if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil { t.Fatal(err) } - if m, ok := moduleOf(t, mods, "a"); !ok || m.Src != 0 || m.FP == "" { - t.Fatalf("module a = %+v (%v), want a fingerprint with Src 0", m, ok) + if m, ok := moduleOf(t, mods, "a"); !ok || m.Src != 0 || m.Path != "modules/a" { + t.Fatalf("module a = %+v (%v), want the merge written with Src 0", m, ok) } // A source revision that names another head of the repository: a merge of the head it does not name keeps Src 0. if err := ix.ApplySource(ctx, 7, sourceOf("novox/mesh-delivery", otherMerge)); err != nil {