From 1dc9d80cfb58e245d10bf14cc38659ac623faa67 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 23 Oct 2026 14:44:14 +0200 Subject: [PATCH] Keep a key written ahead of the source head and a key with no path when reconciling (issue 1900) --- .../cmd/mesh-delivery-indexer/main.go | 2 +- .../cmd/mesh-delivery-indexer/named.go | 21 +++++++++++++++ .../internal/index/candidates.go | 4 ++- .../internal/index/inputs_test.go | 2 +- .../internal/index/modules.go | 22 ++++++++++++++- .../internal/index/remove_test.go | 27 +++++++++++++++++++ 6 files changed, 74 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 73c1f32..bb63b1b 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -295,7 +295,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } defer calls.Close() ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf, - Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call)} + Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call)} // The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key // main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does // not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate. diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go index 572b7a6..562c6a9 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go @@ -72,3 +72,24 @@ func trunkVia(call toolCaller) func(ctx context.Context, repository, sha string) return dirs, nil } } + +// containsVia asks the forge whether a commit is in another (gitea.gitea_contains: contains is true for the same commit or an ancestor). +func containsVia(call toolCaller) func(ctx context.Context, repository, commit, in string) (bool, error) { + return func(_ context.Context, repository, commit, in string) (bool, error) { + owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/") + if !ok { + return false, fmt.Errorf("%q is not /", repository) + } + raw, err := call("gitea.gitea_contains", map[string]any{"owner": owner, "repo": repo, "commit": commit, "in": in}) + if err != nil { + return false, err + } + var a struct { + Contains *bool `json:"contains"` + } + if err := json.Unmarshal(raw, &a); err != nil || a.Contains == nil { + return false, fmt.Errorf("the answer of gitea.gitea_contains names no contains: %v", err) + } + return *a.Contains, nil + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index e502672..7a14378 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -53,7 +53,9 @@ type Indexer struct { // has none. Named func(ctx context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error) Trunk func(ctx context.Context, repository, sha string) ([]string, error) - Logf func(format string, args ...any) + // Contains says whether a commit is the second or an ancestor of it (gitea.gitea_contains). + Contains func(ctx context.Context, repository, commit, in string) (bool, error) + Logf func(format string, args ...any) mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach source sourceNow diff --git a/modules/mesh-delivery-indexer/internal/index/inputs_test.go b/modules/mesh-delivery-indexer/internal/index/inputs_test.go index 1643349..5d5db4d 100644 --- a/modules/mesh-delivery-indexer/internal/index/inputs_test.go +++ b/modules/mesh-delivery-indexer/internal/index/inputs_test.go @@ -111,7 +111,7 @@ func TestABaseEntersByTheFingerprintOfItsModuleOrNamesItsAbsence(t *testing.T) { } // Through the indexer: modules.mesh-tools holds a fingerprint of the new form (a record written by a later release). ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) - mods.put("mesh-tools", mustEncode(t, delivery.Module{Repository: "novox/mesh-sdk", // a base held of another repository stays when the catalogue merges (issue 1900) + mods.put("mesh-tools", mustEncode(t, delivery.Module{Repository: "novox/mesh-tools", // a base held of another repository stays when the catalogue merges (issue 1900) Commit: merge, None: "x", FP2: fp, Manifest: strings.Repeat("a", 64)})) f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "t"}), map[string]string{"modules/avahi": aModuleWithABase(t)}, nil) if err := ix.Apply(context.Background(), f); err != nil { diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index e03714f..6dda09b 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -227,7 +227,27 @@ func (ix *Indexer) Reconcile(ctx context.Context) error { for _, d := range dirs { there[pathOf(strings.Trim(d, "/"))] = true } - if err := ix.dropRemoved(ctx, repository, mine, func(_ string, m delivery.Module) bool { return there[m.Path] }); err != nil { + head := heads[repository] + // The source can lag a merge already applied: a key written at a commit the head read does not hold (a merge heard + // first) is not read as removed, only one whose commit the head is, or contains. + stays := func(_ string, m delivery.Module) bool { + if m.Path == "" || there[m.Path] { + return true + } + if m.Commit == head { + return false + } + if m.Commit == "" || ix.Contains == nil { + return true + } + in, err := ix.Contains(ctx, repository, m.Commit, head) + if err != nil { + ix.logf("%s: whether %s is in %s cannot be read, so the module at %s is kept: %v", repository, m.Commit, head, m.Path, err) + return true + } + return !in + } + if err := ix.dropRemoved(ctx, repository, mine, stays); err != nil { return err } } diff --git a/modules/mesh-delivery-indexer/internal/index/remove_test.go b/modules/mesh-delivery-indexer/internal/index/remove_test.go index 9f6a128..012fbdc 100644 --- a/modules/mesh-delivery-indexer/internal/index/remove_test.go +++ b/modules/mesh-delivery-indexer/internal/index/remove_test.go @@ -3,6 +3,7 @@ package index import ( "context" "errors" + "strings" "testing" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -83,6 +84,7 @@ func TestStartingDeletesTheKeysTheTrunkDoesNotHold(t *testing.T) { // 1900/AC3 asked = repository + "@" + sha return []string{"modules/a"}, nil } + ix.Contains = func(context.Context, string, string, string) (bool, error) { return true, nil } if err := ix.Reconcile(ctx); err != nil { t.Fatal(err) } @@ -115,3 +117,28 @@ func TestStartingDeletesNothingWhenTheTrunkCannotBeRead(t *testing.T) { // 1900/ } _ = delivery.RootPath } + +func TestAKeyWrittenAheadOfTheSourceHeadStaysAndOneWithoutAPathIsSkipped(t *testing.T) { // 1900/AC3 (review): the source lags a merge already applied + ix, mods := twoModules(t) // written at the merge commit C + ctx := context.Background() + const older = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + if err := ix.ApplySource(ctx, 3, sourceOf(catalogueRepo, older)); err != nil { + t.Fatal(err) + } + mods.put("nopath", mustEncode(t, delivery.Module{Repository: catalogueRepo, Commit: older, None2: "x", Manifest: strings.Repeat("a", 64)})) + ix.Trunk = func(context.Context, string, string) ([]string, error) { return []string{"modules/a"}, nil } + ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { return commit == in, nil } // C is not in B + if err := ix.Reconcile(ctx); err != nil { + t.Fatal(err) + } + for _, k := range []string{"a", "b", "nopath"} { + if _, ok := mods.vals[k]; !ok { + t.Errorf("modules.%s was deleted", k) + } + } + ix.Contains = func(context.Context, string, string, string) (bool, error) { return false, errors.New("forge down") } + _ = ix.Reconcile(ctx) + if _, ok := mods.vals["b"]; !ok { + t.Error("modules.b deleted though whether its commit is in the head cannot be read") + } +}