diff --git a/modules/mesh-delivery-indexer/internal/index/named.go b/modules/mesh-delivery-indexer/internal/index/named.go index 63a753b6..0de60062 100644 --- a/modules/mesh-delivery-indexer/internal/index/named.go +++ b/modules/mesh-delivery-indexer/internal/index/named.go @@ -67,6 +67,9 @@ type Named struct { gone map[string]time.Time // the want of a member removed (§693), when reads map[string]bool // the streams read whole at the start ready bool + // failed holds the first event that could not be applied while the start read ran: what it named is unknown, so the + // sets are not ready. + failed error } // NewNamed is the sets over store. @@ -124,6 +127,9 @@ func (n *Named) HearRunning(ctx context.Context, data []byte) error { func (n *Named) apply(ctx context.Context, member string, wanted bool, fp string, at time.Time, removed bool) error { n.mu.Lock() moved, err := n.applyLocked(ctx, member, wanted, fp, at, removed) + if err != nil && !n.ready && n.failed == nil { + n.failed = err + } n.mu.Unlock() if moved && n.Changed != nil { n.Changed(ctx) @@ -229,6 +235,16 @@ func (n *Named) ReadDone(ctx context.Context, stream string) error { n.mu.Unlock() return nil } + // A stream read empty, or an event of the read that could not be applied, is not "none names it": the sets stay not ready + // and no entry is removed until the next start reads them whole. + if len(n.want) == 0 || len(n.run) == 0 || n.failed != nil { + err := n.failed + if err == nil { + err = fmt.Errorf("the start read named %d wants and %d running builds: a stream read empty is not read", len(n.want), len(n.run)) + } + n.mu.Unlock() + return fmt.Errorf("no entry is removed: %w", err) + } lost, err := n.reconcileLocked(ctx) if err == nil { n.ready = true @@ -272,6 +288,13 @@ func (n *Named) reconcileLocked(ctx context.Context) (bool, error) { return lost, errors.Join(errs...) } +// Counts is how many members the wants and the running builds heard name, for what a removal says it read. +func (n *Named) Counts() (wants, running int) { + n.mu.Lock() + defer n.mu.Unlock() + return len(n.want), len(n.run) +} + // Members is the set of a fingerprint, sorted; none is an empty list. func (n *Named) Members(ctx context.Context, fingerprint string) ([]string, error) { return NamedMembers(ctx, n.Store, fingerprint) @@ -299,7 +322,18 @@ func (n *Named) IsNamed(ctx context.Context, fingerprint string) (bool, error) { n.mu.Lock() defer n.mu.Unlock() m, err := n.Members(ctx, fingerprint) - return err != nil || len(m) > 0, err + if err != nil || len(m) > 0 { + return true, err + } + // What was heard names it even when the write of its set failed or has not landed. + for _, src := range []map[string]namedAt{n.want, n.run} { + for _, v := range src { + if v.fp == fingerprint { + return true, nil + } + } + } + return false, nil } // Drop removes one member from one fingerprint's set by hand (indexer_named_drop), for a member a want or a running build diff --git a/modules/mesh-delivery-indexer/internal/index/revision_tools.go b/modules/mesh-delivery-indexer/internal/index/revision_tools.go index a5a235cf..476e25ca 100644 --- a/modules/mesh-delivery-indexer/internal/index/revision_tools.go +++ b/modules/mesh-delivery-indexer/internal/index/revision_tools.go @@ -149,6 +149,10 @@ func (ix *Indexer) removeUnlisted(ctx context.Context) ([]string, error) { // A revision that cannot be read may list any entry: none is removed rather than one it still needs. return nil, fmt.Errorf("no entry is removed while a revision cannot be read: %s", strings.Join(list.Unread, "; ")) } + if ix.NamedSets != nil { + w, r := ix.NamedSets.Counts() + ix.logf("catalog graph: removal reads %d wants and %d running builds, %d entries unlisted", w, r, len(list.Unlisted)) + } var removed []string for _, e := range list.Unlisted { fp, digest := ix.entryFingerprint(ctx, e) diff --git a/modules/mesh-delivery-indexer/internal/index/wanted_names_test.go b/modules/mesh-delivery-indexer/internal/index/wanted_names_test.go index 78a0f659..f3d23fc8 100644 --- a/modules/mesh-delivery-indexer/internal/index/wanted_names_test.go +++ b/modules/mesh-delivery-indexer/internal/index/wanted_names_test.go @@ -118,6 +118,7 @@ func TestAnEntryNoRevisionListsStaysWhileAWantNamesItAndGoesAfterTheWantMovesOn( t.Fatal("an entry was removed before the start's read of the wants and the running builds ended") } namedEvent{wanted: true, node: "n1", module: "avahi", fp: fp, min: 1}.apply(t, n) + namedEvent{node: "n9", module: "other", fp: namedFp("e"), min: 1}.apply(t, n) if err := n.ReadDone(ctx, NamedWanted); err != nil { t.Fatal(err) } @@ -155,6 +156,7 @@ func TestARestartReconcilesTheSetsWithWhatTheStreamsHoldNow(t *testing.T) { t.Fatalf("a restart removed from %v before it read both streams", got) } namedEvent{wanted: true, node: "n2", module: "m", fp: old, min: 1}.apply(t, after) + namedEvent{node: "n2", module: "m", fp: old, min: 1}.apply(t, after) _ = after.ReadDone(ctx, NamedWanted) _ = after.ReadDone(ctx, NamedRunning) if got, _ := after.Members(ctx, old); !slices.Equal(got, []string{"n2.m"}) { @@ -203,3 +205,105 @@ func TestNamedDropRemovesAMemberByHandAndEntriesReadsTheSet(t *testing.T) { } var _ stdio.Tool = tool } + +// §675 (the kind met on 2026-11-16: 76 entries removed at a start whose reads were refused): a start read that ended with a +// stream empty, or with an event that could not be applied, is not "none names it": no entry is removed. +func TestNoEntryIsRemovedWhileAStartReadWasEmptyOrAnEventWasNotApplied(t *testing.T) { + ctx := context.Background() + fp := namedFp("c") + setup := func(n *Named) (*Indexer, func() bool) { + ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), NamedSets: n, Now: func() time.Time { return namedBase }} + if err := ix.keepEntries(ctx, map[string]meshcatalog.CatalogEntry{"m": {Module: "m", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}); err != nil { + t.Fatal(err) + } + if err := ix.Revisions.Delete(ctx, "trunk.1"); err != nil { + t.Fatal(err) + } + return ix, func() bool { + _, err := Entry(ctx, ix.Revisions, ix.EntryIndex, meshcatalog.EntryInput{Fingerprint: fp}) + return err == nil + } + } + t.Run("a stream read empty", func(t *testing.T) { + n := NewNamed(newMem(), nil) + ix, found := setup(n) + namedEvent{wanted: true, node: "n1", module: "m", fp: namedFp("d"), min: 1}.apply(t, n) + _ = n.ReadDone(ctx, NamedWanted) + if err := n.ReadDone(ctx, NamedRunning); err == nil || n.Ready() { + t.Fatalf("the sets are ready after the running stream was read empty (err %v)", err) + } + ix.RemoveUnlisted(ctx) + if !found() { + t.Error("an entry was removed after a stream was read empty") + } + }) + t.Run("an event whose set was not written", func(t *testing.T) { + st := &failingPuts{Store: newMem()} + n := NewNamed(st, nil) + ix, found := setup(n) + st.fail = true // the state refuses writes, as during the outage + e := namedEvent{wanted: true, node: "n1", module: "m", fp: fp, min: 1} + at := namedBase.Add(time.Minute) + data, _ := json.Marshal(meshwants.WantedEvent{V: 1, Node: e.node, Module: e.module, Fp: e.fp, Revision: 1, At: &at}) + if err := n.HearWanted(ctx, data); err == nil { + t.Fatal("the write was refused and no error was said") + } + st.fail = false + namedEvent{node: "n1", module: "m", fp: fp, min: 1}.apply(t, n) + _ = n.ReadDone(ctx, NamedWanted) + if err := n.ReadDone(ctx, NamedRunning); err == nil || n.Ready() { + t.Fatalf("the sets are ready though an event of the read was not applied (err %v)", err) + } + ix.RemoveUnlisted(ctx) + if !found() { + t.Error("an entry a want names was removed after its set could not be written") + } + }) + t.Run("a set not written still names its entry once ready", func(t *testing.T) { + n := NewNamed(newMem(), nil) + ix, found := setup(n) + namedEvent{wanted: true, node: "n1", module: "m", fp: fp, min: 1}.apply(t, n) + namedEvent{node: "n2", module: "m", fp: namedFp("e"), min: 1}.apply(t, n) + _ = n.ReadDone(ctx, NamedWanted) + _ = n.ReadDone(ctx, NamedRunning) + for _, k := range mustKeys(t, n.Store) { // the state lost its keys (a store that was refused earlier) + _, rev, _, _ := n.Store.Get(ctx, k) + if err := n.Store.Delete(ctx, k, rev); err != nil { + t.Fatal(err) + } + } + ix.RemoveUnlisted(ctx) + if !found() { + t.Error("an entry a heard want names was removed because the state holds no key for it") + } + }) +} + +func mustKeys(t *testing.T, s Store) []string { + t.Helper() + k, err := s.Keys(context.Background()) + if err != nil { + t.Fatal(err) + } + return k +} + +// failingPuts is a Store whose writes are refused while fail is set. +type failingPuts struct { + Store + fail bool +} + +func (f *failingPuts) Create(ctx context.Context, key string, value []byte) error { + if f.fail { + return fmt.Errorf("refused") + } + return f.Store.Create(ctx, key, value) +} + +func (f *failingPuts) Update(ctx context.Context, key string, value []byte, revision uint64) error { + if f.fail { + return fmt.Errorf("refused") + } + return f.Store.Update(ctx, key, value, revision) +}