indexer: no catalog graph entry is removed before what is wanted and running was read in full #230

Merged
mesh-tools merged 1 commits from fix/indexer-graph-removal-needs-full-read into main 2026-11-16 19:56:34 +00:00
3 changed files with 143 additions and 1 deletions
@@ -67,6 +67,9 @@ type Named struct {
gone map[string]time.Time // the want of a member removed (§693), when
reads map[string]bool // the streams read whole at the start
ready bool
// failed holds the first event that could not be applied while the start read ran: what it named is unknown, so the
// sets are not ready.
failed error
}
// NewNamed is the sets over store.
@@ -124,6 +127,9 @@ func (n *Named) HearRunning(ctx context.Context, data []byte) error {
func (n *Named) apply(ctx context.Context, member string, wanted bool, fp string, at time.Time, removed bool) error {
n.mu.Lock()
moved, err := n.applyLocked(ctx, member, wanted, fp, at, removed)
if err != nil && !n.ready && n.failed == nil {
n.failed = err
}
n.mu.Unlock()
if moved && n.Changed != nil {
n.Changed(ctx)
@@ -229,6 +235,16 @@ func (n *Named) ReadDone(ctx context.Context, stream string) error {
n.mu.Unlock()
return nil
}
// A stream read empty, or an event of the read that could not be applied, is not "none names it": the sets stay not ready
// and no entry is removed until the next start reads them whole.
if len(n.want) == 0 || len(n.run) == 0 || n.failed != nil {
err := n.failed
if err == nil {
err = fmt.Errorf("the start read named %d wants and %d running builds: a stream read empty is not read", len(n.want), len(n.run))
}
n.mu.Unlock()
return fmt.Errorf("no entry is removed: %w", err)
}
lost, err := n.reconcileLocked(ctx)
if err == nil {
n.ready = true
@@ -272,6 +288,13 @@ func (n *Named) reconcileLocked(ctx context.Context) (bool, error) {
return lost, errors.Join(errs...)
}
// Counts is how many members the wants and the running builds heard name, for what a removal says it read.
func (n *Named) Counts() (wants, running int) {
n.mu.Lock()
defer n.mu.Unlock()
return len(n.want), len(n.run)
}
// Members is the set of a fingerprint, sorted; none is an empty list.
func (n *Named) Members(ctx context.Context, fingerprint string) ([]string, error) {
return NamedMembers(ctx, n.Store, fingerprint)
@@ -299,7 +322,18 @@ func (n *Named) IsNamed(ctx context.Context, fingerprint string) (bool, error) {
n.mu.Lock()
defer n.mu.Unlock()
m, err := n.Members(ctx, fingerprint)
return err != nil || len(m) > 0, err
if err != nil || len(m) > 0 {
return true, err
}
// What was heard names it even when the write of its set failed or has not landed.
for _, src := range []map[string]namedAt{n.want, n.run} {
for _, v := range src {
if v.fp == fingerprint {
return true, nil
}
}
}
return false, nil
}
// Drop removes one member from one fingerprint's set by hand (indexer_named_drop), for a member a want or a running build
@@ -149,6 +149,10 @@ func (ix *Indexer) removeUnlisted(ctx context.Context) ([]string, error) {
// A revision that cannot be read may list any entry: none is removed rather than one it still needs.
return nil, fmt.Errorf("no entry is removed while a revision cannot be read: %s", strings.Join(list.Unread, "; "))
}
if ix.NamedSets != nil {
w, r := ix.NamedSets.Counts()
ix.logf("catalog graph: removal reads %d wants and %d running builds, %d entries unlisted", w, r, len(list.Unlisted))
}
var removed []string
for _, e := range list.Unlisted {
fp, digest := ix.entryFingerprint(ctx, e)
@@ -118,6 +118,7 @@ func TestAnEntryNoRevisionListsStaysWhileAWantNamesItAndGoesAfterTheWantMovesOn(
t.Fatal("an entry was removed before the start's read of the wants and the running builds ended")
}
namedEvent{wanted: true, node: "n1", module: "avahi", fp: fp, min: 1}.apply(t, n)
namedEvent{node: "n9", module: "other", fp: namedFp("e"), min: 1}.apply(t, n)
if err := n.ReadDone(ctx, NamedWanted); err != nil {
t.Fatal(err)
}
@@ -155,6 +156,7 @@ func TestARestartReconcilesTheSetsWithWhatTheStreamsHoldNow(t *testing.T) {
t.Fatalf("a restart removed from %v before it read both streams", got)
}
namedEvent{wanted: true, node: "n2", module: "m", fp: old, min: 1}.apply(t, after)
namedEvent{node: "n2", module: "m", fp: old, min: 1}.apply(t, after)
_ = after.ReadDone(ctx, NamedWanted)
_ = after.ReadDone(ctx, NamedRunning)
if got, _ := after.Members(ctx, old); !slices.Equal(got, []string{"n2.m"}) {
@@ -203,3 +205,105 @@ func TestNamedDropRemovesAMemberByHandAndEntriesReadsTheSet(t *testing.T) {
}
var _ stdio.Tool = tool
}
// §675 (the kind met on 2026-11-16: 76 entries removed at a start whose reads were refused): a start read that ended with a
// stream empty, or with an event that could not be applied, is not "none names it": no entry is removed.
func TestNoEntryIsRemovedWhileAStartReadWasEmptyOrAnEventWasNotApplied(t *testing.T) {
ctx := context.Background()
fp := namedFp("c")
setup := func(n *Named) (*Indexer, func() bool) {
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), NamedSets: n, Now: func() time.Time { return namedBase }}
if err := ix.keepEntries(ctx, map[string]meshcatalog.CatalogEntry{"m": {Module: "m", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}); err != nil {
t.Fatal(err)
}
if err := ix.Revisions.Delete(ctx, "trunk.1"); err != nil {
t.Fatal(err)
}
return ix, func() bool {
_, err := Entry(ctx, ix.Revisions, ix.EntryIndex, meshcatalog.EntryInput{Fingerprint: fp})
return err == nil
}
}
t.Run("a stream read empty", func(t *testing.T) {
n := NewNamed(newMem(), nil)
ix, found := setup(n)
namedEvent{wanted: true, node: "n1", module: "m", fp: namedFp("d"), min: 1}.apply(t, n)
_ = n.ReadDone(ctx, NamedWanted)
if err := n.ReadDone(ctx, NamedRunning); err == nil || n.Ready() {
t.Fatalf("the sets are ready after the running stream was read empty (err %v)", err)
}
ix.RemoveUnlisted(ctx)
if !found() {
t.Error("an entry was removed after a stream was read empty")
}
})
t.Run("an event whose set was not written", func(t *testing.T) {
st := &failingPuts{Store: newMem()}
n := NewNamed(st, nil)
ix, found := setup(n)
st.fail = true // the state refuses writes, as during the outage
e := namedEvent{wanted: true, node: "n1", module: "m", fp: fp, min: 1}
at := namedBase.Add(time.Minute)
data, _ := json.Marshal(meshwants.WantedEvent{V: 1, Node: e.node, Module: e.module, Fp: e.fp, Revision: 1, At: &at})
if err := n.HearWanted(ctx, data); err == nil {
t.Fatal("the write was refused and no error was said")
}
st.fail = false
namedEvent{node: "n1", module: "m", fp: fp, min: 1}.apply(t, n)
_ = n.ReadDone(ctx, NamedWanted)
if err := n.ReadDone(ctx, NamedRunning); err == nil || n.Ready() {
t.Fatalf("the sets are ready though an event of the read was not applied (err %v)", err)
}
ix.RemoveUnlisted(ctx)
if !found() {
t.Error("an entry a want names was removed after its set could not be written")
}
})
t.Run("a set not written still names its entry once ready", func(t *testing.T) {
n := NewNamed(newMem(), nil)
ix, found := setup(n)
namedEvent{wanted: true, node: "n1", module: "m", fp: fp, min: 1}.apply(t, n)
namedEvent{node: "n2", module: "m", fp: namedFp("e"), min: 1}.apply(t, n)
_ = n.ReadDone(ctx, NamedWanted)
_ = n.ReadDone(ctx, NamedRunning)
for _, k := range mustKeys(t, n.Store) { // the state lost its keys (a store that was refused earlier)
_, rev, _, _ := n.Store.Get(ctx, k)
if err := n.Store.Delete(ctx, k, rev); err != nil {
t.Fatal(err)
}
}
ix.RemoveUnlisted(ctx)
if !found() {
t.Error("an entry a heard want names was removed because the state holds no key for it")
}
})
}
func mustKeys(t *testing.T, s Store) []string {
t.Helper()
k, err := s.Keys(context.Background())
if err != nil {
t.Fatal(err)
}
return k
}
// failingPuts is a Store whose writes are refused while fail is set.
type failingPuts struct {
Store
fail bool
}
func (f *failingPuts) Create(ctx context.Context, key string, value []byte) error {
if f.fail {
return fmt.Errorf("refused")
}
return f.Store.Create(ctx, key, value)
}
func (f *failingPuts) Update(ctx context.Context, key string, value []byte, revision uint64) error {
if f.fail {
return fmt.Errorf("refused")
}
return f.Store.Update(ctx, key, value, revision)
}