From 1dd16bc783e7156ab8ff820e620fdc5507aab1e5 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 12:10:51 +0100 Subject: [PATCH 1/2] The kept trunk reads the newest revision again when the one named is removed under the read The pin judge removes a trunk revision, its list and its entries, as soon as a newer one is kept; take-ins keep one every few seconds, so the read of the newest revision failed with no such object and the first view of the catalog half never came. Builds: design 47 G A-487, A-487b Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../internal/index/catalog_revisions.go | 26 ++++++++--- .../internal/index/kept_trunk_test.go | 43 +++++++++++++++++++ 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index 7a104b44..68e0fce5 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -522,6 +522,9 @@ type KeptTrunk struct { manifests map[string]string } +// rereadTries is how many times a read names the newest revision again when the one it named was removed under it. +const rereadTries = 5 + // Reread reads the newest trunk revision of the bucket and holds its view; a bucket keeping none holds an empty view at 0. func (k *KeptTrunk) Reread(ctx context.Context, o Objects) error { k.read.Lock() @@ -537,16 +540,29 @@ func (k *KeptTrunk) reread(ctx context.Context, o Objects) error { } return err } - newest, any, err := newestTrunk(ctx, o) + // The pin judge removes a trunk revision, its list and the entries only it names, as soon as a newer one is kept, and + // revisions are kept seconds apart while take-ins run: the revision named newest can be gone before its entries are read. + // A newer one exists then, so the read names the newest again, a few times, before it fails. + var ( + newest meshcatalog.CatalogRevision + any bool + entries []meshcatalog.CatalogEntry + err error + ) + for try := 0; try < rereadTries; try++ { + entries = nil + if newest, any, err = newestTrunk(ctx, o); err == nil && any { + _, entries, err = ReadRevision(ctx, o, RevisionObject(newest)) + } + if !errors.Is(err, ErrNoObject) { + break + } + } if err != nil { return fail(err) } - var entries []meshcatalog.CatalogEntry mods, held, at := Store(newOverlay(nil)), Store(newOverlay(nil)), int64(0) if any { - if _, entries, err = ReadRevision(ctx, o, RevisionObject(newest)); err != nil { - return fail(err) - } if mods, held, at, err = viewOfEntries(newest.Revision, entries); err != nil { return fail(err) } diff --git a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go index c955644c..e3c6b01c 100644 --- a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go +++ b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go @@ -144,3 +144,46 @@ func TestTheVerbModuleReadsOneModule(t *testing.T) { t.Errorf("module nope = %v, want the trunk holds none", err) } } + +// goneUnderRead is a bucket whose pin judge runs while the first entry of the revision named newest is read: a newer revision +// is kept and the one named is removed, its list and its entries, as the indexer's judge does. +type goneUnderRead struct { + *memObjects + once atomic.Bool + prune func() +} + +func (g *goneUnderRead) Get(ctx context.Context, name string) ([]byte, error) { + if strings.HasPrefix(name, "entry.") && g.once.CompareAndSwap(false, true) { + g.prune() + } + return g.memObjects.Get(ctx, name) +} + +// A revision removed while it is read (take-ins keep one every few seconds, each removing the last) is read again as the newest +// one, never failed: on novox the first view of the catalog half never came while revisions 487 to 493 were kept. +func TestTheKeptTrunkReadsTheNewerRevisionWhenTheOneNamedIsRemovedUnderIt(t *testing.T) { + mem := newMemObjects(0) + keep := func(revision int64) { + data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built"}) + digest := fmt.Sprintf("d%d", revision) + rev := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}} + if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{digest: data}); err != nil { + t.Fatal(err) + } + } + keep(1) + objects := &goneUnderRead{memObjects: mem} + objects.prune = func() { + keep(2) + _ = mem.Delete(context.Background(), RevisionObject(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"})) + _ = mem.Delete(context.Background(), EntryObject("d1")) + } + kept := &KeptTrunk{} + if err := kept.Reread(context.Background(), objects); err != nil { + t.Fatalf("the read failed when the revision named was removed under it: %v", err) + } + if rev, _, ok, err := kept.Newest(context.Background(), objects); err != nil || !ok || rev.Revision != 2 { + t.Errorf("the view holds revision %d (%v, %v), want 2", rev.Revision, ok, err) + } +} -- 2.54.0 From 33143527f29bf904863e46dd0d14e3b2b786dab3 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 19 Nov 2026 12:17:40 +0100 Subject: [PATCH 2/2] A read of the newest revision names the newest again while it is newer than the one that failed Replaces the five tries: progress bounds the loop, the same revision failing twice is the read's failure. One helper, readNewestOf, serves the kept trunk's reads, TrunkView and BranchView; a read by number still fails loudly. Builds: design 47 G A-487d (hq 61bf1215), A-487, A-487b Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../internal/index/catalog_revisions.go | 55 ++++++++++--------- .../internal/index/kept_trunk_test.go | 42 ++++++++++++++ 2 files changed, 70 insertions(+), 27 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index 68e0fce5..4772043a 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -182,7 +182,7 @@ func newestOf(ctx context.Context, o Objects, branch string) (meshcatalog.Catalo name := RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: newest}) data, err := o.Get(ctx, name) if err != nil { - return meshcatalog.CatalogRevision{}, false, err + return meshcatalog.CatalogRevision{Branch: branch, Revision: newest}, false, err // the number that failed, for readNewestOf } var rev meshcatalog.CatalogRevision if err := json.Unmarshal(data, &rev); err != nil { @@ -191,6 +191,31 @@ func newestOf(ctx context.Context, o Objects, branch string) (meshcatalog.Catalo return rev, true, nil } +// readNewestOf is the newest revision of a branch and its entries whole (design 47 G A-487d). The pin judge removes a revision +// as soon as a newer one is kept, so a revision whose list or entry is removed under the read is read again as the newest, +// while the newest is newer than the one that failed; the same revision failing twice is the read's failure. Progress bounds +// the loop. No revision kept is false. +func readNewestOf(ctx context.Context, o Objects, branch string) (meshcatalog.CatalogRevision, []meshcatalog.CatalogEntry, bool, error) { + var failed int64 + for { + newest, any, err := newestOf(ctx, o, branch) + if err == nil && !any { + return newest, nil, false, nil + } + var entries []meshcatalog.CatalogEntry + if err == nil { + _, entries, err = ReadRevision(ctx, o, RevisionObject(newest)) + if err == nil { + return newest, entries, true, nil + } + } + if !errors.Is(err, ErrNoObject) || newest.Revision == 0 || newest.Revision <= failed { + return newest, nil, false, err + } + failed = newest.Revision + } +} + // changedEntries are the modules whose entry differs between two revisions' lists, one gone or added included, by name. func changedEntries(before, after []meshcatalog.EntryRef) []string { was := map[string]string{} @@ -438,14 +463,10 @@ func TrunkView(ctx context.Context, o Objects) (modules, nodes Store, at int64, // BranchView is TrunkView of a watched branch: the view read from the newest revision the bucket keeps of that branch // (design 47 G ยง291, A-93), empty and at 0 when it keeps none. func BranchView(ctx context.Context, o Objects, branch string) (modules, nodes Store, at int64, err error) { - newest, any, err := newestOf(ctx, o, branch) + newest, entries, any, err := readNewestOf(ctx, o, branch) if err != nil || !any { return newOverlay(nil), newOverlay(nil), 0, err } - _, entries, err := ReadRevision(ctx, o, RevisionObject(newest)) - if err != nil { - return nil, nil, 0, err - } return viewOfEntries(newest.Revision, entries) } @@ -522,9 +543,6 @@ type KeptTrunk struct { manifests map[string]string } -// rereadTries is how many times a read names the newest revision again when the one it named was removed under it. -const rereadTries = 5 - // Reread reads the newest trunk revision of the bucket and holds its view; a bucket keeping none holds an empty view at 0. func (k *KeptTrunk) Reread(ctx context.Context, o Objects) error { k.read.Lock() @@ -540,24 +558,7 @@ func (k *KeptTrunk) reread(ctx context.Context, o Objects) error { } return err } - // The pin judge removes a trunk revision, its list and the entries only it names, as soon as a newer one is kept, and - // revisions are kept seconds apart while take-ins run: the revision named newest can be gone before its entries are read. - // A newer one exists then, so the read names the newest again, a few times, before it fails. - var ( - newest meshcatalog.CatalogRevision - any bool - entries []meshcatalog.CatalogEntry - err error - ) - for try := 0; try < rereadTries; try++ { - entries = nil - if newest, any, err = newestTrunk(ctx, o); err == nil && any { - _, entries, err = ReadRevision(ctx, o, RevisionObject(newest)) - } - if !errors.Is(err, ErrNoObject) { - break - } - } + newest, entries, any, err := readNewestOf(ctx, o, TrunkBranch) if err != nil { return fail(err) } diff --git a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go index e3c6b01c..badac4e4 100644 --- a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go +++ b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go @@ -187,3 +187,45 @@ func TestTheKeptTrunkReadsTheNewerRevisionWhenTheOneNamedIsRemovedUnderIt(t *tes t.Errorf("the view holds revision %d (%v, %v), want 2", rev.Revision, ok, err) } } + +// A-487d: the same revision failing twice is the read's failure, said as it is; a read by number never reads another. +func TestTheKeptTrunkFailsWhenTheSameRevisionIsGoneTwice(t *testing.T) { + mem := newMemObjects(0) + data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built"}) + rev := meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: "d1"}}} + if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{"d1": data}); err != nil { + t.Fatal(err) + } + _ = mem.Delete(context.Background(), EntryObject("d1")) // revision 1 stays listed, its entry lost, no newer revision + for name, read := range map[string]func() error{ + "kept trunk": func() error { return (&KeptTrunk{}).Reread(context.Background(), mem) }, + "TrunkView": func() error { _, _, _, err := TrunkView(context.Background(), mem); return err }, + } { + if err := read(); !errors.Is(err, ErrNoObject) { + t.Errorf("%s: %v, want the lost entry said (no such object)", name, err) + } + } +} + +// A-487d: BranchView and TrunkView name the newest again when the one named is removed under the read. +func TestTrunkViewReadsTheNewerRevisionWhenTheOneNamedIsRemovedUnderIt(t *testing.T) { + mem := newMemObjects(0) + keep := func(revision int64) { + data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built"}) + digest := fmt.Sprintf("d%d", revision) + rev := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}} + if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{digest: data}); err != nil { + t.Fatal(err) + } + } + keep(1) + objects := &goneUnderRead{memObjects: mem} + objects.prune = func() { + keep(2) + _ = mem.Delete(context.Background(), RevisionObject(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"})) + _ = mem.Delete(context.Background(), EntryObject("d1")) + } + if _, _, at, err := TrunkView(context.Background(), objects); err != nil || at != 2 { + t.Errorf("TrunkView read revision %d (%v), want 2", at, err) + } +} -- 2.54.0