From b2b449c913dc21b3083287f0b719411af57344d4 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 9 Nov 2026 21:21:10 +0100 Subject: [PATCH] The indexer says filling, full and room of its catalog graph revisions and holds a merge while they are full (milestone 47/G W4.6) Design 47, Two graphs: a keep that would pass the store's cap refuses the next revision and says full, naming the source revision held pending with the reason objects full (Pending.Held, which indexer_pending shows); a keep crossing 80 % says filling; once something removes revisions, RoomAgain says room below the filling mark and evaluates pending again (no timer). Each event is said once by its duplicate id. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-indexer/main.go | 1 + .../cmd/mesh-delivery-indexer/objects.go | 20 ++++++ .../internal/index/candidates.go | 3 + .../internal/index/catalog_revisions.go | 66 +++++++++++++++++++ .../internal/index/pending.go | 26 +++++++- .../internal/index/pending_test.go | 52 ++++++++++++++- 6 files changed, 166 insertions(+), 2 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 9faf2d03..24d386d0 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -447,6 +447,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro SeatDefinitions: awaitedStore{b: seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats, Message: messageVia(calls.CallKept), Revisions: revisions, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1) SayCatalogRevision: sayCatalogRevision(conn, cred), + SayFill: sayFill(conn, cred), // The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear; // until then every merge is taken in at once, as before. Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.PendingBucket, "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)}, diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go index 025301db..7b250e47 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go @@ -144,3 +144,23 @@ func sayCatalogRevision(conn *nats.Conn, cred process.Account) func(context.Cont return conn.PublishMsg(msg) } } + +// sayFill publishes the seat mesh-catalog's event filling, full or room of the catalog graph revisions under the duplicate +// id given (milestone 47/G W4.6), so one said twice is heard once. +func sayFill(conn *nats.Conn, cred process.Account) func(context.Context, string, string, meshcatalog.Fill) error { + return func(_ context.Context, event, id string, f meshcatalog.Fill) error { + data, err := json.Marshal(f) + if err != nil { + return err + } + msg := nats.NewMsg(seatEventSubject(event)) + msg.Header.Set("x-event-id", id) + msg.Header.Set("x-source", cred.Module) + msg.Header.Set("x-node", cred.Node) + msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano)) + msg.Header.Set("content-type", "application/json") + msg.Header.Set(nats.MsgIdHdr, id) + msg.Data = data + return conn.PublishMsg(msg) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 5d9888dd..5fcfa42b 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -102,6 +102,9 @@ type Indexer struct { Revisions Objects // SayCatalogRevision says the seat mesh-catalog's event catalog-revision for a revision kept; nil says nothing. SayCatalogRevision func(ctx context.Context, e meshcatalog.CatalogRevisionEvent) error + // SayFill says the seat mesh-catalog's event filling, full or room of the catalog graph revisions, under the duplicate + // id given (W4.6). + SayFill func(ctx context.Context, event, id string, f meshcatalog.Fill) error // Graph is the module graph's state (GraphBucket, graph.go; novox/hq ADR 0603 §2); nil writes no graph. Graph Store // Files asks the forge for every blob under each directory at a commit, by directory then path relative to it, each diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index 408b5951..b044b4bc 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -268,10 +268,22 @@ func (ix *Indexer) keepEntries(ctx context.Context, entries map[string]meshcatal if (any && len(changed) == 0) || (!any && len(entries) == 0) { return nil // nothing moved; an empty first revision is never kept, so the first start's catch-up still runs } + used, capacity, fillErr := ix.Revisions.Fill(ctx) + if fillErr == nil && capacity > 0 && used+newBytes(ctx, ix.Revisions, rev, objects) > capacity { + // W4.6 (design 47, Two graphs): a full store refuses the next revision, said once per source revision, and the + // take-in that made it holds its pending merge with the reason objects full until room is said. + refused := fmt.Sprintf("the next revision; source revision %d held pending with the reason %s", src, HeldObjectsFull) + ix.logf("catalog graph: revision %d of source revision %d is not kept: %s is full (%d of %d bytes)", rev.Revision, src, CatalogRevisionsObjects, used, capacity) + ix.sayFill(ctx, meshcatalog.EventFull, fmt.Sprintf("%d", src), meshcatalog.Fill{Bytes: used, Cap: capacity, Refused: refused}) + return fmt.Errorf("revision %d of source revision %d: %w", rev.Revision, src, ErrObjectsFull) + } if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err) return err } + if after, capacity, err := ix.Revisions.Fill(ctx); fillErr == nil && err == nil && capacity > 0 && used*5 < capacity*4 && after*5 >= capacity*4 { + ix.sayFill(ctx, meshcatalog.EventFilling, fmt.Sprintf("%d", rev.Revision), meshcatalog.Fill{Bytes: after, Cap: capacity}) + } ix.logf("catalog graph: revision %d of source revision %d kept, %d entries, %d changed", rev.Revision, src, len(rev.Entries), len(changed)) if ix.SayCatalogRevision != nil { if err := ix.SayCatalogRevision(ctx, meshcatalog.CatalogRevisionEvent{V: CatalogEntryLayout, Revision: rev.Revision, Kind: rev.Kind, @@ -282,6 +294,60 @@ func (ix *Indexer) keepEntries(ctx context.Context, entries map[string]meshcatal return nil } +// HeldObjectsFull is the reason a pending merge is held while the catalog graph revisions are full (Pending.Held). +const HeldObjectsFull = "objects full" + +// ErrObjectsFull is a revision refused because the catalog graph revisions are at their cap. +var ErrObjectsFull = errors.New("the catalog graph revisions are full") + +// newBytes is the size of what keeping rev adds: its list and each entry the bucket does not hold yet. +func newBytes(ctx context.Context, o Objects, rev meshcatalog.CatalogRevision, entries map[string][]byte) int64 { + held, _ := o.Names(ctx) + have := make(map[string]bool, len(held)) + for _, n := range held { + have[n] = true + } + list, _ := json.Marshal(rev) + size := int64(len(list)) + for _, ref := range rev.Entries { + if !have[EntryObject(ref.Digest)] { + size += int64(len(entries[ref.Digest])) + } + } + return size +} + +// sayFill says the seat's event filling, full or room of the catalog graph revisions, under the duplicate id of the +// event and what it is about, so the same one said twice is heard once. +func (ix *Indexer) sayFill(ctx context.Context, event, about string, f meshcatalog.Fill) { + if ix.SayFill == nil { + return + } + at := ix.now().UTC() + f.At, f.Store = &at, CatalogRevisionsObjects + if err := ix.SayFill(ctx, event, event+"-"+about, f); err != nil { + ix.logf("catalog graph: %s is not said: %v", event, err) + } +} + +// RoomAgain says room and evaluates pending again when a merge held with the reason objects full finds the store +// below its filling mark (80 % of its cap): called after anything removes catalog graph revisions (no timer). It answers whether room was said. +func (ix *Indexer) RoomAgain(ctx context.Context) (bool, error) { + if ix.Revisions == nil || ix.Pending == nil { + return false, nil + } + entries, err := ReadPending(ctx, ix.Pending, ix.logf) + if err != nil || !slices.ContainsFunc(entries, func(e PendingEntry) bool { return e.Value.Held == HeldObjectsFull }) { + return false, err + } + used, capacity, err := ix.Revisions.Fill(ctx) + if err != nil || (capacity > 0 && used*5 >= capacity*4) { + return false, err // room is said below the filling mark, 80 %, so one refusal is not followed by another at once + } + ix.sayFill(ctx, meshcatalog.EventRoom, fmt.Sprintf("%d", used), meshcatalog.Fill{Bytes: used, Cap: capacity}) + return true, ix.PendingAgain(ctx) // room in catalog-revisions: pending evaluated again +} + // TrunkView is the indexer's view of the trunk read from the newest trunk revision the bucket keeps (the lead's ruling // on milestone 47/G finding 1; ADR 0656 §6, one form of the fact): a store in memory holding each module's value as // ModuleValueOf reads it back from its entry, under its modules key, a store in memory of each module's node of the module diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index c732188f..17b32ef0 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -95,6 +95,9 @@ type Pending struct { // UnreadSince is when the message was first found unread: past meshconditionsclient.CallUnansweredAfter, with the forge // heard since, it is the finding call-unanswered (unreadcall.go, ADR 0668 §3). UnreadSince time.Time `json:"unread-since,omitzero"` + // Held is why a ready merge is not taken in although its revision names this build: objects full (HeldObjectsFull), + // until room is said (RoomAgain). + Held string `json:"held,omitempty"` } // PendingEntry is one key of the pending state as read: its key, revision, value and the merged message decoded. For a @@ -782,11 +785,32 @@ func (ix *Indexer) evaluate(ctx context.Context, own string, start bool) (Applie if errors.Is(err, errVerdictsNotYet) { return a, "the verdicts state is not on the bus yet: the revision waits for it, taken in when it binds", nil } + if errors.Is(err, ErrObjectsFull) { + ix.holdFull(ctx, ready) + return a, ix.waitingLine(w) + "; held: " + HeldObjectsFull, nil + } return a, ix.waitingLine(w), err } return Applied{}, ix.waitingLine(w), ix.holdFor(ctx, ready, names, own, dry) } +// holdFull marks the ready merges held with the reason objects full (Pending.Held), read again at their revisions now +// (the take-in's claim moved them), so indexer_pending shows why they wait. +func (ix *Indexer) holdFull(ctx context.Context, ready []PendingEntry) { + entries, err := ReadPending(ctx, ix.Pending, ix.logf) + if err != nil { + ix.logf("take-in: the merges held by a full %s are not marked: %v", CatalogRevisionsObjects, err) + return + } + var held []PendingEntry + for _, e := range entries { + if slices.ContainsFunc(ready, func(r PendingEntry) bool { return r.Key == e.Key }) && e.Value.Held != HeldObjectsFull { + held = append(held, e) + } + } + ix.markSaid(ctx, held, nil, func(p *Pending) { p.Held = HeldObjectsFull }) +} + // holdLost holds a revision whose newest merge of a repository cannot be read (the forge bridge's stream no longer holds // it, or the bus refuses the read): said once on take-in.pending with the lost keys (TakeIn.Lost), and on each lost entry // (Pending.Lost, the reason, which indexer_pending shows), until a newer merge of its repository covers it or @@ -886,7 +910,7 @@ func (ix *Indexer) dry() *dryRun { Requirements: nil, Findings: nil, SeatDefinitions: nil, ReadFile: ix.ReadFile, SaySeatDefinition: nil, Holds: nil, Counted: nil, AskEngines: nil, HoldBounds: nil, OwnBuildState: nil, underSeats: true, // A dry run keeps no catalog graph revision (catalog_revisions.go): its revision is the take-in's to keep. - Revisions: nil, SayCatalogRevision: nil} + Revisions: nil, SayCatalogRevision: nil, SayFill: nil} x.Logf = func(format string, args ...any) { ix.logf("take-in, computed and not written: "+format, args...) } x.Store, d.candidates = over(ix.Store) x.Modules, d.modules = over(ix.Modules) diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index 6a82b105..8487b9fd 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -744,6 +744,56 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) } } }) + t.Run("W4.6 a full store holds the merge with the reason objects full, says full once, and room takes it in", func(t *testing.T) { + b := newTakeInBed(t) + b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + named := b.fp(t, Module) + b.ix.ownMu.Lock() + b.ix.ownFP = named + b.ix.ownMu.Unlock() + store := newMemObjects(100) // 90 bytes held by another revision: full for this one + _ = store.Put(ctx, "trunk.0", make([]byte, 90)) + b.ix.Revisions = store + var said []string + b.ix.SayFill = func(_ context.Context, event, id string, f meshcatalog.Fill) error { + said = append(said, event+" "+f.Refused) + return nil + } + if _, _, err := b.ix.TakeInAtStart(ctx); err != nil { + t.Fatal(err) + } + var p Pending + _ = json.Unmarshal(b.pending.vals[b.keys()[0]], &p) + if len(b.keys()) != 1 || p.Held != HeldObjectsFull || len(said) != 1 || !strings.HasPrefix(said[0], "full ") || !strings.Contains(said[0], HeldObjectsFull) { + t.Fatalf("pending %v held %q, said %q: want the merge held with %q and full said", b.keys(), p.Held, said, HeldObjectsFull) + } + if roomed, err := b.ix.RoomAgain(ctx); err != nil || roomed { + t.Fatalf("room while still full: %v %v", roomed, err) + } + _ = store.Delete(ctx, "trunk.0") // the other revision removed: room + store.capacity = 0 + if roomed, err := b.ix.RoomAgain(ctx); err != nil || !roomed || len(b.keys()) != 0 || said[len(said)-1] != "room " { + t.Errorf("room said %v (%v), said %q, pending %v: want room said and the merge taken in", roomed, err, said, b.keys()) + } + // The same take-in into a store it fills to 85 % of its cap says filling once, and is taken in. + kept := store.used() + b2 := newTakeInBed(t) + b2.arrive(t, 2, b2.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + b2.ix.ownMu.Lock() + b2.ix.ownFP = b2.fp(t, Module) + b2.ix.ownMu.Unlock() + b2.ix.Revisions = newMemObjects(kept * 100 / 85) + var filling []meshcatalog.Fill + b2.ix.SayFill = func(_ context.Context, event, _ string, f meshcatalog.Fill) error { + if event == meshcatalog.EventFilling { + filling = append(filling, f) + } + return nil + } + if _, _, err := b2.ix.TakeInAtStart(ctx); err != nil || len(b2.keys()) != 0 || len(filling) != 1 || filling[0].Store != CatalogRevisionsObjects { + t.Errorf("a take-in to 85 %% said filling %+v (%v), pending %v: want it said once and the merge taken in", filling, err, b2.keys()) + } + }) t.Run("T7 a toolchain report evaluates pending again: the revision then names the running build and is taken in", func(t *testing.T) { b := newTakeInBed(t) b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) @@ -970,7 +1020,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", "Unjudged", "JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk", "Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", - "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "Holds", "Counted", "AskEngines", "HoldBounds", "OpenTries", "TryRecovery", "OwnBuildState", "Revisions", "SayCatalogRevision"} { + "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "Holds", "Counted", "AskEngines", "HoldBounds", "OpenTries", "TryRecovery", "OwnBuildState", "Revisions", "SayCatalogRevision", "SayFill"} { named[f] = true } ty := reflect.TypeOf(Indexer{})