The indexer says filling, full and room of its catalog graph revisions and holds a merge while they are full (milestone 47/G W4.6)
Design 47, Two graphs: a keep that would pass the store's cap refuses the next revision and says full, naming the source revision held pending with the reason objects full (Pending.Held, which indexer_pending shows); a keep crossing 80 % says filling; once something removes revisions, RoomAgain says room below the filling mark and evaluates pending again (no timer). Each event is said once by its duplicate id. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ
This commit is contained in:
@@ -447,6 +447,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
SeatDefinitions: awaitedStore{b: seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats,
|
||||
Message: messageVia(calls.CallKept), Revisions: revisions, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1)
|
||||
SayCatalogRevision: sayCatalogRevision(conn, cred),
|
||||
SayFill: sayFill(conn, cred),
|
||||
// The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear;
|
||||
// until then every merge is taken in at once, as before.
|
||||
Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.PendingBucket, "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)},
|
||||
|
||||
@@ -144,3 +144,23 @@ func sayCatalogRevision(conn *nats.Conn, cred process.Account) func(context.Cont
|
||||
return conn.PublishMsg(msg)
|
||||
}
|
||||
}
|
||||
|
||||
// sayFill publishes the seat mesh-catalog's event filling, full or room of the catalog graph revisions under the duplicate
|
||||
// id given (milestone 47/G W4.6), so one said twice is heard once.
|
||||
func sayFill(conn *nats.Conn, cred process.Account) func(context.Context, string, string, meshcatalog.Fill) error {
|
||||
return func(_ context.Context, event, id string, f meshcatalog.Fill) error {
|
||||
data, err := json.Marshal(f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg(seatEventSubject(event))
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", cred.Module)
|
||||
msg.Header.Set("x-node", cred.Node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Header.Set(nats.MsgIdHdr, id)
|
||||
msg.Data = data
|
||||
return conn.PublishMsg(msg)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -102,6 +102,9 @@ type Indexer struct {
|
||||
Revisions Objects
|
||||
// SayCatalogRevision says the seat mesh-catalog's event catalog-revision for a revision kept; nil says nothing.
|
||||
SayCatalogRevision func(ctx context.Context, e meshcatalog.CatalogRevisionEvent) error
|
||||
// SayFill says the seat mesh-catalog's event filling, full or room of the catalog graph revisions, under the duplicate
|
||||
// id given (W4.6).
|
||||
SayFill func(ctx context.Context, event, id string, f meshcatalog.Fill) error
|
||||
// Graph is the module graph's state (GraphBucket, graph.go; novox/hq ADR 0603 §2); nil writes no graph.
|
||||
Graph Store
|
||||
// Files asks the forge for every blob under each directory at a commit, by directory then path relative to it, each
|
||||
|
||||
@@ -268,10 +268,22 @@ func (ix *Indexer) keepEntries(ctx context.Context, entries map[string]meshcatal
|
||||
if (any && len(changed) == 0) || (!any && len(entries) == 0) {
|
||||
return nil // nothing moved; an empty first revision is never kept, so the first start's catch-up still runs
|
||||
}
|
||||
used, capacity, fillErr := ix.Revisions.Fill(ctx)
|
||||
if fillErr == nil && capacity > 0 && used+newBytes(ctx, ix.Revisions, rev, objects) > capacity {
|
||||
// W4.6 (design 47, Two graphs): a full store refuses the next revision, said once per source revision, and the
|
||||
// take-in that made it holds its pending merge with the reason objects full until room is said.
|
||||
refused := fmt.Sprintf("the next revision; source revision %d held pending with the reason %s", src, HeldObjectsFull)
|
||||
ix.logf("catalog graph: revision %d of source revision %d is not kept: %s is full (%d of %d bytes)", rev.Revision, src, CatalogRevisionsObjects, used, capacity)
|
||||
ix.sayFill(ctx, meshcatalog.EventFull, fmt.Sprintf("%d", src), meshcatalog.Fill{Bytes: used, Cap: capacity, Refused: refused})
|
||||
return fmt.Errorf("revision %d of source revision %d: %w", rev.Revision, src, ErrObjectsFull)
|
||||
}
|
||||
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
|
||||
ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err)
|
||||
return err
|
||||
}
|
||||
if after, capacity, err := ix.Revisions.Fill(ctx); fillErr == nil && err == nil && capacity > 0 && used*5 < capacity*4 && after*5 >= capacity*4 {
|
||||
ix.sayFill(ctx, meshcatalog.EventFilling, fmt.Sprintf("%d", rev.Revision), meshcatalog.Fill{Bytes: after, Cap: capacity})
|
||||
}
|
||||
ix.logf("catalog graph: revision %d of source revision %d kept, %d entries, %d changed", rev.Revision, src, len(rev.Entries), len(changed))
|
||||
if ix.SayCatalogRevision != nil {
|
||||
if err := ix.SayCatalogRevision(ctx, meshcatalog.CatalogRevisionEvent{V: CatalogEntryLayout, Revision: rev.Revision, Kind: rev.Kind,
|
||||
@@ -282,6 +294,60 @@ func (ix *Indexer) keepEntries(ctx context.Context, entries map[string]meshcatal
|
||||
return nil
|
||||
}
|
||||
|
||||
// HeldObjectsFull is the reason a pending merge is held while the catalog graph revisions are full (Pending.Held).
|
||||
const HeldObjectsFull = "objects full"
|
||||
|
||||
// ErrObjectsFull is a revision refused because the catalog graph revisions are at their cap.
|
||||
var ErrObjectsFull = errors.New("the catalog graph revisions are full")
|
||||
|
||||
// newBytes is the size of what keeping rev adds: its list and each entry the bucket does not hold yet.
|
||||
func newBytes(ctx context.Context, o Objects, rev meshcatalog.CatalogRevision, entries map[string][]byte) int64 {
|
||||
held, _ := o.Names(ctx)
|
||||
have := make(map[string]bool, len(held))
|
||||
for _, n := range held {
|
||||
have[n] = true
|
||||
}
|
||||
list, _ := json.Marshal(rev)
|
||||
size := int64(len(list))
|
||||
for _, ref := range rev.Entries {
|
||||
if !have[EntryObject(ref.Digest)] {
|
||||
size += int64(len(entries[ref.Digest]))
|
||||
}
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
// sayFill says the seat's event filling, full or room of the catalog graph revisions, under the duplicate id of the
|
||||
// event and what it is about, so the same one said twice is heard once.
|
||||
func (ix *Indexer) sayFill(ctx context.Context, event, about string, f meshcatalog.Fill) {
|
||||
if ix.SayFill == nil {
|
||||
return
|
||||
}
|
||||
at := ix.now().UTC()
|
||||
f.At, f.Store = &at, CatalogRevisionsObjects
|
||||
if err := ix.SayFill(ctx, event, event+"-"+about, f); err != nil {
|
||||
ix.logf("catalog graph: %s is not said: %v", event, err)
|
||||
}
|
||||
}
|
||||
|
||||
// RoomAgain says room and evaluates pending again when a merge held with the reason objects full finds the store
|
||||
// below its filling mark (80 % of its cap): called after anything removes catalog graph revisions (no timer). It answers whether room was said.
|
||||
func (ix *Indexer) RoomAgain(ctx context.Context) (bool, error) {
|
||||
if ix.Revisions == nil || ix.Pending == nil {
|
||||
return false, nil
|
||||
}
|
||||
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
|
||||
if err != nil || !slices.ContainsFunc(entries, func(e PendingEntry) bool { return e.Value.Held == HeldObjectsFull }) {
|
||||
return false, err
|
||||
}
|
||||
used, capacity, err := ix.Revisions.Fill(ctx)
|
||||
if err != nil || (capacity > 0 && used*5 >= capacity*4) {
|
||||
return false, err // room is said below the filling mark, 80 %, so one refusal is not followed by another at once
|
||||
}
|
||||
ix.sayFill(ctx, meshcatalog.EventRoom, fmt.Sprintf("%d", used), meshcatalog.Fill{Bytes: used, Cap: capacity})
|
||||
return true, ix.PendingAgain(ctx) // room in catalog-revisions: pending evaluated again
|
||||
}
|
||||
|
||||
// TrunkView is the indexer's view of the trunk read from the newest trunk revision the bucket keeps (the lead's ruling
|
||||
// on milestone 47/G finding 1; ADR 0656 §6, one form of the fact): a store in memory holding each module's value as
|
||||
// ModuleValueOf reads it back from its entry, under its modules key, a store in memory of each module's node of the module
|
||||
|
||||
@@ -95,6 +95,9 @@ type Pending struct {
|
||||
// UnreadSince is when the message was first found unread: past meshconditionsclient.CallUnansweredAfter, with the forge
|
||||
// heard since, it is the finding call-unanswered (unreadcall.go, ADR 0668 §3).
|
||||
UnreadSince time.Time `json:"unread-since,omitzero"`
|
||||
// Held is why a ready merge is not taken in although its revision names this build: objects full (HeldObjectsFull),
|
||||
// until room is said (RoomAgain).
|
||||
Held string `json:"held,omitempty"`
|
||||
}
|
||||
|
||||
// PendingEntry is one key of the pending state as read: its key, revision, value and the merged message decoded. For a
|
||||
@@ -782,11 +785,32 @@ func (ix *Indexer) evaluate(ctx context.Context, own string, start bool) (Applie
|
||||
if errors.Is(err, errVerdictsNotYet) {
|
||||
return a, "the verdicts state is not on the bus yet: the revision waits for it, taken in when it binds", nil
|
||||
}
|
||||
if errors.Is(err, ErrObjectsFull) {
|
||||
ix.holdFull(ctx, ready)
|
||||
return a, ix.waitingLine(w) + "; held: " + HeldObjectsFull, nil
|
||||
}
|
||||
return a, ix.waitingLine(w), err
|
||||
}
|
||||
return Applied{}, ix.waitingLine(w), ix.holdFor(ctx, ready, names, own, dry)
|
||||
}
|
||||
|
||||
// holdFull marks the ready merges held with the reason objects full (Pending.Held), read again at their revisions now
|
||||
// (the take-in's claim moved them), so indexer_pending shows why they wait.
|
||||
func (ix *Indexer) holdFull(ctx context.Context, ready []PendingEntry) {
|
||||
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
|
||||
if err != nil {
|
||||
ix.logf("take-in: the merges held by a full %s are not marked: %v", CatalogRevisionsObjects, err)
|
||||
return
|
||||
}
|
||||
var held []PendingEntry
|
||||
for _, e := range entries {
|
||||
if slices.ContainsFunc(ready, func(r PendingEntry) bool { return r.Key == e.Key }) && e.Value.Held != HeldObjectsFull {
|
||||
held = append(held, e)
|
||||
}
|
||||
}
|
||||
ix.markSaid(ctx, held, nil, func(p *Pending) { p.Held = HeldObjectsFull })
|
||||
}
|
||||
|
||||
// holdLost holds a revision whose newest merge of a repository cannot be read (the forge bridge's stream no longer holds
|
||||
// it, or the bus refuses the read): said once on take-in.pending with the lost keys (TakeIn.Lost), and on each lost entry
|
||||
// (Pending.Lost, the reason, which indexer_pending shows), until a newer merge of its repository covers it or
|
||||
@@ -886,7 +910,7 @@ func (ix *Indexer) dry() *dryRun {
|
||||
Requirements: nil, Findings: nil, SeatDefinitions: nil, ReadFile: ix.ReadFile, SaySeatDefinition: nil,
|
||||
Holds: nil, Counted: nil, AskEngines: nil, HoldBounds: nil, OwnBuildState: nil, underSeats: true,
|
||||
// A dry run keeps no catalog graph revision (catalog_revisions.go): its revision is the take-in's to keep.
|
||||
Revisions: nil, SayCatalogRevision: nil}
|
||||
Revisions: nil, SayCatalogRevision: nil, SayFill: nil}
|
||||
x.Logf = func(format string, args ...any) { ix.logf("take-in, computed and not written: "+format, args...) }
|
||||
x.Store, d.candidates = over(ix.Store)
|
||||
x.Modules, d.modules = over(ix.Modules)
|
||||
|
||||
@@ -744,6 +744,56 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("W4.6 a full store holds the merge with the reason objects full, says full once, and room takes it in", func(t *testing.T) {
|
||||
b := newTakeInBed(t)
|
||||
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
named := b.fp(t, Module)
|
||||
b.ix.ownMu.Lock()
|
||||
b.ix.ownFP = named
|
||||
b.ix.ownMu.Unlock()
|
||||
store := newMemObjects(100) // 90 bytes held by another revision: full for this one
|
||||
_ = store.Put(ctx, "trunk.0", make([]byte, 90))
|
||||
b.ix.Revisions = store
|
||||
var said []string
|
||||
b.ix.SayFill = func(_ context.Context, event, id string, f meshcatalog.Fill) error {
|
||||
said = append(said, event+" "+f.Refused)
|
||||
return nil
|
||||
}
|
||||
if _, _, err := b.ix.TakeInAtStart(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var p Pending
|
||||
_ = json.Unmarshal(b.pending.vals[b.keys()[0]], &p)
|
||||
if len(b.keys()) != 1 || p.Held != HeldObjectsFull || len(said) != 1 || !strings.HasPrefix(said[0], "full ") || !strings.Contains(said[0], HeldObjectsFull) {
|
||||
t.Fatalf("pending %v held %q, said %q: want the merge held with %q and full said", b.keys(), p.Held, said, HeldObjectsFull)
|
||||
}
|
||||
if roomed, err := b.ix.RoomAgain(ctx); err != nil || roomed {
|
||||
t.Fatalf("room while still full: %v %v", roomed, err)
|
||||
}
|
||||
_ = store.Delete(ctx, "trunk.0") // the other revision removed: room
|
||||
store.capacity = 0
|
||||
if roomed, err := b.ix.RoomAgain(ctx); err != nil || !roomed || len(b.keys()) != 0 || said[len(said)-1] != "room " {
|
||||
t.Errorf("room said %v (%v), said %q, pending %v: want room said and the merge taken in", roomed, err, said, b.keys())
|
||||
}
|
||||
// The same take-in into a store it fills to 85 % of its cap says filling once, and is taken in.
|
||||
kept := store.used()
|
||||
b2 := newTakeInBed(t)
|
||||
b2.arrive(t, 2, b2.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
b2.ix.ownMu.Lock()
|
||||
b2.ix.ownFP = b2.fp(t, Module)
|
||||
b2.ix.ownMu.Unlock()
|
||||
b2.ix.Revisions = newMemObjects(kept * 100 / 85)
|
||||
var filling []meshcatalog.Fill
|
||||
b2.ix.SayFill = func(_ context.Context, event, _ string, f meshcatalog.Fill) error {
|
||||
if event == meshcatalog.EventFilling {
|
||||
filling = append(filling, f)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if _, _, err := b2.ix.TakeInAtStart(ctx); err != nil || len(b2.keys()) != 0 || len(filling) != 1 || filling[0].Store != CatalogRevisionsObjects {
|
||||
t.Errorf("a take-in to 85 %% said filling %+v (%v), pending %v: want it said once and the merge taken in", filling, err, b2.keys())
|
||||
}
|
||||
})
|
||||
t.Run("T7 a toolchain report evaluates pending again: the revision then names the running build and is taken in", func(t *testing.T) {
|
||||
b := newTakeInBed(t)
|
||||
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
@@ -970,7 +1020,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", "Unjudged",
|
||||
"JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk",
|
||||
"Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn",
|
||||
"Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "Holds", "Counted", "AskEngines", "HoldBounds", "OpenTries", "TryRecovery", "OwnBuildState", "Revisions", "SayCatalogRevision"} {
|
||||
"Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "Holds", "Counted", "AskEngines", "HoldBounds", "OpenTries", "TryRecovery", "OwnBuildState", "Revisions", "SayCatalogRevision", "SayFill"} {
|
||||
named[f] = true
|
||||
}
|
||||
ty := reflect.TypeOf(Indexer{})
|
||||
|
||||
Reference in New Issue
Block a user