From 04fe2e30feb9f8a946304241b9444207d3f9fd04 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 21:35:09 +0100 Subject: [PATCH] H-A-497 indexer: a hand ask of a held branch answers at once indexer_take_in_pending ran the whole chain of held take-ins inside the one loop that acts on the takeins asks, minutes long, so its answer was cut and a second ask waited behind it and answered "holds no source-moved". The ask now starts the chain beside the loop and answers at once, "chain started at source , wait"; an ask that meets a running chain answers its state. The chain's state (branch, source being taken in, since, how many wait) is the key chain. of the takeins state, written at each step and deleted at the chain's end, skipped by the asks' loop and shown by indexer_pending. A key a stop left is deleted at the indexer's start. Builds: design 47 H2 H-A-497 (corrected), its addendum and its second addendum --- .../cmd/mesh-delivery-indexer/main.go | 3 +- .../cmd/mesh-delivery-indexer/sourcemoved.go | 30 ++-- .../mesh-delivery-indexer/sourcemoved_test.go | 92 ++++++++++++ .../cmd/mesh-delivery-indexer/takein.go | 4 +- .../internal/index/held.go | 134 +++++++++++++++++- .../internal/index/takein_source_test.go | 25 +++- 6 files changed, 271 insertions(+), 17 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index e48e3cbc6..801db723f 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -664,7 +664,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro sources.refresh() }) // The held take-ins are taken in again from the state held at the start, before a branch's consumer gives a later one - // (design 47 H, H-A-114, H-A-118). + // (design 47 H, H-A-114, H-A-118); a chain's key a stop left is deleted first (H2 H-A-497 addendum). + ix.EndChains(ctx) sources.takeAgain() sources.refresh() if stopped != nil { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go index 4a42bcdde..35a99fb30 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go @@ -265,24 +265,34 @@ type actor struct { s *sourceMoved } -// HeldAgain takes a branch's held source revisions in again now (indexer_take_in_pending). +// HeldAgain takes a branch's held source revisions in again now (indexer_take_in_pending), beside the asks' loop, and answers +// at once (design 47 H2 H-A-497 and its addendum): a chain that runs already answers its state from the state takeins; held +// sources start the chain at the first, written there before it runs, answered with how many wait. func (a actor) HeldAgain(ctx context.Context, branch string) index.TakenIn { r := index.TakenIn{Module: branch} - if hs, err := a.HeldOf(ctx, branch); err != nil || len(hs) == 0 { + if c, running, err := a.ChainOf(ctx, branch); err != nil { + r.Answer = "the state takeins is not read: " + err.Error() + return r + } else if running { + r.Held = true + r.Answer = fmt.Sprintf("a chain runs: source %d taken in since %s, %d wait", c.Source, c.Since.Format(time.RFC3339), c.Waiting) + return r + } + hs, err := a.HeldOf(ctx, branch) + if err != nil || len(hs) == 0 { r.Answer = "the branch " + branch + " holds no source-moved" if err != nil { r.Answer = "the state held is not read: " + err.Error() } return r } - a.s.takeAgain() - hs, err := a.HeldOf(ctx, branch) - still := err != nil || len(hs) > 0 - r.Stored, r.Held = !still, still - r.Answer = "taken in" - if still { - r.Answer = "still held: indexer_pending says why" - } + c := a.StartChain(ctx, hs) + go func() { + a.s.takeAgain() + a.EndChain(a.s.ctx, branch) + }() + r.Held = true + r.Answer = fmt.Sprintf("chain started at source %d, %d wait", c.Source, c.Waiting) return r } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go index 54707cdaf..9723494fa 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go @@ -97,6 +97,98 @@ func TestAHeldSourceMovedIsSkippedByHand(t *testing.T) { } } +// chainStore is a takeins state in memory, safe beside the chain's own goroutine, which says on ended when a chain key goes. +type chainStore struct { + mu sync.Mutex + vals map[string][]byte + ended chan string +} + +func (c *chainStore) Get(_ context.Context, k string) ([]byte, uint64, bool, error) { + c.mu.Lock() + defer c.mu.Unlock() + v, ok := c.vals[k] + return v, 1, ok, nil +} +func (c *chainStore) Keys(context.Context) ([]string, error) { + c.mu.Lock() + defer c.mu.Unlock() + var out []string + for k := range c.vals { + out = append(out, k) + } + return out, nil +} +func (c *chainStore) Create(_ context.Context, k string, v []byte) error { + c.mu.Lock() + defer c.mu.Unlock() + c.vals[k] = v + return nil +} +func (c *chainStore) Update(ctx context.Context, k string, v []byte, _ uint64) error { + return c.Create(ctx, k, v) +} +func (c *chainStore) Delete(_ context.Context, k string, _ uint64) error { + c.mu.Lock() + _, had := c.vals[k] + delete(c.vals, k) + c.mu.Unlock() + if had && strings.HasPrefix(k, index.ChainPrefix) { + c.ended <- k + } + return nil +} + +// Design 47 H2 H-A-497 and its addendum: a hand ask of a branch with held sources starts the chain beside the asks' loop +// and answers at once; an ask that meets the running chain answers its state, which the state takeins holds and +// indexer_pending shows; the chain's key goes at its end. +func TestAHandAskOfAHeldBranchAnswersAtOnce(t *testing.T) { + ctx := context.Background() + held := &memStore{vals: map[string][]byte{}} + takeIns := &chainStore{vals: map[string][]byte{}, ended: make(chan string, 4)} + ix := &index.Indexer{Held: held, TakeIns: takeIns, Logf: func(string, ...any) {}, + Now: func() time.Time { return time.Date(2026, 11, 22, 21, 7, 0, 0, time.UTC) }} + handlers := map[string]func(jetstream.Msg){} + s := &sourceMoved{ctx: ctx, ix: ix, + branches: func(context.Context) ([]string, error) { return []string{"main"}, nil }, + bind: func(_ context.Context, branch string, each func(jetstream.Msg)) (func(), error) { + handlers[branch] = each + return func() {}, nil + }, + logf: func(string, ...any) {}} + s.refresh() + for _, n := range []int{114, 115, 116} { + m := &fakeMsg{subject: fmt.Sprintf("mesh.seat.mesh-delivery.event.source-moved.main.%d", n), + data: []byte(fmt.Sprintf(`{"v":1,"branch":"main","source":%d,"call":"%d123456789abcde","heads":[]}`, n, n))} + handlers["main"](msgOf{f: m}) + } + a := actor{ix, s} + s.again.Lock() // the chain waits for the lock, so it is still running at the second ask + if r := a.HeldAgain(ctx, "main"); r.Answer != "chain started at source 114, 2 wait" { + s.again.Unlock() + t.Fatalf("the first ask answered %+v, want the chain started at once", r) + } + r := a.HeldAgain(ctx, "main") + var pending any + var err error + for _, tool := range index.HeldTools(held, takeIns, nil, ix.Now, nil) { + if tool.Name == index.PendingTool { + pending, err = tool.Run(map[string]any{}) + } + } + s.again.Unlock() + chains := pending.(index.PendingAnswer).Chains + if r.Answer != "a chain runs: source 114 taken in since 2026-11-22T21:07:00Z, 2 wait" || !r.Held { + t.Errorf("the ask meeting the chain answered %+v, want its state at once", r) + } + if err != nil || len(chains) != 1 || chains[0].Branch != "main" || chains[0].Source != 114 || chains[0].Waiting != 2 { + t.Errorf("indexer_pending shows chains %+v (%v), want main's at 114 with 2 waiting", chains, err) + } + if k := <-takeIns.ended; k != index.ChainKey("main") { + t.Errorf("the key %s went, want main's chain", k) + } +} + // revisionObjects is the catalog revisions' object bucket in memory. type revisionObjects map[string][]byte diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go index 11a52cd33..b86f544c9 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go @@ -33,8 +33,8 @@ type takerIn interface { // written back, by compare-and-set at its revision, with what the indexer stored or why not. A value // answered already (this indexer's own write) or deleted is left. func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) { - if e.Operation() != jetstream.KeyValuePut || strings.HasPrefix(e.Key(), index.RunningPrefix) { - return // a running take-in's waiting-on is the indexer's own record, no ask (H-A-409) + if e.Operation() != jetstream.KeyValuePut || strings.HasPrefix(e.Key(), index.RunningPrefix) || strings.HasPrefix(e.Key(), index.ChainPrefix) { + return // a running take-in's waiting-on and a running chain are the indexer's own records, no ask (H-A-409, H-A-497) } a, err := index.DecodeTakeIn(e.Value()) if err != nil { diff --git a/modules/mesh-delivery-indexer/internal/index/held.go b/modules/mesh-delivery-indexer/internal/index/held.go index 41f1d971e..dc5c04988 100644 --- a/modules/mesh-delivery-indexer/internal/index/held.go +++ b/modules/mesh-delivery-indexer/internal/index/held.go @@ -51,6 +51,9 @@ type PendingAnswer struct { Held []HeldTakeIn `json:"held"` // Running are the take-ins that run, each with what it waits on (design 47 H2 §803, H-A-409). Running []RunningTakeIn `json:"running,omitempty"` + // Chains are the running chains of held take-ins, each branch's source being taken in, since when and how many wait + // (H-A-497 addendum). + Chains []HeldChain `json:"chains,omitempty"` } // ErrHeldUnwritten says a source-moved that could not be taken in is not written into the state held either: its message @@ -178,22 +181,142 @@ func (ix *Indexer) TakeHeldIn(ctx context.Context) { return } stopped, landed := map[string]bool{}, map[string]int64{} + behind := map[string]int{} for _, h := range all { + behind[h.Branch]++ + } + for i, h := range all { + behind[h.Branch]-- if stopped[h.Branch] { continue } if prev, ok := landed[h.Branch]; ok { ix.logf("take-in: held source %d of %s taken in next, after %d (H-A-497)", h.Source, h.Branch, prev) } + ix.writeChain(ctx, HeldChain{Branch: h.Branch, Source: h.Source, Since: ix.now().UTC(), Waiting: behind[h.Branch]}) if _, err := ix.TakeInSource(ctx, h.Event()); err == nil { landed[h.Branch] = h.Source } else { ix.logf("take-in: source revision %d of %s is still held: %v", h.Source, h.Branch, err) stopped[h.Branch] = true } + // The chain of a branch ends at its first held again or its last held (H-A-497 addendum). + if stopped[h.Branch] || !slices.ContainsFunc(all[i+1:], func(n HeldTakeIn) bool { return n.Branch == h.Branch }) { + ix.endChain(ctx, h.Branch) + } } } +// ChainPrefix is the takeins keys of the chains of held take-ins: `chain.`, the branch escaped as a key's name is, +// holding the branch's chain while it runs (design 47 H2 H-A-497 and its addendum): written at each step, deleted at its end. +const ChainPrefix = "chain." + +// ChainKey is the takeins key of branch's chain. +func ChainKey(branch string) string { return ChainPrefix + keys.EscapeName(branch) } + +// HeldChain is a running chain of a branch's held take-ins (H-A-497 addendum): the source being taken in, since when, and +// how many held sources of the branch wait behind it. +type HeldChain struct { + Branch string `json:"branch"` + Source int64 `json:"source"` + Since time.Time `json:"since"` + Waiting int `json:"waiting"` +} + +// writeChain writes branch's chain at its step c into the state takeins; a failed write is said. +func (ix *Indexer) writeChain(ctx context.Context, c HeldChain) { + if ix.TakeIns == nil { + return + } + data, err := json.Marshal(c) + if err == nil { + err = write(ctx, ix.TakeIns, ChainKey(c.Branch), func([]byte, bool) ([]byte, bool, error) { return data, false, nil }) + } + if err != nil { + ix.logf("take-in: the chain of %s at source revision %d is not written into %s: %v", c.Branch, c.Source, TakeInsState, err) + } +} + +// EndChain deletes branch's chain from the state takeins once the take-in its ask started returned, so a chain whose held +// sources another act took first is never left running. +func (ix *Indexer) EndChain(ctx context.Context, branch string) { ix.endChain(ctx, branch) } + +// EndChains deletes every chain the state takeins holds, at the indexer's start (H-A-497 addendum): no chain runs before the +// start's take-in, so a key left by a stop between a chain's last take-in and its end would answer "a chain runs" for good. +func (ix *Indexer) EndChains(ctx context.Context) { + chains, err := ReadChains(ctx, ix.TakeIns) + if err != nil { + ix.logf("take-in: the chains in %s are not read, so none left by a stop is deleted: %v", TakeInsState, err) + return + } + for _, c := range chains { + ix.logf("take-in: the chain of %s at source revision %d, left by a stop, is deleted at the start", c.Branch, c.Source) + ix.endChain(ctx, c.Branch) + } +} + +// endChain deletes branch's chain from the state takeins. +func (ix *Indexer) endChain(ctx context.Context, branch string) { + if ix.TakeIns == nil { + return + } + if err := write(ctx, ix.TakeIns, ChainKey(branch), func([]byte, bool) ([]byte, bool, error) { return nil, true, nil }); err != nil { + ix.logf("take-in: the chain of %s is not deleted from %s: %v", branch, TakeInsState, err) + } +} + +// StartChain writes branch's chain at its first held source, before the chain is taken in beside the asks (H-A-497 +// addendum), so an ask after it finds it running; it answers that chain. +func (ix *Indexer) StartChain(ctx context.Context, held []HeldTakeIn) HeldChain { + c := HeldChain{Branch: held[0].Branch, Source: held[0].Source, Since: ix.now().UTC(), Waiting: len(held) - 1} + ix.writeChain(ctx, c) + return c +} + +// ChainOf is branch's running chain in the state takeins, and whether one runs. +func (ix *Indexer) ChainOf(ctx context.Context, branch string) (HeldChain, bool, error) { + if ix.TakeIns == nil { + return HeldChain{}, false, nil + } + v, _, found, err := ix.TakeIns.Get(ctx, ChainKey(branch)) + if err != nil || !found { + return HeldChain{}, false, err + } + var c HeldChain + if err := json.Unmarshal(v, &c); err != nil { + return HeldChain{}, false, fmt.Errorf("%s.%s: %w", TakeInsState, ChainKey(branch), err) + } + return c, true, nil +} + +// ReadChains are the running chains the state takeins holds (H-A-497 addendum), each branch's once. +func ReadChains(ctx context.Context, st Store) ([]HeldChain, error) { + if st == nil { + return nil, nil + } + ks, err := st.Keys(ctx) + if err != nil { + return nil, err + } + slices.Sort(ks) + var out []HeldChain + for _, k := range ks { + if !strings.HasPrefix(k, ChainPrefix) { + continue + } + v, _, found, err := st.Get(ctx, k) + if err != nil { + return nil, err + } + var c HeldChain + if !found || json.Unmarshal(v, &c) != nil { + continue + } + out = append(out, c) + } + return out, nil +} + // SkipTakeIn ends a held source revision unwritten by hand (indexer_take_in_skip, §744): it says take-in.taken with no entries, // skipped, by and why, and deletes the branch's held take-in; the caller acknowledges the source-moved. func (ix *Indexer) SkipTakeIn(ctx context.Context, e meshdelivery.SourceMovedEvent, by, why string) error { @@ -243,7 +366,7 @@ func HeldTools(held, takeIns Store, ask func(key string, value []byte) (TakeInAs {Name: PendingTool, Description: "Each watched branch's held take-in (design 47 H §744): the source revision of the gate's source-moved the indexer " + "could not take in (the catalog store full, a read of the forge unanswered), why and since when, in order, each kept here once " + - "acknowledged and taken in again on the indexer's room and at its start; and each running take-in that waits over a minute on another part, its waiting-on (part, verb, since). indexer_take_in_pending takes it again now, indexer_take_in_skip ends it unwritten.", + "acknowledged and taken in again on the indexer's room and at its start; each running take-in that waits over a minute on another part, its waiting-on (part, verb, since); and each running chain of held take-ins, the source being taken in, since when and how many wait. indexer_take_in_pending takes it again now, indexer_take_in_skip ends it unwritten.", Input: map[string]any{"type": "object", "properties": map[string]any{}}, Run: func(map[string]any) (any, error) { h, err := ReadHeld(context.Background(), held) @@ -251,11 +374,16 @@ func HeldTools(held, takeIns Store, ask func(key string, value []byte) (TakeInAs return PendingAnswer{Held: h}, err } r, err := ReadRunning(context.Background(), takeIns) - return PendingAnswer{Held: h, Running: r}, err + if err != nil { + return PendingAnswer{Held: h, Running: r}, err + } + c, err := ReadChains(context.Background(), takeIns) + return PendingAnswer{Held: h, Running: r, Chains: c}, err }}, {Name: TakeInPendingTool, Description: "Take a watched branch's held source-moved in again now (design 47 H §744), as the indexer's room would: " + - "answers whether it was taken in. indexer_pending names the held branches.", + "the held sources are taken in in order beside the asks, and it answers at once that the chain started at a source and " + + "how many wait, or, while a chain runs, its state (design 47 H2 H-A-497). indexer_pending names the held branches and the running chains.", Input: map[string]any{"type": "object", "required": []string{"branch"}, "properties": map[string]any{ "branch": map[string]any{"type": "string", "description": "the watched branch, as indexer_pending names it"}, "why": map[string]any{"type": "string", "description": "why it is taken again now"}}}, diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go index 9ff2077df..856bee2a7 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go +++ b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go @@ -384,6 +384,16 @@ func TestATakeInOnAFullStoreIsHeldAndSaidOnce(t *testing.T) { } var lines []string ix.Logf = func(format string, args ...any) { lines = append(lines, fmt.Sprintf(format, args...)) } + ix.TakeIns = newMem() + // H-A-497 addendum: while the chain runs, its key names the source being taken in and how many wait behind it. + var steps []string + read := ix.Read + ix.Read = func(ctx context.Context, repository, sha, dir string) (RepoRead, error) { + if c, ok, _ := ix.ChainOf(ctx, "main"); ok { + steps = append(steps, fmt.Sprintf("%d/%d", c.Source, c.Waiting)) + } + return read(ctx, repository, sha, dir) + } objects.capacity = 1 << 22 if ok, err := ix.RoomAgain(ctx); err != nil || !ok || rooms != 1 { t.Fatalf("room said %v %d times (%v), want once the store has room", ok, rooms, err) @@ -416,7 +426,20 @@ func TestATakeInOnAFullStoreIsHeldAndSaidOnce(t *testing.T) { t.Errorf("source 6 was taken in behind a held one: %q", l) } } - ix.Logf = nil + if !slices.Contains(steps, "4/2") || !slices.Contains(steps, "5/1") { + t.Errorf("the chain's key read at each take-in %v, want main at 4 with 2 waiting and at 5 with 1", steps) + } + // H-A-497 addendum: the chain's key in the state takeins goes at its end, here its first held again, and one a stop + // left is deleted at the start. + if chains, err := ReadChains(ctx, ix.TakeIns); err != nil || len(chains) != 0 { + t.Errorf("chains %+v (%v) after the chain stopped, want none", chains, err) + } + ix.writeChain(ctx, HeldChain{Branch: "main", Source: 9, Waiting: 1}) + if ix.EndChains(ctx); func() bool { c, _ := ReadChains(ctx, ix.TakeIns); return len(c) != 0 }() { + t.Error("a chain's key left by a stop stays after the start") + } + ix.Read = read + ix.Logf, ix.TakeIns = nil, nil for _, src := range []int64{5, 6} { ix.unhold(ctx, "main", src) }