diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index e48e3cbc6..801db723f 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -664,7 +664,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro sources.refresh() }) // The held take-ins are taken in again from the state held at the start, before a branch's consumer gives a later one - // (design 47 H, H-A-114, H-A-118). + // (design 47 H, H-A-114, H-A-118); a chain's key a stop left is deleted first (H2 H-A-497 addendum). + ix.EndChains(ctx) sources.takeAgain() sources.refresh() if stopped != nil { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go index 4a42bcdde..35a99fb30 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go @@ -265,24 +265,34 @@ type actor struct { s *sourceMoved } -// HeldAgain takes a branch's held source revisions in again now (indexer_take_in_pending). +// HeldAgain takes a branch's held source revisions in again now (indexer_take_in_pending), beside the asks' loop, and answers +// at once (design 47 H2 H-A-497 and its addendum): a chain that runs already answers its state from the state takeins; held +// sources start the chain at the first, written there before it runs, answered with how many wait. func (a actor) HeldAgain(ctx context.Context, branch string) index.TakenIn { r := index.TakenIn{Module: branch} - if hs, err := a.HeldOf(ctx, branch); err != nil || len(hs) == 0 { + if c, running, err := a.ChainOf(ctx, branch); err != nil { + r.Answer = "the state takeins is not read: " + err.Error() + return r + } else if running { + r.Held = true + r.Answer = fmt.Sprintf("a chain runs: source %d taken in since %s, %d wait", c.Source, c.Since.Format(time.RFC3339), c.Waiting) + return r + } + hs, err := a.HeldOf(ctx, branch) + if err != nil || len(hs) == 0 { r.Answer = "the branch " + branch + " holds no source-moved" if err != nil { r.Answer = "the state held is not read: " + err.Error() } return r } - a.s.takeAgain() - hs, err := a.HeldOf(ctx, branch) - still := err != nil || len(hs) > 0 - r.Stored, r.Held = !still, still - r.Answer = "taken in" - if still { - r.Answer = "still held: indexer_pending says why" - } + c := a.StartChain(ctx, hs) + go func() { + a.s.takeAgain() + a.EndChain(a.s.ctx, branch) + }() + r.Held = true + r.Answer = fmt.Sprintf("chain started at source %d, %d wait", c.Source, c.Waiting) return r } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go index 54707cdaf..9723494fa 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go @@ -97,6 +97,98 @@ func TestAHeldSourceMovedIsSkippedByHand(t *testing.T) { } } +// chainStore is a takeins state in memory, safe beside the chain's own goroutine, which says on ended when a chain key goes. +type chainStore struct { + mu sync.Mutex + vals map[string][]byte + ended chan string +} + +func (c *chainStore) Get(_ context.Context, k string) ([]byte, uint64, bool, error) { + c.mu.Lock() + defer c.mu.Unlock() + v, ok := c.vals[k] + return v, 1, ok, nil +} +func (c *chainStore) Keys(context.Context) ([]string, error) { + c.mu.Lock() + defer c.mu.Unlock() + var out []string + for k := range c.vals { + out = append(out, k) + } + return out, nil +} +func (c *chainStore) Create(_ context.Context, k string, v []byte) error { + c.mu.Lock() + defer c.mu.Unlock() + c.vals[k] = v + return nil +} +func (c *chainStore) Update(ctx context.Context, k string, v []byte, _ uint64) error { + return c.Create(ctx, k, v) +} +func (c *chainStore) Delete(_ context.Context, k string, _ uint64) error { + c.mu.Lock() + _, had := c.vals[k] + delete(c.vals, k) + c.mu.Unlock() + if had && strings.HasPrefix(k, index.ChainPrefix) { + c.ended <- k + } + return nil +} + +// Design 47 H2 H-A-497 and its addendum: a hand ask of a branch with held sources starts the chain beside the asks' loop +// and answers at once; an ask that meets the running chain answers its state, which the state takeins holds and +// indexer_pending shows; the chain's key goes at its end. +func TestAHandAskOfAHeldBranchAnswersAtOnce(t *testing.T) { + ctx := context.Background() + held := &memStore{vals: map[string][]byte{}} + takeIns := &chainStore{vals: map[string][]byte{}, ended: make(chan string, 4)} + ix := &index.Indexer{Held: held, TakeIns: takeIns, Logf: func(string, ...any) {}, + Now: func() time.Time { return time.Date(2026, 11, 22, 21, 7, 0, 0, time.UTC) }} + handlers := map[string]func(jetstream.Msg){} + s := &sourceMoved{ctx: ctx, ix: ix, + branches: func(context.Context) ([]string, error) { return []string{"main"}, nil }, + bind: func(_ context.Context, branch string, each func(jetstream.Msg)) (func(), error) { + handlers[branch] = each + return func() {}, nil + }, + logf: func(string, ...any) {}} + s.refresh() + for _, n := range []int{114, 115, 116} { + m := &fakeMsg{subject: fmt.Sprintf("mesh.seat.mesh-delivery.event.source-moved.main.%d", n), + data: []byte(fmt.Sprintf(`{"v":1,"branch":"main","source":%d,"call":"%d123456789abcde","heads":[]}`, n, n))} + handlers["main"](msgOf{f: m}) + } + a := actor{ix, s} + s.again.Lock() // the chain waits for the lock, so it is still running at the second ask + if r := a.HeldAgain(ctx, "main"); r.Answer != "chain started at source 114, 2 wait" { + s.again.Unlock() + t.Fatalf("the first ask answered %+v, want the chain started at once", r) + } + r := a.HeldAgain(ctx, "main") + var pending any + var err error + for _, tool := range index.HeldTools(held, takeIns, nil, ix.Now, nil) { + if tool.Name == index.PendingTool { + pending, err = tool.Run(map[string]any{}) + } + } + s.again.Unlock() + chains := pending.(index.PendingAnswer).Chains + if r.Answer != "a chain runs: source 114 taken in since 2026-11-22T21:07:00Z, 2 wait" || !r.Held { + t.Errorf("the ask meeting the chain answered %+v, want its state at once", r) + } + if err != nil || len(chains) != 1 || chains[0].Branch != "main" || chains[0].Source != 114 || chains[0].Waiting != 2 { + t.Errorf("indexer_pending shows chains %+v (%v), want main's at 114 with 2 waiting", chains, err) + } + if k := <-takeIns.ended; k != index.ChainKey("main") { + t.Errorf("the key %s went, want main's chain", k) + } +} + // revisionObjects is the catalog revisions' object bucket in memory. type revisionObjects map[string][]byte diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go index 11a52cd33..b86f544c9 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go @@ -33,8 +33,8 @@ type takerIn interface { // written back, by compare-and-set at its revision, with what the indexer stored or why not. A value // answered already (this indexer's own write) or deleted is left. func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) { - if e.Operation() != jetstream.KeyValuePut || strings.HasPrefix(e.Key(), index.RunningPrefix) { - return // a running take-in's waiting-on is the indexer's own record, no ask (H-A-409) + if e.Operation() != jetstream.KeyValuePut || strings.HasPrefix(e.Key(), index.RunningPrefix) || strings.HasPrefix(e.Key(), index.ChainPrefix) { + return // a running take-in's waiting-on and a running chain are the indexer's own records, no ask (H-A-409, H-A-497) } a, err := index.DecodeTakeIn(e.Value()) if err != nil { diff --git a/modules/mesh-delivery-indexer/internal/index/held.go b/modules/mesh-delivery-indexer/internal/index/held.go index 41f1d971e..dc5c04988 100644 --- a/modules/mesh-delivery-indexer/internal/index/held.go +++ b/modules/mesh-delivery-indexer/internal/index/held.go @@ -51,6 +51,9 @@ type PendingAnswer struct { Held []HeldTakeIn `json:"held"` // Running are the take-ins that run, each with what it waits on (design 47 H2 §803, H-A-409). Running []RunningTakeIn `json:"running,omitempty"` + // Chains are the running chains of held take-ins, each branch's source being taken in, since when and how many wait + // (H-A-497 addendum). + Chains []HeldChain `json:"chains,omitempty"` } // ErrHeldUnwritten says a source-moved that could not be taken in is not written into the state held either: its message @@ -178,22 +181,142 @@ func (ix *Indexer) TakeHeldIn(ctx context.Context) { return } stopped, landed := map[string]bool{}, map[string]int64{} + behind := map[string]int{} for _, h := range all { + behind[h.Branch]++ + } + for i, h := range all { + behind[h.Branch]-- if stopped[h.Branch] { continue } if prev, ok := landed[h.Branch]; ok { ix.logf("take-in: held source %d of %s taken in next, after %d (H-A-497)", h.Source, h.Branch, prev) } + ix.writeChain(ctx, HeldChain{Branch: h.Branch, Source: h.Source, Since: ix.now().UTC(), Waiting: behind[h.Branch]}) if _, err := ix.TakeInSource(ctx, h.Event()); err == nil { landed[h.Branch] = h.Source } else { ix.logf("take-in: source revision %d of %s is still held: %v", h.Source, h.Branch, err) stopped[h.Branch] = true } + // The chain of a branch ends at its first held again or its last held (H-A-497 addendum). + if stopped[h.Branch] || !slices.ContainsFunc(all[i+1:], func(n HeldTakeIn) bool { return n.Branch == h.Branch }) { + ix.endChain(ctx, h.Branch) + } } } +// ChainPrefix is the takeins keys of the chains of held take-ins: `chain.`, the branch escaped as a key's name is, +// holding the branch's chain while it runs (design 47 H2 H-A-497 and its addendum): written at each step, deleted at its end. +const ChainPrefix = "chain." + +// ChainKey is the takeins key of branch's chain. +func ChainKey(branch string) string { return ChainPrefix + keys.EscapeName(branch) } + +// HeldChain is a running chain of a branch's held take-ins (H-A-497 addendum): the source being taken in, since when, and +// how many held sources of the branch wait behind it. +type HeldChain struct { + Branch string `json:"branch"` + Source int64 `json:"source"` + Since time.Time `json:"since"` + Waiting int `json:"waiting"` +} + +// writeChain writes branch's chain at its step c into the state takeins; a failed write is said. +func (ix *Indexer) writeChain(ctx context.Context, c HeldChain) { + if ix.TakeIns == nil { + return + } + data, err := json.Marshal(c) + if err == nil { + err = write(ctx, ix.TakeIns, ChainKey(c.Branch), func([]byte, bool) ([]byte, bool, error) { return data, false, nil }) + } + if err != nil { + ix.logf("take-in: the chain of %s at source revision %d is not written into %s: %v", c.Branch, c.Source, TakeInsState, err) + } +} + +// EndChain deletes branch's chain from the state takeins once the take-in its ask started returned, so a chain whose held +// sources another act took first is never left running. +func (ix *Indexer) EndChain(ctx context.Context, branch string) { ix.endChain(ctx, branch) } + +// EndChains deletes every chain the state takeins holds, at the indexer's start (H-A-497 addendum): no chain runs before the +// start's take-in, so a key left by a stop between a chain's last take-in and its end would answer "a chain runs" for good. +func (ix *Indexer) EndChains(ctx context.Context) { + chains, err := ReadChains(ctx, ix.TakeIns) + if err != nil { + ix.logf("take-in: the chains in %s are not read, so none left by a stop is deleted: %v", TakeInsState, err) + return + } + for _, c := range chains { + ix.logf("take-in: the chain of %s at source revision %d, left by a stop, is deleted at the start", c.Branch, c.Source) + ix.endChain(ctx, c.Branch) + } +} + +// endChain deletes branch's chain from the state takeins. +func (ix *Indexer) endChain(ctx context.Context, branch string) { + if ix.TakeIns == nil { + return + } + if err := write(ctx, ix.TakeIns, ChainKey(branch), func([]byte, bool) ([]byte, bool, error) { return nil, true, nil }); err != nil { + ix.logf("take-in: the chain of %s is not deleted from %s: %v", branch, TakeInsState, err) + } +} + +// StartChain writes branch's chain at its first held source, before the chain is taken in beside the asks (H-A-497 +// addendum), so an ask after it finds it running; it answers that chain. +func (ix *Indexer) StartChain(ctx context.Context, held []HeldTakeIn) HeldChain { + c := HeldChain{Branch: held[0].Branch, Source: held[0].Source, Since: ix.now().UTC(), Waiting: len(held) - 1} + ix.writeChain(ctx, c) + return c +} + +// ChainOf is branch's running chain in the state takeins, and whether one runs. +func (ix *Indexer) ChainOf(ctx context.Context, branch string) (HeldChain, bool, error) { + if ix.TakeIns == nil { + return HeldChain{}, false, nil + } + v, _, found, err := ix.TakeIns.Get(ctx, ChainKey(branch)) + if err != nil || !found { + return HeldChain{}, false, err + } + var c HeldChain + if err := json.Unmarshal(v, &c); err != nil { + return HeldChain{}, false, fmt.Errorf("%s.%s: %w", TakeInsState, ChainKey(branch), err) + } + return c, true, nil +} + +// ReadChains are the running chains the state takeins holds (H-A-497 addendum), each branch's once. +func ReadChains(ctx context.Context, st Store) ([]HeldChain, error) { + if st == nil { + return nil, nil + } + ks, err := st.Keys(ctx) + if err != nil { + return nil, err + } + slices.Sort(ks) + var out []HeldChain + for _, k := range ks { + if !strings.HasPrefix(k, ChainPrefix) { + continue + } + v, _, found, err := st.Get(ctx, k) + if err != nil { + return nil, err + } + var c HeldChain + if !found || json.Unmarshal(v, &c) != nil { + continue + } + out = append(out, c) + } + return out, nil +} + // SkipTakeIn ends a held source revision unwritten by hand (indexer_take_in_skip, §744): it says take-in.taken with no entries, // skipped, by and why, and deletes the branch's held take-in; the caller acknowledges the source-moved. func (ix *Indexer) SkipTakeIn(ctx context.Context, e meshdelivery.SourceMovedEvent, by, why string) error { @@ -243,7 +366,7 @@ func HeldTools(held, takeIns Store, ask func(key string, value []byte) (TakeInAs {Name: PendingTool, Description: "Each watched branch's held take-in (design 47 H §744): the source revision of the gate's source-moved the indexer " + "could not take in (the catalog store full, a read of the forge unanswered), why and since when, in order, each kept here once " + - "acknowledged and taken in again on the indexer's room and at its start; and each running take-in that waits over a minute on another part, its waiting-on (part, verb, since). indexer_take_in_pending takes it again now, indexer_take_in_skip ends it unwritten.", + "acknowledged and taken in again on the indexer's room and at its start; each running take-in that waits over a minute on another part, its waiting-on (part, verb, since); and each running chain of held take-ins, the source being taken in, since when and how many wait. indexer_take_in_pending takes it again now, indexer_take_in_skip ends it unwritten.", Input: map[string]any{"type": "object", "properties": map[string]any{}}, Run: func(map[string]any) (any, error) { h, err := ReadHeld(context.Background(), held) @@ -251,11 +374,16 @@ func HeldTools(held, takeIns Store, ask func(key string, value []byte) (TakeInAs return PendingAnswer{Held: h}, err } r, err := ReadRunning(context.Background(), takeIns) - return PendingAnswer{Held: h, Running: r}, err + if err != nil { + return PendingAnswer{Held: h, Running: r}, err + } + c, err := ReadChains(context.Background(), takeIns) + return PendingAnswer{Held: h, Running: r, Chains: c}, err }}, {Name: TakeInPendingTool, Description: "Take a watched branch's held source-moved in again now (design 47 H §744), as the indexer's room would: " + - "answers whether it was taken in. indexer_pending names the held branches.", + "the held sources are taken in in order beside the asks, and it answers at once that the chain started at a source and " + + "how many wait, or, while a chain runs, its state (design 47 H2 H-A-497). indexer_pending names the held branches and the running chains.", Input: map[string]any{"type": "object", "required": []string{"branch"}, "properties": map[string]any{ "branch": map[string]any{"type": "string", "description": "the watched branch, as indexer_pending names it"}, "why": map[string]any{"type": "string", "description": "why it is taken again now"}}}, diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go index 9ff2077df..856bee2a7 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go +++ b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go @@ -384,6 +384,16 @@ func TestATakeInOnAFullStoreIsHeldAndSaidOnce(t *testing.T) { } var lines []string ix.Logf = func(format string, args ...any) { lines = append(lines, fmt.Sprintf(format, args...)) } + ix.TakeIns = newMem() + // H-A-497 addendum: while the chain runs, its key names the source being taken in and how many wait behind it. + var steps []string + read := ix.Read + ix.Read = func(ctx context.Context, repository, sha, dir string) (RepoRead, error) { + if c, ok, _ := ix.ChainOf(ctx, "main"); ok { + steps = append(steps, fmt.Sprintf("%d/%d", c.Source, c.Waiting)) + } + return read(ctx, repository, sha, dir) + } objects.capacity = 1 << 22 if ok, err := ix.RoomAgain(ctx); err != nil || !ok || rooms != 1 { t.Fatalf("room said %v %d times (%v), want once the store has room", ok, rooms, err) @@ -416,7 +426,20 @@ func TestATakeInOnAFullStoreIsHeldAndSaidOnce(t *testing.T) { t.Errorf("source 6 was taken in behind a held one: %q", l) } } - ix.Logf = nil + if !slices.Contains(steps, "4/2") || !slices.Contains(steps, "5/1") { + t.Errorf("the chain's key read at each take-in %v, want main at 4 with 2 waiting and at 5 with 1", steps) + } + // H-A-497 addendum: the chain's key in the state takeins goes at its end, here its first held again, and one a stop + // left is deleted at the start. + if chains, err := ReadChains(ctx, ix.TakeIns); err != nil || len(chains) != 0 { + t.Errorf("chains %+v (%v) after the chain stopped, want none", chains, err) + } + ix.writeChain(ctx, HeldChain{Branch: "main", Source: 9, Waiting: 1}) + if ix.EndChains(ctx); func() bool { c, _ := ReadChains(ctx, ix.TakeIns); return len(c) != 0 }() { + t.Error("a chain's key left by a stop stays after the start") + } + ix.Read = read + ix.Logf, ix.TakeIns = nil, nil for _, src := range []int64{5, 6} { ix.unhold(ctx, "main", src) }