diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 512312a8..408e6b1e 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -590,6 +590,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro _ = m.Ack() sources.refresh() }) + // The held take-ins are taken in again from the state held at the start, before a branch's consumer gives a later one + // (design 47 H, H-A-114, H-A-118). + sources.takeAgain() sources.refresh() if stopped != nil { return stopped diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go index 7146a8e4..6b12486f 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go @@ -29,16 +29,8 @@ type sourceMoved struct { logf func(string, ...any) mu sync.Mutex - bound map[string]func() // by branch: the consumer's stop - held map[string]heldTake // by branch: the source-moved held unacknowledged (design 47 H §744) - again sync.Mutex // one take-in again at a time -} - -// heldTake is a source-moved its branch's consumer gave and the indexer could not take in: unacknowledged, so its consumer -// gives no later one of that branch, and taken in again on the indexer's own room or by hand, never on a timer. -type heldTake struct { - m jetstream.Msg - e meshdelivery.SourceMovedEvent + bound map[string]func() // by branch: the consumer's stop + again sync.Mutex // one take-in again at a time } // watchedBranches are the branches every registration watches, each once, sorted. @@ -104,8 +96,10 @@ func (s *sourceMoved) refresh() { } } -// heard takes one source-moved of a branch in and acknowledges it once taken. One of the old form (no field its form requires, -// §765) is refused for good with a journal line naming it; any other failure leaves it unacknowledged, held. +// heard takes one source-moved of a branch in and acknowledges it once taken or written into the state held (design 47 H §744, +// H-A-114, H-A-118): a branch that holds earlier ones holds it behind them, so its source revisions are taken in in their order. +// One of the old form (no field its form requires, §765) is refused for good with a journal line naming it; one that is neither +// taken in nor written into the state held stays unacknowledged, and the bus gives it again. func (s *sourceMoved) heard(branch string, m jetstream.Msg) { var e meshdelivery.SourceMovedEvent if err := json.Unmarshal(m.Data(), &e); err != nil { @@ -113,49 +107,36 @@ func (s *sourceMoved) heard(branch string, m jetstream.Msg) { _ = m.Term() return } - if _, err := s.ix.TakeInSource(s.ctx, e); err != nil { - if errors.Is(err, index.ErrOldForm) { - s.logf("source-moved of %s on %s is refused, the old form: %v", branch, m.Subject(), err) - _ = m.Term() - return - } - s.logf("source-moved: source revision %d of %s is held, not taken in: %v", e.Source, branch, err) - s.mu.Lock() - if s.held == nil { - s.held = map[string]heldTake{} - } - s.held[branch] = heldTake{m: m, e: e} - s.mu.Unlock() + s.again.Lock() + defer s.again.Unlock() + behind, err := s.ix.HoldBehind(s.ctx, e) + if !behind && err == nil { + _, err = s.ix.TakeInSource(s.ctx, e) + } + switch { + case errors.Is(err, index.ErrOldForm): + s.logf("source-moved of %s on %s is refused, the old form: %v", branch, m.Subject(), err) + _ = m.Term() return + case errors.Is(err, index.ErrHeldUnwritten): + s.logf("source-moved: source revision %d of %s is neither taken in nor held, so it is left unacknowledged: %v", e.Source, branch, err) + return + case err != nil: + s.logf("source-moved: source revision %d of %s is held, not taken in: %v", e.Source, branch, err) + case behind: + s.logf("source-moved: source revision %d of %s is held behind the branch's earlier held ones", e.Source, branch) } if err := m.Ack(); err != nil { - s.logf("source-moved: source revision %d of %s is taken in and its acknowledgement failed: %v", e.Source, branch, err) + s.logf("source-moved: source revision %d of %s is taken in or held and its acknowledgement failed: %v", e.Source, branch, err) } } -// takeAgain takes every held source-moved in again, on the indexer's own room (H-A-50): each taken in is acknowledged, so its -// branch's consumer gives the next; one that still cannot be taken in stays held. +// takeAgain takes every held source revision in again from the state held, each branch in its order, on the indexer's own room +// (H-A-50) and at its start (H-A-114). func (s *sourceMoved) takeAgain() { s.again.Lock() defer s.again.Unlock() - s.mu.Lock() - held := make(map[string]heldTake, len(s.held)) - for b, h := range s.held { - held[b] = h - } - s.mu.Unlock() - for branch, h := range held { - if _, err := s.ix.TakeInSource(s.ctx, h.e); err != nil { - s.logf("source-moved: source revision %d of %s is still held: %v", h.e.Source, branch, err) - continue - } - s.mu.Lock() - delete(s.held, branch) - s.mu.Unlock() - if err := h.m.Ack(); err != nil { - s.logf("source-moved: source revision %d of %s is taken in again and its acknowledgement failed: %v", h.e.Source, branch, err) - } - } + s.ix.TakeHeldIn(s.ctx) } // actor is the indexer and its hearing of source-moved, as the asks of the state takeins act on them. @@ -164,20 +145,19 @@ type actor struct { s *sourceMoved } -// HeldAgain takes a branch's held source-moved in again now (indexer_take_in_pending). -func (a actor) HeldAgain(_ context.Context, branch string) index.TakenIn { +// HeldAgain takes a branch's held source revisions in again now (indexer_take_in_pending). +func (a actor) HeldAgain(ctx context.Context, branch string) index.TakenIn { r := index.TakenIn{Module: branch} - a.s.mu.Lock() - _, held := a.s.held[branch] - a.s.mu.Unlock() - if !held { + if hs, err := a.HeldOf(ctx, branch); err != nil || len(hs) == 0 { r.Answer = "the branch " + branch + " holds no source-moved" + if err != nil { + r.Answer = "the state held is not read: " + err.Error() + } return r } a.s.takeAgain() - a.s.mu.Lock() - _, still := a.s.held[branch] - a.s.mu.Unlock() + hs, err := a.HeldOf(ctx, branch) + still := err != nil || len(hs) > 0 r.Stored, r.Held = !still, still r.Answer = "taken in" if still { @@ -186,26 +166,21 @@ func (a actor) HeldAgain(_ context.Context, branch string) index.TakenIn { return r } -// HeldSkip acknowledges a branch's held source-moved unwritten (indexer_take_in_skip): take-in.taken says it skipped. +// HeldSkip ends a branch's held source revision unwritten (indexer_take_in_skip): take-in.taken says it skipped. func (a actor) HeldSkip(ctx context.Context, branch string, source int64, by, why string) index.TakenIn { r := index.TakenIn{Module: branch} - a.s.mu.Lock() - h, held := a.s.held[branch] - a.s.mu.Unlock() - if !held || h.e.Source != source { + a.s.again.Lock() + defer a.s.again.Unlock() + hs, err := a.HeldOf(ctx, branch) + i := slices.IndexFunc(hs, func(h index.HeldTakeIn) bool { return h.Source == source }) + if err != nil || i < 0 { r.Answer = fmt.Sprintf("the branch %s holds no source revision %d", branch, source) return r } - if err := a.Indexer.SkipTakeIn(ctx, h.e, by, why); err != nil { + if err := a.Indexer.SkipTakeIn(ctx, hs[i].Event(), by, why); err != nil { r.Answer = "not skipped: " + err.Error() return r } - a.s.mu.Lock() - delete(a.s.held, branch) - a.s.mu.Unlock() - if err := h.m.Ack(); err != nil { - a.s.logf("source-moved: source revision %d of %s is skipped and its acknowledgement failed: %v", source, branch, err) - } r.Stored, r.Answer = true, "skipped" return r } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go index e29c7a5a..31e5fb6f 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go @@ -51,11 +51,12 @@ func TestEachWatchedBranchHasItsConsumerAndTheOldFormIsRefused(t *testing.T) { } } -// Design 47 H §744: a source-moved that cannot be taken in stays unacknowledged, held; indexer_take_in_skip acknowledges it and -// take-in.taken says it skipped. +// Design 47 H §744, H-A-114, H-A-118: a source-moved that cannot be taken in is acknowledged once written into the state held, +// a later one of its branch is held behind it, and indexer_take_in_skip ends the named one: take-in.taken says it skipped. func TestAHeldSourceMovedIsSkippedByHand(t *testing.T) { var said []string - ix := &index.Indexer{SayTakeIn: func(_ context.Context, event, id string, _ any) error { said = append(said, event+" "+id); return nil }} + held := &memStore{vals: map[string][]byte{}} + ix := &index.Indexer{Held: held, SayTakeIn: func(_ context.Context, event, id string, _ any) error { said = append(said, event+" "+id); return nil }} handlers := map[string]func(jetstream.Msg){} s := &sourceMoved{ctx: context.Background(), ix: ix, branches: func(context.Context) ([]string, error) { return []string{"main"}, nil }, @@ -67,17 +68,23 @@ func TestAHeldSourceMovedIsSkippedByHand(t *testing.T) { s.refresh() m := &fakeMsg{subject: "mesh.seat.mesh-delivery.event.source-moved.main.4", data: []byte(`{"v":1,"branch":"main","source":4,"call":"0123456789abcdef","heads":[]}`)} handlers["main"](msgOf{f: m}) - if m.acked != 0 || m.termd != 0 { - t.Fatalf("a source-moved the indexer cannot take in was acknowledged %d and terminated %d times, want held", m.acked, m.termd) + later := &fakeMsg{subject: "mesh.seat.mesh-delivery.event.source-moved.main.5", data: []byte(`{"v":1,"branch":"main","source":5,"call":"1123456789abcdef","heads":[]}`)} + handlers["main"](msgOf{f: later}) + hs, err := ix.HeldOf(context.Background(), "main") + if m.acked != 1 || m.termd != 0 || later.acked != 1 || err != nil || len(hs) != 2 || hs[0].Source != 4 || hs[1].Source != 5 || hs[0].Call != "0123456789abcdef" { + t.Fatalf("acknowledged %d and %d, held %+v (%v); want both acknowledged once kept in order in the state held", m.acked, later.acked, hs, err) } a := actor{ix, s} - if r := a.HeldSkip(context.Background(), "main", 3, "operator", "stale"); r.Stored || m.acked != 0 { + if r := a.HeldSkip(context.Background(), "main", 3, "operator", "stale"); r.Stored { t.Errorf("a skip of another source answered %+v", r) } - if r := a.HeldSkip(context.Background(), "main", 4, "operator", "stale"); !r.Stored || m.acked != 1 { - t.Errorf("the skip answered %+v and acknowledged %d times, want it skipped and acknowledged once", r, m.acked) + if r := a.HeldSkip(context.Background(), "main", 4, "operator", "stale"); !r.Stored { + t.Errorf("the skip answered %+v, want it skipped", r) } if !slices.Contains(said, "take-in.taken "+index.TakenID("main", 4)) { t.Errorf("said %v, want take-in.taken of main 4", said) } + if hs, _ := ix.HeldOf(context.Background(), "main"); len(hs) != 1 || hs[0].Source != 5 { + t.Errorf("held %+v after the skip, want source 5 alone", hs) + } } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go index f155742f..9c2dbddd 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go @@ -363,7 +363,8 @@ func absent(v []byte) bool { return v == nil } // revision main.11 carrying avahi as its one mover, pinned pipeline.main.11, said once as take-in.taken; the same source-moved // given again says the same from the kept movers and keeps no second revision; // - pipeline-started of main.12 pins it, a restart keeps main.11 while its pipeline runs and removes main.10; -// - with the store full, main.13 is held, said once as take-in.pending and written in the state held; nothing takes it in on a +// - with the store full, main.13 is held, said once as take-in.pending, written whole in the state held and acknowledged +// (H-A-114, H-A-118); nothing takes it in on a // timer: main.11's pipeline-ended frees its pin, the revision goes, the indexer says its room and takes main.13 in on it; // - beta's take-in whose commit the forge cannot read is held while main's next goes on; the hand skip, with why, is recorded in // the hand-act log before it is asked, and take-in.taken says it skipped; @@ -470,9 +471,16 @@ func TestIndexerWhole(t *testing.T) { t.Fatalf("take-in.pending %+v names no why", p) } untilKey(t, ctx, held, keys.EscapeName("main"), func(v []byte) bool { - var h index.HeldTakeIn - return v != nil && json.Unmarshal(v, &h) == nil && h.Source == 13 && h.Why != "" && !h.Since.IsZero() + var hs []index.HeldTakeIn + return v != nil && json.Unmarshal(v, &hs) == nil && len(hs) == 1 && hs[0].Source == 13 && hs[0].Why != "" && !hs[0].Since.IsZero() && + hs[0].Call != "" && len(hs[0].Heads) == 1 && hs[0].Heads[0].Commit == c13 }) + // H-A-114: the held source-moved is acknowledged on the bus, kept only in the state held. + if cons, err := testAccount.NamedEventConsumer(ctx, js, meshdelivery.StreamSourceMoved, keys.EscapeName("main")); err != nil { + t.Fatal(err) + } else if info, err := cons.Info(ctx); err != nil || info.NumAckPending != 0 { + t.Errorf("the held source-moved's consumer %+v (%v), want nothing unacknowledged", info, err) + } setCap(t, ctx, js, 1<<30) // a larger cap alone wakes nothing: no timer takes a held take-in in sayPipeline(t, ctx, js, true, "main", 11) if got := heard.next(t, "main", 13); len(got.Entries) != 1 || got.Entries[0].Commit != c13 { diff --git a/modules/mesh-delivery-indexer/internal/index/held.go b/modules/mesh-delivery-indexer/internal/index/held.go index ac2fcdd9..ab9fae0d 100644 --- a/modules/mesh-delivery-indexer/internal/index/held.go +++ b/modules/mesh-delivery-indexer/internal/index/held.go @@ -16,9 +16,9 @@ import ( "git.novox.be/novox/mesh-sdk/go/keys" ) -// HeldState is the indexer's state held (design 47 H §744, H-A-97): one key per watched branch, the branch escaped as a key's -// name is, holding the source-moved its consumer gave and the indexer could not take in, written when it is held and deleted at -// its take-in or skip. indexer_pending reads it. +// HeldState is the indexer's state held (design 47 H §744, H-A-97, H-A-118): one key per watched branch, the branch escaped as +// a key's name is, holding the branch's held source-moved events in order, oldest first, each written when it is held (its +// message acknowledged once written) and taken out at its take-in or skip; the key goes with its last. indexer_pending reads it. const HeldState = "held" // The held take-in's tools (§744). @@ -28,56 +28,156 @@ const ( TakeInSkipTool = "indexer_take_in_skip" ) -// HeldTakeIn is a value of the state held: the branch's held source revision, why it is held and since when. +// HeldTakeIn is one held source-moved of the state held: its branch, source revision, why it is held and since when (H-A-97), +// and the event's call, heads and start, so a take-in after a restart reads nothing of any verb (H-A-118). type HeldTakeIn struct { - Branch string `json:"branch"` - Source int64 `json:"source"` - Why string `json:"why"` - Since time.Time `json:"since"` + Branch string `json:"branch"` + Source int64 `json:"source"` + Why string `json:"why"` + Since time.Time `json:"since"` + Call string `json:"call"` + Heads []meshdelivery.SourceMovedEventHeadsItem `json:"heads"` + Start bool `json:"start,omitempty"` } -// PendingAnswer is indexer_pending's answer (H-A-102): every branch's held take-in, in the state held's form (since in RFC 3339). +// Event is the held source-moved, as the gate said it. +func (h HeldTakeIn) Event() meshdelivery.SourceMovedEvent { + return meshdelivery.SourceMovedEvent{V: keys.Version, Branch: h.Branch, Call: h.Call, Heads: h.Heads, Source: h.Source, Start: h.Start} +} + +// PendingAnswer is indexer_pending's answer (H-A-102): every branch's held take-ins, in the state held's form (since in RFC 3339). type PendingAnswer struct { Held []HeldTakeIn `json:"held"` } -// hold writes the branch's held take-in and says take-in.pending, once per source: a source already held is neither written nor -// said again, so its since stays. -func (ix *Indexer) hold(ctx context.Context, e meshdelivery.SourceMovedEvent, why string) { - if ix.Held != nil { - key := keys.EscapeName(e.Branch) - if v, _, found, err := ix.Held.Get(ctx, key); err == nil && found { - var h HeldTakeIn - if json.Unmarshal(v, &h) == nil && h.Source == e.Source { - return +// ErrHeldUnwritten says a source-moved that could not be taken in is not written into the state held either: its message +// stays unacknowledged, so the bus gives it again. +var ErrHeldUnwritten = errors.New("the held take-in is not written into the state held") + +// decodeHeld reads a key of the state held: the branch's held events in order. +func decodeHeld(v []byte) ([]HeldTakeIn, error) { + var hs []HeldTakeIn + if err := json.Unmarshal(v, &hs); err != nil { + return nil, fmt.Errorf("not a value of the state %s: %w", HeldState, err) + } + return hs, nil +} + +// HeldOf is the branch's held events in order; none when it holds none. +func (ix *Indexer) HeldOf(ctx context.Context, branch string) ([]HeldTakeIn, error) { + if ix.Held == nil { + return nil, nil + } + v, _, found, err := ix.Held.Get(ctx, keys.EscapeName(branch)) + if err != nil || !found { + return nil, err + } + return decodeHeld(v) +} + +// hold writes e into its branch's held events, after those held before it, and says take-in.pending once per source: a source +// already held is neither written nor said again, so its since stays. ErrHeldUnwritten when it is not written. +func (ix *Indexer) hold(ctx context.Context, e meshdelivery.SourceMovedEvent, why string) error { + if ix.Held == nil { + return fmt.Errorf("%w: the indexer keeps no state %s", ErrHeldUnwritten, HeldState) + } + key, added := keys.EscapeName(e.Branch), false + err := write(ctx, ix.Held, key, func(v []byte, found bool) ([]byte, bool, error) { + var hs []HeldTakeIn + if found { + var err error + if hs, err = decodeHeld(v); err != nil { + return nil, false, err } } - data, err := json.Marshal(HeldTakeIn{Branch: e.Branch, Source: e.Source, Why: why, Since: ix.now().UTC()}) - if err == nil { - err = write(ctx, ix.Held, key, func([]byte, bool) ([]byte, bool, error) { return data, false, nil }) - } - if err != nil { - ix.logf("take-in: %s.%s is not written: %v", HeldState, key, err) + for _, h := range hs { + if h.Source == e.Source { + added = false + return nil, false, nil + } } + added = true + hs = append(hs, HeldTakeIn{Branch: e.Branch, Source: e.Source, Why: why, Since: ix.now().UTC(), Call: e.Call, Heads: e.Heads, Start: e.Start}) + data, err := json.Marshal(hs) + return data, false, err + }) + if err != nil { + ix.logf("take-in: %s.%s is not written: %v", HeldState, key, err) + return fmt.Errorf("%w: %v", ErrHeldUnwritten, err) + } + if !added { + return nil } ix.logf("take-in: source revision %d of %s is held: %s", e.Source, e.Branch, why) if ix.SayTakeIn == nil { - return + return nil } p := meshcatalog.TakeInPending{Branch: e.Branch, Source: e.Source, Why: why} if err := ix.SayTakeIn(ctx, takein.TakeInPendingEvent, PendingID(e.Branch, e.Source), p); err != nil { ix.logf("take-in: take-in.pending of %s is not said: %v", BranchObject(e.Branch, e.Source), err) } + return nil } -// unhold deletes the branch's held take-in, once its source revision is taken in or skipped. -func (ix *Indexer) unhold(ctx context.Context, branch string) { +// HoldBehind writes e behind its branch's earlier held events when the branch holds any, so the branch's source revisions are +// taken in in their order (H-A-118); it answers whether it did. ErrHeldUnwritten when it is not written. +func (ix *Indexer) HoldBehind(ctx context.Context, e meshdelivery.SourceMovedEvent) (bool, error) { + hs, err := ix.HeldOf(ctx, e.Branch) + if err != nil { + return false, fmt.Errorf("%w: %v", ErrHeldUnwritten, err) + } + if len(hs) == 0 { + return false, nil + } + return true, ix.hold(ctx, e, fmt.Sprintf("held behind source revision %d of %s, taken in first", hs[0].Source, e.Branch)) +} + +// unhold takes the branch's held source revision out of the state held, once it is taken in or skipped; the key goes with its last. +func (ix *Indexer) unhold(ctx context.Context, branch string, source int64) { if ix.Held == nil { return } key := keys.EscapeName(branch) - if err := write(ctx, ix.Held, key, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }); err != nil { - ix.logf("take-in: %s.%s is not deleted: %v", HeldState, key, err) + err := write(ctx, ix.Held, key, func(v []byte, found bool) ([]byte, bool, error) { + if !found { + return nil, false, nil + } + hs, err := decodeHeld(v) + if err != nil { + return nil, false, err + } + left := slices.DeleteFunc(slices.Clone(hs), func(h HeldTakeIn) bool { return h.Source == source }) + if len(left) == len(hs) { + return nil, false, nil + } + if len(left) == 0 { + return nil, true, nil + } + data, err := json.Marshal(left) + return data, false, err + }) + if err != nil { + ix.logf("take-in: source revision %d is not taken out of %s.%s: %v", source, HeldState, key, err) + } +} + +// TakeHeldIn takes every branch's held source revisions in, each branch in its order, on the indexer's own room and at its +// start (H-A-114, H-A-118): a branch stops at the first that still cannot be taken in, which stays held with those behind it. +func (ix *Indexer) TakeHeldIn(ctx context.Context) { + all, err := ReadHeld(ctx, ix.Held) + if err != nil { + ix.logf("take-in: the state %s is not read, so no held take-in is taken in again now: %v", HeldState, err) + return + } + stopped := map[string]bool{} + for _, h := range all { + if stopped[h.Branch] { + continue + } + if _, err := ix.TakeInSource(ctx, h.Event()); err != nil { + ix.logf("take-in: source revision %d of %s is still held: %v", h.Source, h.Branch, err) + stopped[h.Branch] = true + } } } @@ -87,29 +187,35 @@ func (ix *Indexer) SkipTakeIn(ctx context.Context, e meshdelivery.SourceMovedEve if err := ix.sayTaken(ctx, meshcatalog.TakeInTaken{Branch: e.Branch, Source: e.Source, Call: e.Call, Skipped: true, By: by, Why: why}); err != nil { return err } - ix.unhold(ctx, e.Branch) + ix.unhold(ctx, e.Branch, e.Source) ix.logf("take-in: source revision %d of %s skipped by %s: %s", e.Source, e.Branch, by, why) return nil } -// ReadHeld is every branch's held take-in, by branch. +// ReadHeld is every branch's held take-ins, by branch, each branch's in its order. func ReadHeld(ctx context.Context, st Store) ([]HeldTakeIn, error) { + out := []HeldTakeIn{} + if st == nil { + return out, nil + } ks, err := st.Keys(ctx) if err != nil { return nil, err } slices.Sort(ks) - out := []HeldTakeIn{} for _, k := range ks { v, _, found, err := st.Get(ctx, k) if err != nil { return nil, err } - var h HeldTakeIn - if !found || json.Unmarshal(v, &h) != nil { + if !found { continue } - out = append(out, h) + hs, err := decodeHeld(v) + if err != nil { + return nil, fmt.Errorf("%s.%s: %w", HeldState, k, err) + } + out = append(out, hs...) } return out, nil } @@ -122,8 +228,8 @@ func HeldTools(held Store, ask func(key string, value []byte) (TakeInAsk, error) return []stdio.Tool{ {Name: PendingTool, Description: "Each watched branch's held take-in (design 47 H §744): the source revision of the gate's source-moved the indexer " + - "could not take in (the catalog store full, a read of the forge unanswered), why and since when; it stays unacknowledged " + - "until the indexer's room wakes it. indexer_take_in_pending takes it again now, indexer_take_in_skip ends it unwritten.", + "could not take in (the catalog store full, a read of the forge unanswered), why and since when, in order, each kept here once " + + "acknowledged and taken in again on the indexer's room and at its start. indexer_take_in_pending takes it again now, indexer_take_in_skip ends it unwritten.", Input: map[string]any{"type": "object", "properties": map[string]any{}}, Run: func(map[string]any) (any, error) { h, err := ReadHeld(context.Background(), held) diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source.go b/modules/mesh-delivery-indexer/internal/index/takein_source.go index 9aef83fc..da51cd4d 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein_source.go +++ b/modules/mesh-delivery-indexer/internal/index/takein_source.go @@ -30,16 +30,19 @@ func TakenID(branch string, source int64) string { // mover (§768). A revision already kept under that name (a // restart before the acknowledgement) is said again from its kept movers, recomputing nothing (H-A-92). The first watched // branch is taken in on the indexer's own view; another on its newest revision's, computed and kept only as that revision. -// An error leaves the source-moved held, said once per source as take-in.pending with branch, source and why (§744): +// An error leaves the source-moved held, written into the state held behind its branch's earlier ones (ErrHeldUnwritten when +// it is not) and said once per source as take-in.pending with branch, source and why (§744, H-A-118): // ErrObjectsFull waits for the indexer's own room, any other is the failed read or write it says; ErrOldForm is never taken in // and said as no take-in. func (ix *Indexer) TakeInSource(ctx context.Context, e meshdelivery.SourceMovedEvent) (meshcatalog.TakeInTaken, error) { taken, err := ix.takeInSource(ctx, e) switch { case err == nil: - ix.unhold(ctx, e.Branch) + ix.unhold(ctx, e.Branch, e.Source) case !errors.Is(err, ErrOldForm): - ix.hold(ctx, e, err.Error()) + if herr := ix.hold(ctx, e, err.Error()); herr != nil { + return taken, fmt.Errorf("%w; %w", err, herr) + } } return taken, err } diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go index e1780a73..b96aaaae 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go +++ b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go @@ -182,12 +182,30 @@ func TestATakeInOnAFullStoreIsHeldAndSaidOnce(t *testing.T) { if held, err := ReadHeld(ctx, ix.Held); err != nil || len(held) != 1 || held[0].Branch != "main" || held[0].Source != 3 || held[0].Why == "" { t.Errorf("held = %+v (%v), want main's source revision 3 and why", held, err) } + // H-A-118: the held value carries the event whole, and a later source of the branch is held behind it, in order. + next := moved + next.Source, next.Call = 4, "1123456789abcdef" + if behind, err := ix.HoldBehind(ctx, next); err != nil || !behind { + t.Fatalf("the later source held behind: %v (%v)", behind, err) + } + if held, _ := ReadHeld(ctx, ix.Held); len(held) != 2 || held[0].Source != 3 || held[1].Source != 4 || held[0].Call != moved.Call || + len(held[0].Heads) != 1 || held[0].Heads[0] != moved.Heads[0] || !reflect.DeepEqual(held[0].Event(), moved) { + t.Fatalf("held = %+v, want 3 then 4, each with its event whole", held) + } objects.capacity = 1 << 22 if ok, err := ix.RoomAgain(ctx); err != nil || !ok || rooms != 1 { t.Fatalf("room said %v %d times (%v), want once the store has room", ok, rooms, err) } - if taken, err := ix.TakeInSource(ctx, moved); err != nil || taken.Revision != 3 || len(*said) != 2 { - t.Errorf("taken again on room: %+v (%v), said %d events", taken, err, len(*said)) + // H-A-114: on room (and at start) the held ones are taken in again from the state held alone, in their order. + ix.TakeHeldIn(ctx) + var taken []int64 + for _, e := range *said { + if !e.Skipped { + taken = append(taken, e.Source) + } + } + if !reflect.DeepEqual(taken, []int64{3, 4}) { + t.Errorf("taken again on room: %v of %+v, want 3 then 4", taken, *said) } if held, _ := ReadHeld(ctx, ix.Held); len(held) != 0 { t.Errorf("held = %+v after the take-in, want none", held)