The indexer makes its own per-branch durables and drops the earlier model's revisions at a take-in (design 47 H, H-A-134, H-A-135) #265
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
@@ -79,10 +80,19 @@ func sourceVia(call toolCaller) func(context.Context, string) (meshdelivery.Sour
|
||||
}
|
||||
}
|
||||
|
||||
// bindBranch binds the indexer's durable of one watched branch on the stream of source-moved through mesh-sdk.
|
||||
// bindBranch binds the indexer's durable of one watched branch on the stream of source-moved through mesh-sdk. One the bus does
|
||||
// not hold the indexer makes itself, as its own kept consumer, and then binds (design 47 H §743, H-A-134: the grants part
|
||||
// composes none until H4b); one the bus holds is bound as it stands, never made again.
|
||||
func bindBranch(js jetstream.JetStream, cred process.Account, logf func(string, ...any)) func(context.Context, string, func(jetstream.Msg)) (func(), error) {
|
||||
return func(ctx context.Context, branch string, each func(jetstream.Msg)) (func(), error) {
|
||||
cons, err := cred.NamedEventConsumer(ctx, js, meshdelivery.StreamSourceMoved, keys.EscapeName(branch))
|
||||
if errors.Is(err, jetstream.ErrConsumerNotFound) {
|
||||
cfg := branchConsumerConfig(cred, branch)
|
||||
if cons, err = js.CreateConsumer(ctx, meshdelivery.StreamSourceMoved, cfg); err != nil {
|
||||
return nil, fmt.Errorf("the consumer %s on %s is not there and is not made: %w", cfg.Durable, meshdelivery.StreamSourceMoved, err)
|
||||
}
|
||||
logf("source-moved: the consumer %s on %s was not there: the indexer made it (H-A-134)", cfg.Durable, meshdelivery.StreamSourceMoved)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -94,6 +104,28 @@ func bindBranch(js jetstream.JetStream, cred process.Account, logf func(string,
|
||||
}
|
||||
}
|
||||
|
||||
// branchConsumerConfig is the indexer's durable of one watched branch as the mesh gives a module's seat-event consumer
|
||||
// (mesh-sdk's SeatEventConsumerConfig), named <node>_<indexer>_<branch> as NamedEventConsumer binds it, filtered to the
|
||||
// branch's source-moved and delivering one message at a time, with the ack wait of 30 s, the at most 5 deliveries and the
|
||||
// start from now the grants part gives a module's consumer on a seat event's stream kept by age (mesh-grants buscompose
|
||||
// heardConsumer): the branch's current source revision is read at the start through the gate's source (H-A-129, H-A-130), so
|
||||
// its past source-moved are never taken in again (design 47 H §743, H-A-90, H-A-134).
|
||||
func branchConsumerConfig(cred process.Account, branch string) jetstream.ConsumerConfig {
|
||||
esc := keys.EscapeName(branch)
|
||||
cfg := cred.SeatEventConsumerConfig([]string{meshdelivery.SubjectSourceMoved(esc, "*")}, false)
|
||||
cfg.Durable = cred.EventConsumerName() + "_" + esc
|
||||
cfg.MaxAckPending = 1
|
||||
cfg.AckWait, cfg.MaxDeliver = branchAckWait, branchMaxDeliver
|
||||
cfg.DeliverPolicy = jetstream.DeliverNewPolicy
|
||||
return cfg
|
||||
}
|
||||
|
||||
// The ack wait and the deliveries of a branch's durable, as the grants part gives a module's consumers (H-A-134, H-A-133).
|
||||
const (
|
||||
branchAckWait = 30 * time.Second
|
||||
branchMaxDeliver = 5
|
||||
)
|
||||
|
||||
// refresh binds the consumer of every watched branch not bound yet: at the start and on each registration-changed. A consumer
|
||||
// not there yet is said and bound at the next registration-changed or start. The start's refresh reads every watched branch's
|
||||
// source revision, a later one each branch whose consumer could not be bound (H-A-129), after the binds.
|
||||
|
||||
@@ -8,10 +8,13 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
@@ -152,3 +155,82 @@ func TestAStartWithNoConsumerTakesTheGatesSourceRevisionIn(t *testing.T) {
|
||||
t.Errorf("asked the gate for %v and said %v; want main asked at each and taken in once", asked, taken)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeBranchJS is the bus as bindBranch uses it: the source-moved stream's consumers by name, and the consumers made.
|
||||
type fakeBranchJS struct {
|
||||
jetstream.JetStream
|
||||
held map[string]*fakeBranchConsumer
|
||||
made []jetstream.ConsumerConfig
|
||||
}
|
||||
|
||||
func (f *fakeBranchJS) Consumer(_ context.Context, stream, name string) (jetstream.Consumer, error) {
|
||||
if c, ok := f.held[stream+"/"+name]; ok {
|
||||
return c, nil
|
||||
}
|
||||
return nil, jetstream.ErrConsumerNotFound
|
||||
}
|
||||
|
||||
func (f *fakeBranchJS) CreateConsumer(_ context.Context, stream string, cfg jetstream.ConsumerConfig) (jetstream.Consumer, error) {
|
||||
f.made = append(f.made, cfg)
|
||||
c := &fakeBranchConsumer{}
|
||||
f.held[stream+"/"+cfg.Durable] = c
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// fakeBranchConsumer hands one source-moved to the hearing as soon as it is consumed.
|
||||
type fakeBranchConsumer struct {
|
||||
jetstream.Consumer
|
||||
heard int
|
||||
}
|
||||
|
||||
type fakeConsumeContext struct{ jetstream.ConsumeContext }
|
||||
|
||||
func (fakeConsumeContext) Stop() {}
|
||||
|
||||
func (c *fakeBranchConsumer) Consume(handle jetstream.MessageHandler, _ ...jetstream.PullConsumeOpt) (jetstream.ConsumeContext, error) {
|
||||
c.heard++
|
||||
handle(msgOf{f: &fakeMsg{subject: "mesh.seat.mesh-delivery.event.source-moved.main.2"}})
|
||||
return fakeConsumeContext{}, nil
|
||||
}
|
||||
|
||||
// Design 47 H §743, H-A-134: a branch's durable the bus does not hold the indexer makes, as a module's seat-event consumer is
|
||||
// made (SeatEventConsumerConfig), named <node>_<indexer>_<branch>, filtered to the branch, one message at a time, and then
|
||||
// hears on it; one the bus holds is bound, never made again.
|
||||
func TestABranchDurableNotThereIsMadeAndHeard(t *testing.T) {
|
||||
cred := process.Account{Node: "n1", Module: "mesh-delivery-indexer"}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
held bool
|
||||
}{{"not there: made", false}, {"there: bound", true}} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
js := &fakeBranchJS{held: map[string]*fakeBranchConsumer{}}
|
||||
name := "n1_mesh-delivery-indexer_" + keys.EscapeName("main")
|
||||
var existing *fakeBranchConsumer
|
||||
if tc.held {
|
||||
existing = &fakeBranchConsumer{}
|
||||
js.held[meshdelivery.StreamSourceMoved+"/"+name] = existing
|
||||
}
|
||||
var heard []string
|
||||
stop, err := bindBranch(js, cred, func(string, ...any) {})(context.Background(), "main", func(m jetstream.Msg) { heard = append(heard, m.Subject()) })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stop()
|
||||
if len(heard) != 1 {
|
||||
t.Fatalf("heard %v, want the one source-moved the consumer hands", heard)
|
||||
}
|
||||
if tc.held {
|
||||
if len(js.made) != 0 || existing.heard != 1 {
|
||||
t.Fatalf("made %v and heard the held one %d times; want it bound, not made again", js.made, existing.heard)
|
||||
}
|
||||
return
|
||||
}
|
||||
want := cred.SeatEventConsumerConfig([]string{meshdelivery.SubjectSourceMoved(keys.EscapeName("main"), "*")}, false)
|
||||
want.Durable, want.MaxAckPending = name, 1
|
||||
want.AckWait, want.MaxDeliver, want.DeliverPolicy = 30*time.Second, 5, jetstream.DeliverNewPolicy
|
||||
if len(js.made) != 1 || !reflect.DeepEqual(js.made[0], want) {
|
||||
t.Fatalf("made %+v, want once %+v", js.made, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -337,6 +337,11 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int
|
||||
} else if len(removed) > 0 {
|
||||
ix.logf("pins: %d revisions removed (%s), no record pins them and none is current", len(removed), strings.Join(removed, ", "))
|
||||
}
|
||||
if dropped, err := ix.dropEarlierFormsLocked(ctx); err != nil {
|
||||
ix.logf("catalog graph: the revisions of the earlier forms (trunk.*, branch.*) are not dropped: %v", err)
|
||||
} else if len(dropped) > 0 {
|
||||
ix.logf("catalog graph: %d revisions of the earlier forms dropped (%s), no record pins them; each branch's newest revision is its current one", len(dropped), strings.Join(dropped, ", "))
|
||||
}
|
||||
if _, err := ix.removeUnlisted(ctx); err != nil {
|
||||
ix.logf("catalog graph: the entries nothing lists or names any more are not removed: %v", err) // at the first revision after none does (§675)
|
||||
}
|
||||
|
||||
@@ -253,25 +253,10 @@ func (ix *Indexer) judgePinsLocked(ctx context.Context) ([]string, error) {
|
||||
return nil, err
|
||||
}
|
||||
current := CurrentRevisions(names)
|
||||
pinned := map[string]bool{}
|
||||
pinKeys, err := ix.Pins.Keys(ctx)
|
||||
pinKeys, pinned, err := ix.pinnedObjects(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, k := range pinKeys {
|
||||
object, ok := pinObject(k)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
v, _, found, err := ix.Pins.Get(ctx, k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var pv PinValue
|
||||
if found && json.Unmarshal(v, &pv) == nil && len(pv.Records) > 0 {
|
||||
pinned[object] = true
|
||||
}
|
||||
}
|
||||
var removed []string
|
||||
for _, n := range names {
|
||||
if _, _, ok := RevisionGroup(n); !ok || pinned[n] {
|
||||
@@ -301,6 +286,72 @@ func (ix *Indexer) judgePinsLocked(ctx context.Context) ([]string, error) {
|
||||
return removed, nil
|
||||
}
|
||||
|
||||
// pinnedObjects are the state pins' keys and the revision objects a record pins.
|
||||
func (ix *Indexer) pinnedObjects(ctx context.Context) ([]string, map[string]bool, error) {
|
||||
pinned := map[string]bool{}
|
||||
pinKeys, err := ix.Pins.Keys(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
for _, k := range pinKeys {
|
||||
object, ok := pinObject(k)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
v, _, found, err := ix.Pins.Get(ctx, k)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var pv PinValue
|
||||
if found && json.Unmarshal(v, &pv) == nil && len(pv.Records) > 0 {
|
||||
pinned[object] = true
|
||||
}
|
||||
}
|
||||
return pinKeys, pinned, nil
|
||||
}
|
||||
|
||||
// earlierForm says whether a revision object is of the earlier model's forms, trunk.<n> or branch.<branch>.<n>, beside the
|
||||
// branch-keyed <branch>.<n> of design 47 H §743 (H-A-135).
|
||||
func earlierForm(name string) bool {
|
||||
if _, _, ok := RevisionGroup(name); !ok {
|
||||
return false
|
||||
}
|
||||
return strings.HasPrefix(name, "trunk.") || strings.HasPrefix(name, "branch.")
|
||||
}
|
||||
|
||||
// dropEarlierFormsLocked drops every revision object of the earlier forms (earlierForm) no record pins, at a branch's take-in
|
||||
// (design 47 H §743, H-A-135): the branch's newest revision is its current one, and an earlier trunk revision kept beside it
|
||||
// would be the newest trunk revision a reader hears. It drops none while the pins do not count yet; the caller holds keepMu.
|
||||
func (ix *Indexer) dropEarlierFormsLocked(ctx context.Context) ([]string, error) {
|
||||
if ix.PinsPending != nil {
|
||||
if pending, err := ix.PinsPending(ctx); err != nil || pending != 0 {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
names, err := ix.Revisions.Names(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pinned := map[string]bool{}
|
||||
if ix.Pins != nil {
|
||||
if _, pinned, err = ix.pinnedObjects(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var dropped []string
|
||||
for _, n := range names {
|
||||
if !earlierForm(n) || pinned[n] {
|
||||
continue
|
||||
}
|
||||
if err := ix.Revisions.Delete(ctx, n); err != nil {
|
||||
return dropped, fmt.Errorf("%s: %w", n, err)
|
||||
}
|
||||
dropped = append(dropped, n)
|
||||
}
|
||||
sort.Strings(dropped)
|
||||
return dropped, nil
|
||||
}
|
||||
|
||||
// PinDrop drops one pin of the state pins by hand, by its key <branch>.<revision> (indexer_pin_drop, design 47 H §765, H-A-57):
|
||||
// every record's, for a pin that names no running pipeline; the finding pin-missing of the key is cleared, and the removal
|
||||
// rule then applies.
|
||||
|
||||
@@ -61,7 +61,8 @@ type RevisionSummary struct {
|
||||
Heads string `json:"heads,omitempty"`
|
||||
At *time.Time `json:"at,omitempty"`
|
||||
Entries int `json:"entries"`
|
||||
// Current says it is the trunk's newest revision, which is kept whatever pins it.
|
||||
// Current says it is the newest revision of its watched branch (CurrentRevisions, §579; H §743, H-A-135: no longer the
|
||||
// newest trunk-kind revision over every branch, and never one of the earlier forms), which is kept whatever pins it.
|
||||
Current bool `json:"current,omitempty"`
|
||||
}
|
||||
|
||||
@@ -84,7 +85,6 @@ func ListRevisions(ctx context.Context, o Objects, candidates bool) (RevisionsAn
|
||||
return out, fmt.Errorf("the objects %s cannot be listed: %w", CatalogRevisionsObjects, err)
|
||||
}
|
||||
listed := map[string]bool{}
|
||||
var newest int64
|
||||
entries := []string{}
|
||||
for _, n := range names {
|
||||
if strings.HasPrefix(n, "entry.") {
|
||||
@@ -103,17 +103,16 @@ func ListRevisions(ctx context.Context, o Objects, candidates bool) (RevisionsAn
|
||||
for _, ref := range rev.Entries {
|
||||
listed[EntryObject(ref.Digest)] = true
|
||||
}
|
||||
if rev.Kind == "trunk" && rev.Revision > newest {
|
||||
newest = rev.Revision
|
||||
}
|
||||
if candidates && rev.Kind != "candidate" {
|
||||
continue
|
||||
}
|
||||
out.Revisions = append(out.Revisions, RevisionSummary{Name: n, Kind: rev.Kind, Revision: rev.Revision, Source: rev.Source,
|
||||
Group: rev.Group, Heads: rev.Heads, At: rev.At, Entries: len(rev.Entries)})
|
||||
}
|
||||
current := CurrentRevisions(names)
|
||||
for i := range out.Revisions {
|
||||
out.Revisions[i].Current = out.Revisions[i].Kind == "trunk" && out.Revisions[i].Revision == newest
|
||||
_, cur := current[out.Revisions[i].Name]
|
||||
out.Revisions[i].Current = cur && out.Revisions[i].Kind == "trunk" && !earlierForm(out.Revisions[i].Name)
|
||||
}
|
||||
slices.SortFunc(out.Revisions, func(a, b RevisionSummary) int { return strings.Compare(a.Name, b.Name) })
|
||||
out.Entries = len(entries)
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
@@ -121,6 +122,39 @@ func TestAFirstSourceRevisionIsTakenInWithNoMover(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H §743, H-A-135: at a branch's take-in the earlier model's revisions (trunk.<n>, branch.<branch>.<n>) no record
|
||||
// pins are dropped, one a record pins stays, and the branch's newest revision is the current one, no longer the newest trunk.
|
||||
func TestATakeInDropsTheEarlierFormsNoRecordPins(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix, objects, _, f := takeInIndexer(t)
|
||||
for _, n := range []string{"trunk.527", "trunk.528", "branch.master.1"} {
|
||||
_, rev, _ := RevisionGroup(n)
|
||||
raw, _ := json.Marshal(meshcatalog.CatalogRevision{Revision: rev, Kind: "trunk", Source: 1000 + rev})
|
||||
if err := objects.Put(ctx, n, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := ix.pin(ctx, "trunk.527", "pipeline.old"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
start := meshdelivery.SourceMovedEvent{V: 1, Branch: "main", Source: 1, Call: "seed.operator.20261120", Start: true,
|
||||
Heads: []meshdelivery.SourceMovedEventHeadsItem{{Repository: f.Repository, Commit: f.Merge}}}
|
||||
if _, err := ix.TakeInSource(ctx, start); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, err := ListRevisions(ctx, objects, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got []string
|
||||
for _, r := range a.Revisions {
|
||||
got = append(got, fmt.Sprintf("%s current=%t", r.Name, r.Current))
|
||||
}
|
||||
if want := []string{"main.1 current=true", "trunk.527 current=false"}; !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("kept %v, want %v: the unpinned earlier forms dropped, the pinned one kept, main.1 current", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H §769: the take-in of a source revision moving the catalogue repository reads `predefined` from that commit's
|
||||
// trees, and one changing a predefined key rewrites it.
|
||||
func TestATakeInOfTheCatalogsCommitRewritesPredefined(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user