diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index c9843751..0f64031f 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -594,6 +594,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro }) hearNamed(ctx, js, cred, notHeard, named, logger.Printf) hearRuns(ctx, js, cred, notHeard, ix, logger.Printf) + // The gate's source revisions, on one durable per watched branch (design 47 H §743, H-A-90): bound now for every branch the + // registrations watch, and for a newly watched one on the indexer's own registration-changed. + sources := &sourceMoved{ctx: ctx, ix: ix, branches: watchedBranches(registry), bind: bindBranch(js, cred, logger.Printf), logf: logger.Printf} + defer sources.stop() + hear(meshcatalog.StreamRegistrationChanged, "a newly watched branch's source revisions are not taken in until the indexer starts again", func(m jetstream.Msg) { + _ = m.Ack() + sources.refresh() + }) + sources.refresh() if stopped != nil { return stopped } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go new file mode 100644 index 00000000..5d4417c4 --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go @@ -0,0 +1,127 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "slices" + "sync" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go/jetstream" + "mesh-delivery-indexer/internal/index" +) + +// sourceMoved is the indexer's hearing of the gate's source-moved (design 47 H §743): one durable consumer per watched branch on +// the event's stream, named __, the branch escaped as a key's name is (H-A-90), each delivering one message +// at a time (the grants part makes it so, H-A-30), acknowledged once its catalog revision and take-in.taken are written. The +// watched branches are the registrations'; a newly watched branch's consumer is bound on the indexer's own +// registration-changed, not only at its start. +type sourceMoved struct { + ctx context.Context + ix *index.Indexer + branches func(context.Context) ([]string, error) // the watched branches the registrations name + bind func(ctx context.Context, branch string, each func(jetstream.Msg)) (stop func(), err error) + logf func(string, ...any) + + mu sync.Mutex + bound map[string]func() // by branch: the consumer's stop +} + +// watchedBranches are the branches every registration watches, each once, sorted. +func watchedBranches(r *index.Registry) func(context.Context) ([]string, error) { + return func(ctx context.Context) ([]string, error) { + a, err := r.RegistrationsAnswer(ctx, meshcatalog.RegistrationsInput{}) + if err != nil { + return nil, err + } + var out []string + for _, reg := range a.Registrations { + for _, b := range reg.Branches { + if !slices.Contains(out, b) { + out = append(out, b) + } + } + } + slices.Sort(out) + return out, nil + } +} + +// bindBranch binds the indexer's durable of one watched branch on the stream of source-moved through mesh-sdk. +func bindBranch(js jetstream.JetStream, cred process.Account, logf func(string, ...any)) func(context.Context, string, func(jetstream.Msg)) (func(), error) { + return func(ctx context.Context, branch string, each func(jetstream.Msg)) (func(), error) { + cons, err := cred.NamedEventConsumer(ctx, js, meshdelivery.StreamSourceMoved, keys.EscapeName(branch)) + if err != nil { + return nil, err + } + hc, err := process.HearEvents(cons, each, logf) + if err != nil { + return nil, err + } + return hc.Stop, nil + } +} + +// refresh binds the consumer of every watched branch not bound yet: at the start and on each registration-changed. A consumer +// not there yet is said and bound at the next registration-changed or start. +func (s *sourceMoved) refresh() { + branches, err := s.branches(s.ctx) + if err != nil { + s.logf("source-moved: the watched branches are not read, so no new branch's consumer is bound: %v", err) + return + } + s.mu.Lock() + defer s.mu.Unlock() + if s.bound == nil { + s.bound = map[string]func(){} + } + for _, b := range branches { + if _, ok := s.bound[b]; ok { + continue + } + branch := b + stop, err := s.bind(s.ctx, branch, func(m jetstream.Msg) { s.heard(branch, m) }) + if err != nil { + s.logf("source-moved: the consumer of the branch %s is not bound, so its source revisions are not taken in until the next registration-changed or start: %v", branch, err) + continue + } + s.bound[branch] = stop + s.logf("source-moved: the consumer of the branch %s is bound", branch) + } +} + +// heard takes one source-moved of a branch in and acknowledges it once taken. One of the old form (no field its form requires, +// §765) is refused for good with a journal line naming it; any other failure leaves it unacknowledged, held. +func (s *sourceMoved) heard(branch string, m jetstream.Msg) { + var e meshdelivery.SourceMovedEvent + if err := json.Unmarshal(m.Data(), &e); err != nil { + s.logf("source-moved of %s on %s is refused, not a source-moved: %v", branch, m.Subject(), err) + _ = m.Term() + return + } + if _, err := s.ix.TakeInSource(s.ctx, e); err != nil { + if errors.Is(err, index.ErrOldForm) { + s.logf("source-moved of %s on %s is refused, the old form: %v", branch, m.Subject(), err) + _ = m.Term() + return + } + s.logf("source-moved: source revision %d of %s is held, not taken in: %v", e.Source, branch, err) + return + } + if err := m.Ack(); err != nil { + s.logf("source-moved: source revision %d of %s is taken in and its acknowledgement failed: %v", e.Source, branch, err) + } +} + +// stop stops every consumer bound. +func (s *sourceMoved) stop() { + s.mu.Lock() + defer s.mu.Unlock() + for _, stop := range s.bound { + stop() + } +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go new file mode 100644 index 00000000..60b3f724 --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go @@ -0,0 +1,51 @@ +package main + +import ( + "context" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/nats-io/nats.go/jetstream" + + "mesh-delivery-indexer/internal/index" +) + +// msgOf is a fakeMsg as the bus gives it: what the hearing reads and does. +type msgOf struct { + jetstream.Msg + f *fakeMsg +} + +func (m msgOf) Subject() string { return m.f.Subject() } +func (m msgOf) Data() []byte { return m.f.Data() } +func (m msgOf) Ack() error { return m.f.Ack() } +func (m msgOf) Term() error { return m.f.Term() } + +// Design 47 H §743, H-A-90, §765: one consumer per watched branch, a newly watched one bound at the next registration-changed; +// a source-moved of the old form (no call) is refused for good with a journal line naming it. +func TestEachWatchedBranchHasItsConsumerAndTheOldFormIsRefused(t *testing.T) { + watched := []string{"main"} + var bound []string + var lines []string + handlers := map[string]func(jetstream.Msg){} + s := &sourceMoved{ctx: context.Background(), ix: &index.Indexer{}, + branches: func(context.Context) ([]string, error) { return watched, nil }, + bind: func(_ context.Context, branch string, each func(jetstream.Msg)) (func(), error) { + bound, handlers[branch] = append(bound, branch), each + return func() {}, nil + }, + logf: func(f string, a ...any) { lines = append(lines, fmt.Sprintf(f, a...)) }} + s.refresh() + watched = []string{"main", "nox-mesh"} + s.refresh() // the registration-changed of a newly watched branch + if !reflect.DeepEqual(bound, []string{"main", "nox-mesh"}) { + t.Fatalf("bound %v, want each watched branch once", bound) + } + old := &fakeMsg{subject: "mesh.seat.mesh-delivery.event.source-moved.main.4", data: []byte(`{"v":1,"branch":"main","source":4,"heads":[]}`)} + handlers["main"](msgOf{f: old}) + if old.termd != 1 || old.acked != 0 || !strings.Contains(strings.Join(lines, "\n"), "the old form") { + t.Errorf("the old form was terminated %d and acknowledged %d times, journal %q; want refused with a line naming it", old.termd, old.acked, lines) + } +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index b8a4bff7..fb318388 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -169,7 +169,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control // internal/broker/state.go:62-93). // The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*). - if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked", + if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-conditions.findings-asked", "mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.run-started", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 41b23137..a3e1cfee 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -81,6 +81,7 @@ "node-build-go.step-ended", "mesh-wants.wait-ended", "mesh-catalog.catalog-revision", + "mesh-catalog.registration-changed", "mesh-delivery.source-moved", "mesh-wants.wanted", "mesh-pipeline-runner.run-started",