I1: one durable consumer per watched branch hears source-moved
The indexer binds one durable consumer per watched branch on the stream of the gate's source-moved, named <node>_<indexer>_<escaped branch> through mesh-sdk's Account.NamedEventConsumer; it takes each message in with TakeInSource and acknowledges it once its revision and take-in.taken are written, and leaves it unacknowledged when the take-in fails. The watched branches are the registrations'; a newly watched branch's consumer is bound on the indexer's own registration-changed, which the manifest now consumes. A source-moved of the old form (no call) is refused for good with a journal line. Builds: design 47 H §743 (one durable per watched branch), §765 (the old form refused); H-A-30, H-A-90 Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
This commit is contained in:
@@ -594,6 +594,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
})
|
||||
hearNamed(ctx, js, cred, notHeard, named, logger.Printf)
|
||||
hearRuns(ctx, js, cred, notHeard, ix, logger.Printf)
|
||||
// The gate's source revisions, on one durable per watched branch (design 47 H §743, H-A-90): bound now for every branch the
|
||||
// registrations watch, and for a newly watched one on the indexer's own registration-changed.
|
||||
sources := &sourceMoved{ctx: ctx, ix: ix, branches: watchedBranches(registry), bind: bindBranch(js, cred, logger.Printf), logf: logger.Printf}
|
||||
defer sources.stop()
|
||||
hear(meshcatalog.StreamRegistrationChanged, "a newly watched branch's source revisions are not taken in until the indexer starts again", func(m jetstream.Msg) {
|
||||
_ = m.Ack()
|
||||
sources.refresh()
|
||||
})
|
||||
sources.refresh()
|
||||
if stopped != nil {
|
||||
return stopped
|
||||
}
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// sourceMoved is the indexer's hearing of the gate's source-moved (design 47 H §743): one durable consumer per watched branch on
|
||||
// the event's stream, named <node>_<indexer>_<branch>, the branch escaped as a key's name is (H-A-90), each delivering one message
|
||||
// at a time (the grants part makes it so, H-A-30), acknowledged once its catalog revision and take-in.taken are written. The
|
||||
// watched branches are the registrations'; a newly watched branch's consumer is bound on the indexer's own
|
||||
// registration-changed, not only at its start.
|
||||
type sourceMoved struct {
|
||||
ctx context.Context
|
||||
ix *index.Indexer
|
||||
branches func(context.Context) ([]string, error) // the watched branches the registrations name
|
||||
bind func(ctx context.Context, branch string, each func(jetstream.Msg)) (stop func(), err error)
|
||||
logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
bound map[string]func() // by branch: the consumer's stop
|
||||
}
|
||||
|
||||
// watchedBranches are the branches every registration watches, each once, sorted.
|
||||
func watchedBranches(r *index.Registry) func(context.Context) ([]string, error) {
|
||||
return func(ctx context.Context) ([]string, error) {
|
||||
a, err := r.RegistrationsAnswer(ctx, meshcatalog.RegistrationsInput{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for _, reg := range a.Registrations {
|
||||
for _, b := range reg.Branches {
|
||||
if !slices.Contains(out, b) {
|
||||
out = append(out, b)
|
||||
}
|
||||
}
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
|
||||
// bindBranch binds the indexer's durable of one watched branch on the stream of source-moved through mesh-sdk.
|
||||
func bindBranch(js jetstream.JetStream, cred process.Account, logf func(string, ...any)) func(context.Context, string, func(jetstream.Msg)) (func(), error) {
|
||||
return func(ctx context.Context, branch string, each func(jetstream.Msg)) (func(), error) {
|
||||
cons, err := cred.NamedEventConsumer(ctx, js, meshdelivery.StreamSourceMoved, keys.EscapeName(branch))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hc, err := process.HearEvents(cons, each, logf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return hc.Stop, nil
|
||||
}
|
||||
}
|
||||
|
||||
// refresh binds the consumer of every watched branch not bound yet: at the start and on each registration-changed. A consumer
|
||||
// not there yet is said and bound at the next registration-changed or start.
|
||||
func (s *sourceMoved) refresh() {
|
||||
branches, err := s.branches(s.ctx)
|
||||
if err != nil {
|
||||
s.logf("source-moved: the watched branches are not read, so no new branch's consumer is bound: %v", err)
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.bound == nil {
|
||||
s.bound = map[string]func(){}
|
||||
}
|
||||
for _, b := range branches {
|
||||
if _, ok := s.bound[b]; ok {
|
||||
continue
|
||||
}
|
||||
branch := b
|
||||
stop, err := s.bind(s.ctx, branch, func(m jetstream.Msg) { s.heard(branch, m) })
|
||||
if err != nil {
|
||||
s.logf("source-moved: the consumer of the branch %s is not bound, so its source revisions are not taken in until the next registration-changed or start: %v", branch, err)
|
||||
continue
|
||||
}
|
||||
s.bound[branch] = stop
|
||||
s.logf("source-moved: the consumer of the branch %s is bound", branch)
|
||||
}
|
||||
}
|
||||
|
||||
// heard takes one source-moved of a branch in and acknowledges it once taken. One of the old form (no field its form requires,
|
||||
// §765) is refused for good with a journal line naming it; any other failure leaves it unacknowledged, held.
|
||||
func (s *sourceMoved) heard(branch string, m jetstream.Msg) {
|
||||
var e meshdelivery.SourceMovedEvent
|
||||
if err := json.Unmarshal(m.Data(), &e); err != nil {
|
||||
s.logf("source-moved of %s on %s is refused, not a source-moved: %v", branch, m.Subject(), err)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
if _, err := s.ix.TakeInSource(s.ctx, e); err != nil {
|
||||
if errors.Is(err, index.ErrOldForm) {
|
||||
s.logf("source-moved of %s on %s is refused, the old form: %v", branch, m.Subject(), err)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
s.logf("source-moved: source revision %d of %s is held, not taken in: %v", e.Source, branch, err)
|
||||
return
|
||||
}
|
||||
if err := m.Ack(); err != nil {
|
||||
s.logf("source-moved: source revision %d of %s is taken in and its acknowledgement failed: %v", e.Source, branch, err)
|
||||
}
|
||||
}
|
||||
|
||||
// stop stops every consumer bound.
|
||||
func (s *sourceMoved) stop() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for _, stop := range s.bound {
|
||||
stop()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// msgOf is a fakeMsg as the bus gives it: what the hearing reads and does.
|
||||
type msgOf struct {
|
||||
jetstream.Msg
|
||||
f *fakeMsg
|
||||
}
|
||||
|
||||
func (m msgOf) Subject() string { return m.f.Subject() }
|
||||
func (m msgOf) Data() []byte { return m.f.Data() }
|
||||
func (m msgOf) Ack() error { return m.f.Ack() }
|
||||
func (m msgOf) Term() error { return m.f.Term() }
|
||||
|
||||
// Design 47 H §743, H-A-90, §765: one consumer per watched branch, a newly watched one bound at the next registration-changed;
|
||||
// a source-moved of the old form (no call) is refused for good with a journal line naming it.
|
||||
func TestEachWatchedBranchHasItsConsumerAndTheOldFormIsRefused(t *testing.T) {
|
||||
watched := []string{"main"}
|
||||
var bound []string
|
||||
var lines []string
|
||||
handlers := map[string]func(jetstream.Msg){}
|
||||
s := &sourceMoved{ctx: context.Background(), ix: &index.Indexer{},
|
||||
branches: func(context.Context) ([]string, error) { return watched, nil },
|
||||
bind: func(_ context.Context, branch string, each func(jetstream.Msg)) (func(), error) {
|
||||
bound, handlers[branch] = append(bound, branch), each
|
||||
return func() {}, nil
|
||||
},
|
||||
logf: func(f string, a ...any) { lines = append(lines, fmt.Sprintf(f, a...)) }}
|
||||
s.refresh()
|
||||
watched = []string{"main", "nox-mesh"}
|
||||
s.refresh() // the registration-changed of a newly watched branch
|
||||
if !reflect.DeepEqual(bound, []string{"main", "nox-mesh"}) {
|
||||
t.Fatalf("bound %v, want each watched branch once", bound)
|
||||
}
|
||||
old := &fakeMsg{subject: "mesh.seat.mesh-delivery.event.source-moved.main.4", data: []byte(`{"v":1,"branch":"main","source":4,"heads":[]}`)}
|
||||
handlers["main"](msgOf{f: old})
|
||||
if old.termd != 1 || old.acked != 0 || !strings.Contains(strings.Join(lines, "\n"), "the old form") {
|
||||
t.Errorf("the old form was terminated %d and acknowledged %d times, journal %q; want refused with a line naming it", old.termd, old.acked, lines)
|
||||
}
|
||||
}
|
||||
@@ -169,7 +169,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control
|
||||
// internal/broker/state.go:62-93).
|
||||
// The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*).
|
||||
if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked",
|
||||
if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-conditions.findings-asked",
|
||||
"mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.run-started", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
||||
}
|
||||
|
||||
@@ -81,6 +81,7 @@
|
||||
"node-build-go.step-ended",
|
||||
"mesh-wants.wait-ended",
|
||||
"mesh-catalog.catalog-revision",
|
||||
"mesh-catalog.registration-changed",
|
||||
"mesh-delivery.source-moved",
|
||||
"mesh-wants.wanted",
|
||||
"mesh-pipeline-runner.run-started",
|
||||
|
||||
Reference in New Issue
Block a user