From 9cddb5921aa939ce705ff29123bdfb9d96ec5dbc Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 10:39:45 +0100 Subject: [PATCH] indexer: take a watched branch's source revision in from the gate's source when its consumer cannot be bound MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At a clean state the grants part makes the per-branch consumers of source-moved only from the modules view, which the indexer answers only after its first take-in, so nothing ever took the trunk in. At its start, and again at each failed bind, the indexer now reads the branch's current source revision through mesh-delivery.source and takes it in as a heard source-moved of it would, when its view keeps no revision of the branch at or above it: start true exactly when it keeps none, every repository the answer names a head. No timer; binding on registration-changed is unchanged. Builds: design 47 H §743, §768, H-A-129, H-A-130 Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../cmd/mesh-delivery-indexer/main.go | 2 +- .../cmd/mesh-delivery-indexer/sourcemoved.go | 112 ++++++++++++++++-- .../mesh-delivery-indexer/sourcemoved_test.go | 64 ++++++++++ .../internal/index/takein_source.go | 34 ++++++ .../internal/index/takein_source_test.go | 32 +++++ .../mesh-delivery-indexer/manifest_test.go | 2 +- modules/mesh-delivery-indexer/module.json | 1 + 7 files changed, 233 insertions(+), 14 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 408e6b1e..bcf46813 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -357,7 +357,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro registry.Pruned = ix.PruneBranchBuilds ix.Registry = registry // The gate's source revisions, on one durable per watched branch (design 47 H §743, H-A-90), bound below. - sources := &sourceMoved{ctx: ctx, ix: ix, branches: watchedBranches(registry), bind: bindBranch(js, cred, logger.Printf), logf: logger.Printf} + sources := &sourceMoved{ctx: ctx, ix: ix, branches: watchedBranches(registry), bind: bindBranch(js, cred, logger.Printf), source: sourceVia(calls.Call), logf: logger.Printf} defer sources.stop() named.Changed = ix.RemoveUnlisted // The first fill of the versions (design 47 G §372): at the start, and again at each catalog revision kept until it is made. diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go index 6b12486f..51cdc875 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go @@ -20,17 +20,22 @@ import ( // the event's stream, named __, the branch escaped as a key's name is (H-A-90), each delivering one message // at a time (the grants part makes it so, H-A-30), acknowledged once its catalog revision and take-in.taken are written. The // watched branches are the registrations'; a newly watched branch's consumer is bound on the indexer's own -// registration-changed, not only at its start. +// registration-changed, not only at its start. At its start, and again whenever a branch's consumer cannot be bound, the +// branch's current source revision is read through the seat mesh-delivery's verb source and taken in as a heard source-moved +// of it would be, when the view keeps no revision of the branch at or above it (H-A-129, H-A-130): at a clean state the +// consumers are made only once the indexer answers its modules view, which it does only after its first take-in. type sourceMoved struct { ctx context.Context ix *index.Indexer branches func(context.Context) ([]string, error) // the watched branches the registrations name bind func(ctx context.Context, branch string, each func(jetstream.Msg)) (stop func(), err error) + source func(ctx context.Context, branch string) (meshdelivery.SourceAnswer, error) // the gate's source revision of a branch logf func(string, ...any) - mu sync.Mutex - bound map[string]func() // by branch: the consumer's stop - again sync.Mutex // one take-in again at a time + mu sync.Mutex + bound map[string]func() // by branch: the consumer's stop + started bool // the start's refresh ran: every branch's source was read once + again sync.Mutex // one take-in again at a time } // watchedBranches are the branches every registration watches, each once, sorted. @@ -53,6 +58,27 @@ func watchedBranches(r *index.Registry) func(context.Context) ([]string, error) } } +// sourceVia reads a watched branch's current source revision through the seat mesh-delivery's verb source, as the +// indexer's own account (H-A-129): a call where an answer is needed. An answer naming why it could not be read is that +// failure. The branch is given by its name, as the gate reads it (it escapes the name for its key itself). +func sourceVia(call toolCaller) func(context.Context, string) (meshdelivery.SourceAnswer, error) { + verb := "seat:" + meshdelivery.Seat + "." + meshdelivery.VerbSource + return func(_ context.Context, branch string) (meshdelivery.SourceAnswer, error) { + var a meshdelivery.SourceAnswer + raw, err := call(verb, meshdelivery.SourceInput{Branch: branch}) + if err != nil { + return a, err + } + if err := json.Unmarshal(raw, &a); err != nil { + return a, fmt.Errorf("the answer of %s is not its seat's answer: %w", verb, err) + } + if a.Error != "" { + return a, fmt.Errorf("%s: %s", verb, a.Error) + } + return a, nil + } +} + // bindBranch binds the indexer's durable of one watched branch on the stream of source-moved through mesh-sdk. func bindBranch(js jetstream.JetStream, cred process.Account, logf func(string, ...any)) func(context.Context, string, func(jetstream.Msg)) (func(), error) { return func(ctx context.Context, branch string, each func(jetstream.Msg)) (func(), error) { @@ -69,30 +95,84 @@ func bindBranch(js jetstream.JetStream, cred process.Account, logf func(string, } // refresh binds the consumer of every watched branch not bound yet: at the start and on each registration-changed. A consumer -// not there yet is said and bound at the next registration-changed or start. +// not there yet is said and bound at the next registration-changed or start. The start's refresh reads every watched branch's +// source revision, a later one each branch whose consumer could not be bound (H-A-129), after the binds. func (s *sourceMoved) refresh() { branches, err := s.branches(s.ctx) if err != nil { s.logf("source-moved: the watched branches are not read, so no new branch's consumer is bound: %v", err) return } + for _, branch := range s.bindAll(branches) { + s.readSource(branch) + } +} + +// bindAll binds the branches not bound yet and answers those whose source revision is to be read now: at the start every +// watched branch, later each one whose consumer could not be bound. +func (s *sourceMoved) bindAll(branches []string) (read []string) { s.mu.Lock() defer s.mu.Unlock() if s.bound == nil { s.bound = map[string]func(){} } + start := !s.started + s.started = true for _, b := range branches { if _, ok := s.bound[b]; ok { + if start { + read = append(read, b) + } continue } branch := b stop, err := s.bind(s.ctx, branch, func(m jetstream.Msg) { s.heard(branch, m) }) if err != nil { - s.logf("source-moved: the consumer of the branch %s is not bound, so its source revisions are not taken in until the next registration-changed or start: %v", branch, err) + s.logf("source-moved: the consumer of the branch %s is not bound, so its later source revisions are not heard until the next registration-changed or start; its current one is read through the gate's source: %v", branch, err) + read = append(read, branch) continue } s.bound[branch] = stop s.logf("source-moved: the consumer of the branch %s is bound", branch) + if start { + read = append(read, branch) + } + } + return read +} + +// readSource reads a branch's current source revision through the gate's verb source and takes it in as a heard +// source-moved of that revision would be (takeIn), when the view keeps no revision of the branch at or above it (H-A-129, +// H-A-130: index.SourceToTake). +func (s *sourceMoved) readSource(branch string) { + if s.source == nil { + return + } + a, err := s.source(s.ctx, branch) + if err != nil { + s.logf("source-moved: the gate's source revision of %s is not read, so it is taken in only once its source-moved is heard: %v", branch, err) + return + } + if a.Revision <= 0 { + s.logf("source-moved: the gate keeps no source revision of %s yet", branch) + return + } + s.again.Lock() + defer s.again.Unlock() + e, take, err := s.ix.SourceToTake(s.ctx, branch, a) + if err != nil { + s.logf("source-moved: the catalog revisions of %s are not read, so the gate's source revision %d of it is not taken in now: %v", branch, a.Revision, err) + return + } + if !take { + return + } + s.logf("source-moved: source revision %d of %s is read through the gate's source and taken in now (start %t)", a.Revision, branch, e.Start) + switch err := s.takeIn(branch, e); { + case errors.Is(err, index.ErrOldForm): + s.logf("source-moved: the gate's source revision %d of %s is refused, the old form: %v", e.Source, branch, err) + case err != nil: + s.logf("source-moved: the gate's source revision %d of %s is not taken in now: %v", e.Source, branch, err) } } @@ -109,10 +189,7 @@ func (s *sourceMoved) heard(branch string, m jetstream.Msg) { } s.again.Lock() defer s.again.Unlock() - behind, err := s.ix.HoldBehind(s.ctx, e) - if !behind && err == nil { - _, err = s.ix.TakeInSource(s.ctx, e) - } + err := s.takeIn(branch, e) switch { case errors.Is(err, index.ErrOldForm): s.logf("source-moved of %s on %s is refused, the old form: %v", branch, m.Subject(), err) @@ -123,14 +200,25 @@ func (s *sourceMoved) heard(branch string, m jetstream.Msg) { return case err != nil: s.logf("source-moved: source revision %d of %s is held, not taken in: %v", e.Source, branch, err) - case behind: - s.logf("source-moved: source revision %d of %s is held behind the branch's earlier held ones", e.Source, branch) } if err := m.Ack(); err != nil { s.logf("source-moved: source revision %d of %s is taken in or held and its acknowledgement failed: %v", e.Source, branch, err) } } +// takeIn takes one source-moved of a branch in, or holds it behind the branch's earlier held ones, so its source revisions +// are taken in in their order; the caller holds s.again. A heard one and one read through the gate's source go this one way. +func (s *sourceMoved) takeIn(branch string, e meshdelivery.SourceMovedEvent) error { + behind, err := s.ix.HoldBehind(s.ctx, e) + if !behind && err == nil { + _, err = s.ix.TakeInSource(s.ctx, e) + } + if behind && err == nil { + s.logf("source-moved: source revision %d of %s is held behind the branch's earlier held ones", e.Source, branch) + } + return err +} + // takeAgain takes every held source revision in again from the state held, each branch in its order, on the indexer's own room // (H-A-50) and at its start (H-A-114). func (s *sourceMoved) takeAgain() { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go index 31e5fb6f..fcd97d79 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "reflect" "slices" @@ -10,6 +11,8 @@ import ( "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + "mesh-delivery-indexer/internal/index" ) @@ -88,3 +91,64 @@ func TestAHeldSourceMovedIsSkippedByHand(t *testing.T) { t.Errorf("held %+v after the skip, want source 5 alone", hs) } } + +// revisionObjects is the catalog revisions' object bucket in memory. +type revisionObjects map[string][]byte + +func (o revisionObjects) Put(_ context.Context, name string, data []byte) error { + o[name] = data + return nil +} +func (o revisionObjects) Get(_ context.Context, name string) ([]byte, error) { + if d, ok := o[name]; ok { + return d, nil + } + return nil, index.ErrNoObject +} +func (o revisionObjects) Delete(_ context.Context, name string) error { delete(o, name); return nil } +func (o revisionObjects) Names(context.Context) ([]string, error) { + var out []string + for n := range o { + out = append(out, n) + } + return out, nil +} +func (o revisionObjects) Fill(context.Context) (int64, int64, error) { return 0, 1 << 30, nil } + +// Design 47 H §743, §768, H-A-129: at a clean state no branch's consumer is there to bind; the start reads each watched +// branch's source revision through the gate's verb source and takes it in, and a failed bind at a registration-changed reads +// it again without taking it in twice. A start whose view keeps the revision reads it and takes nothing in. +func TestAStartWithNoConsumerTakesTheGatesSourceRevisionIn(t *testing.T) { + ctx := context.Background() + var taken []string + objects := revisionObjects{} + ix := &index.Indexer{Module: testIndexer, Revisions: objects, Modules: index.MemoryStore(), Graph: index.MemoryStore(), + Held: &memStore{vals: map[string][]byte{}}, Logf: func(string, ...any) {}, + SayTakeIn: func(_ context.Context, event, id string, _ any) error { + taken = append(taken, event+" "+id) + return nil + }} + var asked []string + newSources := func() *sourceMoved { + return &sourceMoved{ctx: ctx, ix: ix, + branches: func(context.Context) ([]string, error) { return []string{"main"}, nil }, + bind: func(context.Context, string, func(jetstream.Msg)) (func(), error) { + return nil, errors.New("nats: API error: code=404 err_code=10014 description=consumer not found") + }, + source: func(_ context.Context, branch string) (meshdelivery.SourceAnswer, error) { + asked = append(asked, branch) + return meshdelivery.SourceAnswer{Branch: branch, Revision: 1, Call: "0123456789abcdef"}, nil + }, + logf: func(string, ...any) {}} + } + s := newSources() + s.refresh() // the start + if _, ok := objects[index.BranchObject("main", 1)]; !ok || !reflect.DeepEqual(taken, []string{"take-in.taken " + index.TakenID("main", 1)}) { + t.Fatalf("the start kept %v and said %v; want main.1 taken in from the gate's source", objects, taken) + } + s.refresh() // a registration-changed, the consumer still not there + newSources().refresh() // a start whose view keeps the revision + if !reflect.DeepEqual(asked, []string{"main", "main", "main"}) || len(taken) != 1 { + t.Errorf("asked the gate for %v and said %v; want main asked at each and taken in once", asked, taken) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source.go b/modules/mesh-delivery-indexer/internal/index/takein_source.go index da51cd4d..0b4d4947 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein_source.go +++ b/modules/mesh-delivery-indexer/internal/index/takein_source.go @@ -142,3 +142,37 @@ func (ix *Indexer) sayTaken(ctx context.Context, t meshcatalog.TakeInTaken) erro func PendingID(branch string, source int64) string { return takein.TakeInPendingEvent + "." + keys.EscapeName(branch) + "." + strconv.FormatInt(source, 10) } + +// SourceToTake is the source-moved a read of the gate's source revision of a branch, through the seat mesh-delivery's verb +// source, takes in as a heard one of that revision would be (design 47 H H-A-129), and false when the view keeps a revision of +// the branch at or above it, so a revision is never taken in twice. It is the branch's first take-in, start true, exactly +// when the view keeps no revision of the branch; otherwise its movers are judged against the newest kept (H-A-130). Every +// repository the answer names is a head: the movers are the fingerprint changes alone. +func (ix *Indexer) SourceToTake(ctx context.Context, branch string, a meshdelivery.SourceAnswer) (meshdelivery.SourceMovedEvent, bool, error) { + e := meshdelivery.SourceMovedEvent{V: 1, Branch: branch, Source: a.Revision, Call: a.Call, + Heads: make([]meshdelivery.SourceMovedEventHeadsItem, 0, len(a.Repositories))} + if a.Revision <= 0 { + return e, false, nil + } + if ix.Revisions == nil { + return e, false, errors.New("the indexer keeps no catalog revisions") + } + names, err := ix.Revisions.Names(ctx) + if err != nil { + return e, false, fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err) + } + var kept int64 + for _, n := range names { + if r, ok := BranchRevisionOf(branch, n); ok && r > kept { + kept = r + } + } + if kept >= a.Revision { + return e, false, nil + } + e.Start = kept == 0 + for _, r := range a.Repositories { + e.Heads = append(e.Heads, meshdelivery.SourceMovedEventHeadsItem{Repository: r.Repository, Commit: r.Head}) + } + return e, true, nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go index b96aaaae..7545a99f 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go +++ b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go @@ -222,3 +222,35 @@ func TestATakeInOnAFullStoreIsHeldAndSaidOnce(t *testing.T) { t.Errorf("held = %+v after the skip, want none", held) } } + +// Design 47 H §743, §768, H-A-129, H-A-130: the gate's source revision read through its verb source is taken in as a heard +// source-moved of it would be: start true when the view keeps no revision of the branch, every repository it names a head; +// one the view keeps at or above is not taken again; a later one is judged against the newest kept, start false. +func TestASourceRevisionReadFromTheGateIsTakenInOnce(t *testing.T) { + ctx := context.Background() + ix, _, said, f := takeInIndexer(t) + answer := meshdelivery.SourceAnswer{Branch: "main", Revision: 1, Call: "0123456789abcdef", + Repositories: []meshdelivery.SourceHead{{Repository: f.Repository, Trunk: "main", Head: f.Merge}}} + e, take, err := ix.SourceToTake(ctx, "main", answer) + want := meshdelivery.SourceMovedEvent{V: 1, Branch: "main", Source: 1, Call: answer.Call, Start: true, + Heads: []meshdelivery.SourceMovedEventHeadsItem{{Repository: f.Repository, Commit: f.Merge}}} + if err != nil || !take || !reflect.DeepEqual(e, want) { + t.Fatalf("a branch the view keeps none of answered %+v, %t (%v); want %+v taken", e, take, err, want) + } + if taken, err := ix.TakeInSource(ctx, e); err != nil || len(taken.Entries) != 0 || taken.Revision != 1 { + t.Fatalf("the first revision read from the gate said %+v (%v), want revision 1 with no mover", taken, err) + } + if _, take, err := ix.SourceToTake(ctx, "main", answer); take || err != nil { + t.Errorf("a revision the view keeps is taken again (%v)", err) + } + if _, take, _ := ix.SourceToTake(ctx, "nox-mesh", meshdelivery.SourceAnswer{Branch: "nox-mesh"}); take { + t.Error("a branch the gate keeps no revision of is taken in") + } + answer.Revision, answer.Call = 2, "1123456789abcdef" + if e, take, err := ix.SourceToTake(ctx, "main", answer); err != nil || !take || e.Start || e.Source != 2 { + t.Errorf("a later revision answered %+v, %t (%v); want revision 2 taken, start false", e, take, err) + } + if len(*said) != 1 { + t.Errorf("said %d take-in.taken, want 1", len(*said)) + } +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index f84aae13..67592964 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -182,7 +182,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths: // ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a // commit is in another (issue 1900). - if want := []string{"seat:git.trees", "seat:git.tags", "seat:go-package-registry.versions", "seat:npm-package-registry.versions", "seat:git.file", "seat:git.contains", "seat:git.branch", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", + if want := []string{"seat:git.trees", "seat:git.tags", "seat:go-package-registry.versions", "seat:npm-package-registry.versions", "seat:git.file", "seat:git.contains", "seat:git.branch", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index a5d1843b..d71be480 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -152,6 +152,7 @@ "seat:git.file", "seat:git.contains", "seat:git.branch", + "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", -- 2.54.0