Compare commits

..
Author SHA1 Message Date
mesh-tools abb7cfcf5d Merge pull request 'meshcatalogclient: a revision below the held one asks the first view again (H-A-141)' (#266) from fix/47-H1-catalog-client-numbering-restart into main 2026-11-20 14:09:37 +00:00
jschoubben 3956c0cf94 meshcatalogclient: a revision below the held one asks the first view again
A reader whose picture held the earlier model's revision 528 skipped
revisions 1 to 7 of the new numbering in silence: the builder built
every pipeline step as a candidate's, unversioned, so nothing was
released and no want moved. An event below the held revision is now
the state made again: said with both revisions, the view's made time
moved on (so the follower forgets the old numbering's revisions heard
and is not left behind for good), and the first view asked again. An
event equal to the held revision is still dropped.

Builds: design 47 H H-A-141 (hq design/47-H1-answers-59 3d722934), §750, §754
2026-11-20 15:05:06 +01:00
mesh-tools 403fb3068b Merge pull request 'The indexer makes its own per-branch durables and drops the earlier model's revisions at a take-in (design 47 H, H-A-134, H-A-135)' (#265) from fix/47-H1-indexer-durables into main 2026-11-20 11:21:50 +00:00
jschoubben e54bf7f346 indexer: a branch's durable it makes waits 30 s, delivers 5 times, starts from now
As the grants part gives a module's consumer on a seat event's stream kept by age:
without a start from now a made durable would replay every past source-moved of the
branch past the start's guard; the current revision is read at the start already.

Builds: design 47 H §743, H-A-134 (hq 8d30cc42), H-A-129, H-A-130

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
2026-11-20 12:19:50 +01:00
jschoubben f95a164941 indexer: drop the earlier model's trunk and branch revisions at a take-in
catalog-revisions kept trunk.<n> and branch.<branch>.<n> objects beside the
branch-keyed revisions, the newest trunk one marked current, so a reader heard
revision 528 while its picture held 1. A branch's take-in now drops every such
object no record pins, says the count in the journal, and the revisions tool marks
each branch's newest revision current.

Builds: design 47 H §743, H-A-135

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
2026-11-20 12:12:23 +01:00
jschoubben 0cf419619e indexer: make a watched branch's source-moved durable when the bus holds none
The grants part composes no per-branch durable on the source-moved stream, so the
indexer never heard a source revision after its start. Its bind now makes the
durable itself, as a module's seat-event consumer is made (SeatEventConsumerConfig,
named <node>_<indexer>_<branch>, filtered to the branch, one message at a time),
and binds an existing one as it stands.

Builds: design 47 H §743, H-A-134

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
2026-11-20 12:10:46 +01:00
mesh-tools 8b6f6b22d4 Merge pull request 'The indexer reads a branch's source by the verb when its consumer is not bound (design 47 H, H-A-129, H-A-130)' (#264) from fix/47-H1-indexer-bootstrap into main 2026-11-20 09:46:08 +00:00
jschoubben 9cddb5921a indexer: take a watched branch's source revision in from the gate's source when its consumer cannot be bound
At a clean state the grants part makes the per-branch consumers of source-moved only from the modules view, which the
indexer answers only after its first take-in, so nothing ever took the trunk in. At its start, and again at each
failed bind, the indexer now reads the branch's current source revision through mesh-delivery.source and takes it in
as a heard source-moved of it would, when its view keeps no revision of the branch at or above it: start true exactly
when it keeps none, every repository the answer names a head. No timer; binding on registration-changed is unchanged.

Builds: design 47 H §743, §768, H-A-129, H-A-130

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
2026-11-20 10:39:45 +01:00
12 changed files with 586 additions and 48 deletions
@@ -34,7 +34,7 @@ type Half struct {
// ViewName is how the catalog half's revision is named in a part's first-view status (views.Status.View, WaitsFor).
func ViewName(revision int64) string { return strconv.FormatInt(revision, 10) }
// catalogCreated stands for the view's Created in its follower: a revision view is never made again, so one fixed time,
// catalogCreated stands for the view's Created in its follower until its numbering starts again (H-A-141): one fixed time,
// never zero, lets the follower tell a revision heard from one it holds.
var catalogCreated = time.Unix(0, 0).UTC()
@@ -57,6 +57,23 @@ type Catalog struct {
paging *Half
// waits is the revision waited for, 0 when none.
waits int64
// made is when the view's numbering last started again (H-A-141), the follower's Created; zero is catalogCreated.
made time.Time
}
// created is madeLocked, taking mu.
func (c *Catalog) created() time.Time {
c.mu.Lock()
defer c.mu.Unlock()
return c.madeLocked()
}
// madeLocked is the view's Created in its follower: catalogCreated until a numbering started again. The caller holds mu.
func (c *Catalog) madeLocked() time.Time {
if c.made.IsZero() {
return catalogCreated
}
return c.made
}
// NewCatalog is a Catalog of branch asking the seat through call. Nothing is asked before Start.
@@ -146,21 +163,34 @@ func (c *Catalog) Hear(ctx context.Context, data []byte) error {
if c.held != nil {
held = c.held.Revision.Revision
}
if ev.Revision > c.waits {
c.waits = ev.Revision
}
c.mu.Unlock()
if ev.Revision <= held {
if ev.Revision == held {
c.mu.Unlock()
return nil
}
restarted := ev.Revision < held
if restarted {
// A revision below the one the picture holds is the state made again: its numbering started anew (design 47 H
// H-A-141). The view's made time moves on, so the follower forgets the revisions heard of the earlier numbering,
// and what it waits for is this revision.
c.made = time.Now().UTC()
c.waits = ev.Revision
} else if ev.Revision > c.waits {
c.waits = ev.Revision
}
made := c.madeLocked()
c.mu.Unlock()
// The follower keeps the newest revision heard, so an ask that was running when it was heard is asked once more.
c.Follower.Apply(views.Event{Owner: seat.Seat, State: c.Follower.State, Revision: uint64(ev.Revision), Created: catalogCreated, Gap: true})
c.Follower.Apply(views.Event{Owner: seat.Seat, State: c.Follower.State, Revision: uint64(ev.Revision), Created: made, Gap: true})
c.Follower.WaitFor(ctx, ViewName(ev.Revision))
why := fmt.Sprintf("catalog revision %d of %s said, the view holds %d", ev.Revision, c.Branch, held)
if restarted {
why = fmt.Sprintf("catalog revision %d of %s said, below the %d the view holds: the numbering started again", ev.Revision, c.Branch, held)
}
if c.Follower.Logf != nil {
c.Follower.Logf("meshcatalogclient: catalog revision %d of %s said, the view holds %d: %s is asked once more", ev.Revision, c.Branch, held, RevisionVerb)
c.Follower.Logf("meshcatalogclient: %s: %s is asked once more", why, RevisionVerb)
}
go func() {
_ = c.Follower.Gap(ctx, fmt.Sprintf("catalog revision %d of %s said, the view holds %d", ev.Revision, c.Branch, held))
_ = c.Follower.Gap(ctx, why)
}()
return nil
}
@@ -177,7 +207,7 @@ func (c *Catalog) ask(_ context.Context, after string) (views.Answer, error) {
a, err := askRevision(c.call, in)
if errors.Is(err, errNoneYet) {
// A branch with no revision yet is ready and empty, and nothing waits for it (§479, §495).
return views.Answer{V: 1, Owner: seat.Seat, State: c.Follower.State, Created: catalogCreated, Values: map[string]json.RawMessage{}, At: time.Now()}, nil
return views.Answer{V: 1, Owner: seat.Seat, State: c.Follower.State, Created: c.created(), Values: map[string]json.RawMessage{}, At: time.Now()}, nil
}
if err != nil {
return views.Answer{}, err
@@ -212,7 +242,7 @@ func (c *Catalog) ask(_ context.Context, after string) (views.Answer, error) {
if holds {
c.Follower.Holds(ViewName(rev))
}
return views.Answer{V: 1, Owner: seat.Seat, State: c.Follower.State, Revision: uint64(rev), Created: catalogCreated, Values: values, Next: a.Next, At: time.Now()}, nil
return views.Answer{V: 1, Owner: seat.Seat, State: c.Follower.State, Revision: uint64(rev), Created: c.created(), Values: values, Next: a.Next, At: time.Now()}, nil
}
// errNoneYet is the seat's answer `none-yet`: the branch has no revision yet (design 47 G §530).
@@ -188,3 +188,89 @@ func TestCandidatesReachTheHook(t *testing.T) {
t.Errorf("the hook was handed %+v, want the candidate's revision 4 alone", hooked)
}
}
// H-A-141 (design 47 H §750, §754): a catalog-revision below the revision the view holds is the state made again, its
// numbering started anew: said in the log with both revisions, and the first view asked again, so the view follows the new
// numbering and is behind nothing; one equal to the held revision asks nothing; one above is taken in as before.
func TestCatalogFollowsANumberingStartedAgain(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
f := &fakeRevisions{newest: map[string]int64{"main": 528}, asks: map[string]int{}}
var logMu sync.Mutex
var logged []string
c, err := NewCatalog(f.call, "main")
if err != nil {
t.Fatal(err)
}
c.Follower.Wait = time.Nanosecond
c.Follower.Logf = func(format string, args ...any) {
logMu.Lock()
defer logMu.Unlock()
logged = append(logged, fmt.Sprintf(format, args...))
}
came := make(chan string, 16) // the module of each entry that came, one per revision answered
c.Follower.Changed = func(module string, _ json.RawMessage, deleted bool, _ time.Time) {
if !deleted {
came <- module
}
}
if err := c.Start(ctx); err != nil {
t.Fatal(err)
}
<-c.Ready()
<-came // the first view's entry
for _, row := range []struct {
name string
newest int64 // what the seat answers from now
heard int64
asks int // the asks of the first view after the event
holds int64 // the revision the view holds after it
said string // a line the log holds after it; "" none new
restarting bool
}{
{name: "equal to the held revision: dropped", newest: 528, heard: 528, asks: 1, holds: 528},
{name: "above: taken in", newest: 529, heard: 529, asks: 2, holds: 529, said: "catalog revision 529 of main said, the view holds 528"},
{name: "below: the numbering started again", newest: 7, heard: 7, asks: 3, holds: 7, said: "catalog revision 7 of main said, below the 529 the view holds: the numbering started again", restarting: true},
{name: "equal after the restart: dropped", newest: 7, heard: 7, asks: 3, holds: 7},
{name: "above after the restart: taken in", newest: 8, heard: 8, asks: 4, holds: 8, said: "catalog revision 8 of main said, the view holds 7"},
} {
t.Run(row.name, func(t *testing.T) {
logMu.Lock()
before := len(logged)
logMu.Unlock()
f.set("main", row.newest)
if err := c.Hear(ctx, revisionEvent("main", KindTrunk, row.heard)); err != nil {
t.Fatal(err)
}
if row.said != "" {
select {
case m := <-came:
if want := fmt.Sprintf("m-of-main-%d", row.holds); m != want {
t.Fatalf("the entry %s came, want %s", m, want)
}
case <-ctx.Done():
t.Fatalf("the view never took revision %d in", row.holds)
}
}
if h, ok := c.View(); !ok || h.Revision.Revision != row.holds {
t.Fatalf("the view holds %+v, want revision %d", h.Revision, row.holds)
}
if n := f.asked("main"); n != row.asks {
t.Errorf("the first view was asked %d times, want %d", n, row.asks)
}
st := c.Follower.Status()
if st.Behind != "" || st.WaitsFor != "" {
t.Errorf("the view is behind %q, waits for %q, holding %d", st.Behind, st.WaitsFor, st.Revision)
}
logMu.Lock()
fresh := strings.Join(logged[before:], "\n")
logMu.Unlock()
if row.said == "" && strings.Contains(fresh, "meshcatalogclient:") {
t.Errorf("an event the view holds was said: %s", fresh)
}
if row.said != "" && !strings.Contains(fresh, row.said) {
t.Errorf("the log does not say %q: %s", row.said, fresh)
}
})
}
}
@@ -357,7 +357,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
registry.Pruned = ix.PruneBranchBuilds
ix.Registry = registry
// The gate's source revisions, on one durable per watched branch (design 47 H §743, H-A-90), bound below.
sources := &sourceMoved{ctx: ctx, ix: ix, branches: watchedBranches(registry), bind: bindBranch(js, cred, logger.Printf), logf: logger.Printf}
sources := &sourceMoved{ctx: ctx, ix: ix, branches: watchedBranches(registry), bind: bindBranch(js, cred, logger.Printf), source: sourceVia(calls.Call), logf: logger.Printf}
defer sources.stop()
named.Changed = ix.RemoveUnlisted
// The first fill of the versions (design 47 G §372): at the start, and again at each catalog revision kept until it is made.
@@ -7,6 +7,7 @@ import (
"fmt"
"slices"
"sync"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
@@ -20,17 +21,22 @@ import (
// the event's stream, named <node>_<indexer>_<branch>, the branch escaped as a key's name is (H-A-90), each delivering one message
// at a time (the grants part makes it so, H-A-30), acknowledged once its catalog revision and take-in.taken are written. The
// watched branches are the registrations'; a newly watched branch's consumer is bound on the indexer's own
// registration-changed, not only at its start.
// registration-changed, not only at its start. At its start, and again whenever a branch's consumer cannot be bound, the
// branch's current source revision is read through the seat mesh-delivery's verb source and taken in as a heard source-moved
// of it would be, when the view keeps no revision of the branch at or above it (H-A-129, H-A-130): at a clean state the
// consumers are made only once the indexer answers its modules view, which it does only after its first take-in.
type sourceMoved struct {
ctx context.Context
ix *index.Indexer
branches func(context.Context) ([]string, error) // the watched branches the registrations name
bind func(ctx context.Context, branch string, each func(jetstream.Msg)) (stop func(), err error)
source func(ctx context.Context, branch string) (meshdelivery.SourceAnswer, error) // the gate's source revision of a branch
logf func(string, ...any)
mu sync.Mutex
bound map[string]func() // by branch: the consumer's stop
again sync.Mutex // one take-in again at a time
mu sync.Mutex
bound map[string]func() // by branch: the consumer's stop
started bool // the start's refresh ran: every branch's source was read once
again sync.Mutex // one take-in again at a time
}
// watchedBranches are the branches every registration watches, each once, sorted.
@@ -53,10 +59,40 @@ func watchedBranches(r *index.Registry) func(context.Context) ([]string, error)
}
}
// bindBranch binds the indexer's durable of one watched branch on the stream of source-moved through mesh-sdk.
// sourceVia reads a watched branch's current source revision through the seat mesh-delivery's verb source, as the
// indexer's own account (H-A-129): a call where an answer is needed. An answer naming why it could not be read is that
// failure. The branch is given by its name, as the gate reads it (it escapes the name for its key itself).
func sourceVia(call toolCaller) func(context.Context, string) (meshdelivery.SourceAnswer, error) {
verb := "seat:" + meshdelivery.Seat + "." + meshdelivery.VerbSource
return func(_ context.Context, branch string) (meshdelivery.SourceAnswer, error) {
var a meshdelivery.SourceAnswer
raw, err := call(verb, meshdelivery.SourceInput{Branch: branch})
if err != nil {
return a, err
}
if err := json.Unmarshal(raw, &a); err != nil {
return a, fmt.Errorf("the answer of %s is not its seat's answer: %w", verb, err)
}
if a.Error != "" {
return a, fmt.Errorf("%s: %s", verb, a.Error)
}
return a, nil
}
}
// bindBranch binds the indexer's durable of one watched branch on the stream of source-moved through mesh-sdk. One the bus does
// not hold the indexer makes itself, as its own kept consumer, and then binds (design 47 H §743, H-A-134: the grants part
// composes none until H4b); one the bus holds is bound as it stands, never made again.
func bindBranch(js jetstream.JetStream, cred process.Account, logf func(string, ...any)) func(context.Context, string, func(jetstream.Msg)) (func(), error) {
return func(ctx context.Context, branch string, each func(jetstream.Msg)) (func(), error) {
cons, err := cred.NamedEventConsumer(ctx, js, meshdelivery.StreamSourceMoved, keys.EscapeName(branch))
if errors.Is(err, jetstream.ErrConsumerNotFound) {
cfg := branchConsumerConfig(cred, branch)
if cons, err = js.CreateConsumer(ctx, meshdelivery.StreamSourceMoved, cfg); err != nil {
return nil, fmt.Errorf("the consumer %s on %s is not there and is not made: %w", cfg.Durable, meshdelivery.StreamSourceMoved, err)
}
logf("source-moved: the consumer %s on %s was not there: the indexer made it (H-A-134)", cfg.Durable, meshdelivery.StreamSourceMoved)
}
if err != nil {
return nil, err
}
@@ -68,31 +104,107 @@ func bindBranch(js jetstream.JetStream, cred process.Account, logf func(string,
}
}
// branchConsumerConfig is the indexer's durable of one watched branch as the mesh gives a module's seat-event consumer
// (mesh-sdk's SeatEventConsumerConfig), named <node>_<indexer>_<branch> as NamedEventConsumer binds it, filtered to the
// branch's source-moved and delivering one message at a time, with the ack wait of 30 s, the at most 5 deliveries and the
// start from now the grants part gives a module's consumer on a seat event's stream kept by age (mesh-grants buscompose
// heardConsumer): the branch's current source revision is read at the start through the gate's source (H-A-129, H-A-130), so
// its past source-moved are never taken in again (design 47 H §743, H-A-90, H-A-134).
func branchConsumerConfig(cred process.Account, branch string) jetstream.ConsumerConfig {
esc := keys.EscapeName(branch)
cfg := cred.SeatEventConsumerConfig([]string{meshdelivery.SubjectSourceMoved(esc, "*")}, false)
cfg.Durable = cred.EventConsumerName() + "_" + esc
cfg.MaxAckPending = 1
cfg.AckWait, cfg.MaxDeliver = branchAckWait, branchMaxDeliver
cfg.DeliverPolicy = jetstream.DeliverNewPolicy
return cfg
}
// The ack wait and the deliveries of a branch's durable, as the grants part gives a module's consumers (H-A-134, H-A-133).
const (
branchAckWait = 30 * time.Second
branchMaxDeliver = 5
)
// refresh binds the consumer of every watched branch not bound yet: at the start and on each registration-changed. A consumer
// not there yet is said and bound at the next registration-changed or start.
// not there yet is said and bound at the next registration-changed or start. The start's refresh reads every watched branch's
// source revision, a later one each branch whose consumer could not be bound (H-A-129), after the binds.
func (s *sourceMoved) refresh() {
branches, err := s.branches(s.ctx)
if err != nil {
s.logf("source-moved: the watched branches are not read, so no new branch's consumer is bound: %v", err)
return
}
for _, branch := range s.bindAll(branches) {
s.readSource(branch)
}
}
// bindAll binds the branches not bound yet and answers those whose source revision is to be read now: at the start every
// watched branch, later each one whose consumer could not be bound.
func (s *sourceMoved) bindAll(branches []string) (read []string) {
s.mu.Lock()
defer s.mu.Unlock()
if s.bound == nil {
s.bound = map[string]func(){}
}
start := !s.started
s.started = true
for _, b := range branches {
if _, ok := s.bound[b]; ok {
if start {
read = append(read, b)
}
continue
}
branch := b
stop, err := s.bind(s.ctx, branch, func(m jetstream.Msg) { s.heard(branch, m) })
if err != nil {
s.logf("source-moved: the consumer of the branch %s is not bound, so its source revisions are not taken in until the next registration-changed or start: %v", branch, err)
s.logf("source-moved: the consumer of the branch %s is not bound, so its later source revisions are not heard until the next registration-changed or start; its current one is read through the gate's source: %v", branch, err)
read = append(read, branch)
continue
}
s.bound[branch] = stop
s.logf("source-moved: the consumer of the branch %s is bound", branch)
if start {
read = append(read, branch)
}
}
return read
}
// readSource reads a branch's current source revision through the gate's verb source and takes it in as a heard
// source-moved of that revision would be (takeIn), when the view keeps no revision of the branch at or above it (H-A-129,
// H-A-130: index.SourceToTake).
func (s *sourceMoved) readSource(branch string) {
if s.source == nil {
return
}
a, err := s.source(s.ctx, branch)
if err != nil {
s.logf("source-moved: the gate's source revision of %s is not read, so it is taken in only once its source-moved is heard: %v", branch, err)
return
}
if a.Revision <= 0 {
s.logf("source-moved: the gate keeps no source revision of %s yet", branch)
return
}
s.again.Lock()
defer s.again.Unlock()
e, take, err := s.ix.SourceToTake(s.ctx, branch, a)
if err != nil {
s.logf("source-moved: the catalog revisions of %s are not read, so the gate's source revision %d of it is not taken in now: %v", branch, a.Revision, err)
return
}
if !take {
return
}
s.logf("source-moved: source revision %d of %s is read through the gate's source and taken in now (start %t)", a.Revision, branch, e.Start)
switch err := s.takeIn(branch, e); {
case errors.Is(err, index.ErrOldForm):
s.logf("source-moved: the gate's source revision %d of %s is refused, the old form: %v", e.Source, branch, err)
case err != nil:
s.logf("source-moved: the gate's source revision %d of %s is not taken in now: %v", e.Source, branch, err)
}
}
@@ -109,10 +221,7 @@ func (s *sourceMoved) heard(branch string, m jetstream.Msg) {
}
s.again.Lock()
defer s.again.Unlock()
behind, err := s.ix.HoldBehind(s.ctx, e)
if !behind && err == nil {
_, err = s.ix.TakeInSource(s.ctx, e)
}
err := s.takeIn(branch, e)
switch {
case errors.Is(err, index.ErrOldForm):
s.logf("source-moved of %s on %s is refused, the old form: %v", branch, m.Subject(), err)
@@ -123,14 +232,25 @@ func (s *sourceMoved) heard(branch string, m jetstream.Msg) {
return
case err != nil:
s.logf("source-moved: source revision %d of %s is held, not taken in: %v", e.Source, branch, err)
case behind:
s.logf("source-moved: source revision %d of %s is held behind the branch's earlier held ones", e.Source, branch)
}
if err := m.Ack(); err != nil {
s.logf("source-moved: source revision %d of %s is taken in or held and its acknowledgement failed: %v", e.Source, branch, err)
}
}
// takeIn takes one source-moved of a branch in, or holds it behind the branch's earlier held ones, so its source revisions
// are taken in in their order; the caller holds s.again. A heard one and one read through the gate's source go this one way.
func (s *sourceMoved) takeIn(branch string, e meshdelivery.SourceMovedEvent) error {
behind, err := s.ix.HoldBehind(s.ctx, e)
if !behind && err == nil {
_, err = s.ix.TakeInSource(s.ctx, e)
}
if behind && err == nil {
s.logf("source-moved: source revision %d of %s is held behind the branch's earlier held ones", e.Source, branch)
}
return err
}
// takeAgain takes every held source revision in again from the state held, each branch in its order, on the indexer's own room
// (H-A-50) and at its start (H-A-114).
func (s *sourceMoved) takeAgain() {
@@ -2,14 +2,20 @@ package main
import (
"context"
"errors"
"fmt"
"reflect"
"slices"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-indexer/internal/index"
)
@@ -88,3 +94,143 @@ func TestAHeldSourceMovedIsSkippedByHand(t *testing.T) {
t.Errorf("held %+v after the skip, want source 5 alone", hs)
}
}
// revisionObjects is the catalog revisions' object bucket in memory.
type revisionObjects map[string][]byte
func (o revisionObjects) Put(_ context.Context, name string, data []byte) error {
o[name] = data
return nil
}
func (o revisionObjects) Get(_ context.Context, name string) ([]byte, error) {
if d, ok := o[name]; ok {
return d, nil
}
return nil, index.ErrNoObject
}
func (o revisionObjects) Delete(_ context.Context, name string) error { delete(o, name); return nil }
func (o revisionObjects) Names(context.Context) ([]string, error) {
var out []string
for n := range o {
out = append(out, n)
}
return out, nil
}
func (o revisionObjects) Fill(context.Context) (int64, int64, error) { return 0, 1 << 30, nil }
// Design 47 H §743, §768, H-A-129: at a clean state no branch's consumer is there to bind; the start reads each watched
// branch's source revision through the gate's verb source and takes it in, and a failed bind at a registration-changed reads
// it again without taking it in twice. A start whose view keeps the revision reads it and takes nothing in.
func TestAStartWithNoConsumerTakesTheGatesSourceRevisionIn(t *testing.T) {
ctx := context.Background()
var taken []string
objects := revisionObjects{}
ix := &index.Indexer{Module: testIndexer, Revisions: objects, Modules: index.MemoryStore(), Graph: index.MemoryStore(),
Held: &memStore{vals: map[string][]byte{}}, Logf: func(string, ...any) {},
SayTakeIn: func(_ context.Context, event, id string, _ any) error {
taken = append(taken, event+" "+id)
return nil
}}
var asked []string
newSources := func() *sourceMoved {
return &sourceMoved{ctx: ctx, ix: ix,
branches: func(context.Context) ([]string, error) { return []string{"main"}, nil },
bind: func(context.Context, string, func(jetstream.Msg)) (func(), error) {
return nil, errors.New("nats: API error: code=404 err_code=10014 description=consumer not found")
},
source: func(_ context.Context, branch string) (meshdelivery.SourceAnswer, error) {
asked = append(asked, branch)
return meshdelivery.SourceAnswer{Branch: branch, Revision: 1, Call: "0123456789abcdef"}, nil
},
logf: func(string, ...any) {}}
}
s := newSources()
s.refresh() // the start
if _, ok := objects[index.BranchObject("main", 1)]; !ok || !reflect.DeepEqual(taken, []string{"take-in.taken " + index.TakenID("main", 1)}) {
t.Fatalf("the start kept %v and said %v; want main.1 taken in from the gate's source", objects, taken)
}
s.refresh() // a registration-changed, the consumer still not there
newSources().refresh() // a start whose view keeps the revision
if !reflect.DeepEqual(asked, []string{"main", "main", "main"}) || len(taken) != 1 {
t.Errorf("asked the gate for %v and said %v; want main asked at each and taken in once", asked, taken)
}
}
// fakeBranchJS is the bus as bindBranch uses it: the source-moved stream's consumers by name, and the consumers made.
type fakeBranchJS struct {
jetstream.JetStream
held map[string]*fakeBranchConsumer
made []jetstream.ConsumerConfig
}
func (f *fakeBranchJS) Consumer(_ context.Context, stream, name string) (jetstream.Consumer, error) {
if c, ok := f.held[stream+"/"+name]; ok {
return c, nil
}
return nil, jetstream.ErrConsumerNotFound
}
func (f *fakeBranchJS) CreateConsumer(_ context.Context, stream string, cfg jetstream.ConsumerConfig) (jetstream.Consumer, error) {
f.made = append(f.made, cfg)
c := &fakeBranchConsumer{}
f.held[stream+"/"+cfg.Durable] = c
return c, nil
}
// fakeBranchConsumer hands one source-moved to the hearing as soon as it is consumed.
type fakeBranchConsumer struct {
jetstream.Consumer
heard int
}
type fakeConsumeContext struct{ jetstream.ConsumeContext }
func (fakeConsumeContext) Stop() {}
func (c *fakeBranchConsumer) Consume(handle jetstream.MessageHandler, _ ...jetstream.PullConsumeOpt) (jetstream.ConsumeContext, error) {
c.heard++
handle(msgOf{f: &fakeMsg{subject: "mesh.seat.mesh-delivery.event.source-moved.main.2"}})
return fakeConsumeContext{}, nil
}
// Design 47 H §743, H-A-134: a branch's durable the bus does not hold the indexer makes, as a module's seat-event consumer is
// made (SeatEventConsumerConfig), named <node>_<indexer>_<branch>, filtered to the branch, one message at a time, and then
// hears on it; one the bus holds is bound, never made again.
func TestABranchDurableNotThereIsMadeAndHeard(t *testing.T) {
cred := process.Account{Node: "n1", Module: "mesh-delivery-indexer"}
for _, tc := range []struct {
name string
held bool
}{{"not there: made", false}, {"there: bound", true}} {
t.Run(tc.name, func(t *testing.T) {
js := &fakeBranchJS{held: map[string]*fakeBranchConsumer{}}
name := "n1_mesh-delivery-indexer_" + keys.EscapeName("main")
var existing *fakeBranchConsumer
if tc.held {
existing = &fakeBranchConsumer{}
js.held[meshdelivery.StreamSourceMoved+"/"+name] = existing
}
var heard []string
stop, err := bindBranch(js, cred, func(string, ...any) {})(context.Background(), "main", func(m jetstream.Msg) { heard = append(heard, m.Subject()) })
if err != nil {
t.Fatal(err)
}
stop()
if len(heard) != 1 {
t.Fatalf("heard %v, want the one source-moved the consumer hands", heard)
}
if tc.held {
if len(js.made) != 0 || existing.heard != 1 {
t.Fatalf("made %v and heard the held one %d times; want it bound, not made again", js.made, existing.heard)
}
return
}
want := cred.SeatEventConsumerConfig([]string{meshdelivery.SubjectSourceMoved(keys.EscapeName("main"), "*")}, false)
want.Durable, want.MaxAckPending = name, 1
want.AckWait, want.MaxDeliver, want.DeliverPolicy = 30*time.Second, 5, jetstream.DeliverNewPolicy
if len(js.made) != 1 || !reflect.DeepEqual(js.made[0], want) {
t.Fatalf("made %+v, want once %+v", js.made, want)
}
})
}
}
@@ -337,6 +337,11 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int
} else if len(removed) > 0 {
ix.logf("pins: %d revisions removed (%s), no record pins them and none is current", len(removed), strings.Join(removed, ", "))
}
if dropped, err := ix.dropEarlierFormsLocked(ctx); err != nil {
ix.logf("catalog graph: the revisions of the earlier forms (trunk.*, branch.*) are not dropped: %v", err)
} else if len(dropped) > 0 {
ix.logf("catalog graph: %d revisions of the earlier forms dropped (%s), no record pins them; each branch's newest revision is its current one", len(dropped), strings.Join(dropped, ", "))
}
if _, err := ix.removeUnlisted(ctx); err != nil {
ix.logf("catalog graph: the entries nothing lists or names any more are not removed: %v", err) // at the first revision after none does (§675)
}
@@ -253,25 +253,10 @@ func (ix *Indexer) judgePinsLocked(ctx context.Context) ([]string, error) {
return nil, err
}
current := CurrentRevisions(names)
pinned := map[string]bool{}
pinKeys, err := ix.Pins.Keys(ctx)
pinKeys, pinned, err := ix.pinnedObjects(ctx)
if err != nil {
return nil, err
}
for _, k := range pinKeys {
object, ok := pinObject(k)
if !ok {
continue
}
v, _, found, err := ix.Pins.Get(ctx, k)
if err != nil {
return nil, err
}
var pv PinValue
if found && json.Unmarshal(v, &pv) == nil && len(pv.Records) > 0 {
pinned[object] = true
}
}
var removed []string
for _, n := range names {
if _, _, ok := RevisionGroup(n); !ok || pinned[n] {
@@ -301,6 +286,72 @@ func (ix *Indexer) judgePinsLocked(ctx context.Context) ([]string, error) {
return removed, nil
}
// pinnedObjects are the state pins' keys and the revision objects a record pins.
func (ix *Indexer) pinnedObjects(ctx context.Context) ([]string, map[string]bool, error) {
pinned := map[string]bool{}
pinKeys, err := ix.Pins.Keys(ctx)
if err != nil {
return nil, nil, err
}
for _, k := range pinKeys {
object, ok := pinObject(k)
if !ok {
continue
}
v, _, found, err := ix.Pins.Get(ctx, k)
if err != nil {
return nil, nil, err
}
var pv PinValue
if found && json.Unmarshal(v, &pv) == nil && len(pv.Records) > 0 {
pinned[object] = true
}
}
return pinKeys, pinned, nil
}
// earlierForm says whether a revision object is of the earlier model's forms, trunk.<n> or branch.<branch>.<n>, beside the
// branch-keyed <branch>.<n> of design 47 H §743 (H-A-135).
func earlierForm(name string) bool {
if _, _, ok := RevisionGroup(name); !ok {
return false
}
return strings.HasPrefix(name, "trunk.") || strings.HasPrefix(name, "branch.")
}
// dropEarlierFormsLocked drops every revision object of the earlier forms (earlierForm) no record pins, at a branch's take-in
// (design 47 H §743, H-A-135): the branch's newest revision is its current one, and an earlier trunk revision kept beside it
// would be the newest trunk revision a reader hears. It drops none while the pins do not count yet; the caller holds keepMu.
func (ix *Indexer) dropEarlierFormsLocked(ctx context.Context) ([]string, error) {
if ix.PinsPending != nil {
if pending, err := ix.PinsPending(ctx); err != nil || pending != 0 {
return nil, err
}
}
names, err := ix.Revisions.Names(ctx)
if err != nil {
return nil, err
}
pinned := map[string]bool{}
if ix.Pins != nil {
if _, pinned, err = ix.pinnedObjects(ctx); err != nil {
return nil, err
}
}
var dropped []string
for _, n := range names {
if !earlierForm(n) || pinned[n] {
continue
}
if err := ix.Revisions.Delete(ctx, n); err != nil {
return dropped, fmt.Errorf("%s: %w", n, err)
}
dropped = append(dropped, n)
}
sort.Strings(dropped)
return dropped, nil
}
// PinDrop drops one pin of the state pins by hand, by its key <branch>.<revision> (indexer_pin_drop, design 47 H §765, H-A-57):
// every record's, for a pin that names no running pipeline; the finding pin-missing of the key is cleared, and the removal
// rule then applies.
@@ -61,7 +61,8 @@ type RevisionSummary struct {
Heads string `json:"heads,omitempty"`
At *time.Time `json:"at,omitempty"`
Entries int `json:"entries"`
// Current says it is the trunk's newest revision, which is kept whatever pins it.
// Current says it is the newest revision of its watched branch (CurrentRevisions, §579; H §743, H-A-135: no longer the
// newest trunk-kind revision over every branch, and never one of the earlier forms), which is kept whatever pins it.
Current bool `json:"current,omitempty"`
}
@@ -84,7 +85,6 @@ func ListRevisions(ctx context.Context, o Objects, candidates bool) (RevisionsAn
return out, fmt.Errorf("the objects %s cannot be listed: %w", CatalogRevisionsObjects, err)
}
listed := map[string]bool{}
var newest int64
entries := []string{}
for _, n := range names {
if strings.HasPrefix(n, "entry.") {
@@ -103,17 +103,16 @@ func ListRevisions(ctx context.Context, o Objects, candidates bool) (RevisionsAn
for _, ref := range rev.Entries {
listed[EntryObject(ref.Digest)] = true
}
if rev.Kind == "trunk" && rev.Revision > newest {
newest = rev.Revision
}
if candidates && rev.Kind != "candidate" {
continue
}
out.Revisions = append(out.Revisions, RevisionSummary{Name: n, Kind: rev.Kind, Revision: rev.Revision, Source: rev.Source,
Group: rev.Group, Heads: rev.Heads, At: rev.At, Entries: len(rev.Entries)})
}
current := CurrentRevisions(names)
for i := range out.Revisions {
out.Revisions[i].Current = out.Revisions[i].Kind == "trunk" && out.Revisions[i].Revision == newest
_, cur := current[out.Revisions[i].Name]
out.Revisions[i].Current = cur && out.Revisions[i].Kind == "trunk" && !earlierForm(out.Revisions[i].Name)
}
slices.SortFunc(out.Revisions, func(a, b RevisionSummary) int { return strings.Compare(a.Name, b.Name) })
out.Entries = len(entries)
@@ -142,3 +142,37 @@ func (ix *Indexer) sayTaken(ctx context.Context, t meshcatalog.TakeInTaken) erro
func PendingID(branch string, source int64) string {
return takein.TakeInPendingEvent + "." + keys.EscapeName(branch) + "." + strconv.FormatInt(source, 10)
}
// SourceToTake is the source-moved a read of the gate's source revision of a branch, through the seat mesh-delivery's verb
// source, takes in as a heard one of that revision would be (design 47 H H-A-129), and false when the view keeps a revision of
// the branch at or above it, so a revision is never taken in twice. It is the branch's first take-in, start true, exactly
// when the view keeps no revision of the branch; otherwise its movers are judged against the newest kept (H-A-130). Every
// repository the answer names is a head: the movers are the fingerprint changes alone.
func (ix *Indexer) SourceToTake(ctx context.Context, branch string, a meshdelivery.SourceAnswer) (meshdelivery.SourceMovedEvent, bool, error) {
e := meshdelivery.SourceMovedEvent{V: 1, Branch: branch, Source: a.Revision, Call: a.Call,
Heads: make([]meshdelivery.SourceMovedEventHeadsItem, 0, len(a.Repositories))}
if a.Revision <= 0 {
return e, false, nil
}
if ix.Revisions == nil {
return e, false, errors.New("the indexer keeps no catalog revisions")
}
names, err := ix.Revisions.Names(ctx)
if err != nil {
return e, false, fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err)
}
var kept int64
for _, n := range names {
if r, ok := BranchRevisionOf(branch, n); ok && r > kept {
kept = r
}
}
if kept >= a.Revision {
return e, false, nil
}
e.Start = kept == 0
for _, r := range a.Repositories {
e.Heads = append(e.Heads, meshdelivery.SourceMovedEventHeadsItem{Repository: r.Repository, Commit: r.Head})
}
return e, true, nil
}
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"testing"
@@ -121,6 +122,39 @@ func TestAFirstSourceRevisionIsTakenInWithNoMover(t *testing.T) {
}
}
// Design 47 H §743, H-A-135: at a branch's take-in the earlier model's revisions (trunk.<n>, branch.<branch>.<n>) no record
// pins are dropped, one a record pins stays, and the branch's newest revision is the current one, no longer the newest trunk.
func TestATakeInDropsTheEarlierFormsNoRecordPins(t *testing.T) {
ctx := context.Background()
ix, objects, _, f := takeInIndexer(t)
for _, n := range []string{"trunk.527", "trunk.528", "branch.master.1"} {
_, rev, _ := RevisionGroup(n)
raw, _ := json.Marshal(meshcatalog.CatalogRevision{Revision: rev, Kind: "trunk", Source: 1000 + rev})
if err := objects.Put(ctx, n, raw); err != nil {
t.Fatal(err)
}
}
if err := ix.pin(ctx, "trunk.527", "pipeline.old"); err != nil {
t.Fatal(err)
}
start := meshdelivery.SourceMovedEvent{V: 1, Branch: "main", Source: 1, Call: "seed.operator.20261120", Start: true,
Heads: []meshdelivery.SourceMovedEventHeadsItem{{Repository: f.Repository, Commit: f.Merge}}}
if _, err := ix.TakeInSource(ctx, start); err != nil {
t.Fatal(err)
}
a, err := ListRevisions(ctx, objects, false)
if err != nil {
t.Fatal(err)
}
var got []string
for _, r := range a.Revisions {
got = append(got, fmt.Sprintf("%s current=%t", r.Name, r.Current))
}
if want := []string{"main.1 current=true", "trunk.527 current=false"}; !reflect.DeepEqual(got, want) {
t.Errorf("kept %v, want %v: the unpinned earlier forms dropped, the pinned one kept, main.1 current", got, want)
}
}
// Design 47 H §769: the take-in of a source revision moving the catalogue repository reads `predefined` from that commit's
// trees, and one changing a predefined key rewrites it.
func TestATakeInOfTheCatalogsCommitRewritesPredefined(t *testing.T) {
@@ -222,3 +256,35 @@ func TestATakeInOnAFullStoreIsHeldAndSaidOnce(t *testing.T) {
t.Errorf("held = %+v after the skip, want none", held)
}
}
// Design 47 H §743, §768, H-A-129, H-A-130: the gate's source revision read through its verb source is taken in as a heard
// source-moved of it would be: start true when the view keeps no revision of the branch, every repository it names a head;
// one the view keeps at or above is not taken again; a later one is judged against the newest kept, start false.
func TestASourceRevisionReadFromTheGateIsTakenInOnce(t *testing.T) {
ctx := context.Background()
ix, _, said, f := takeInIndexer(t)
answer := meshdelivery.SourceAnswer{Branch: "main", Revision: 1, Call: "0123456789abcdef",
Repositories: []meshdelivery.SourceHead{{Repository: f.Repository, Trunk: "main", Head: f.Merge}}}
e, take, err := ix.SourceToTake(ctx, "main", answer)
want := meshdelivery.SourceMovedEvent{V: 1, Branch: "main", Source: 1, Call: answer.Call, Start: true,
Heads: []meshdelivery.SourceMovedEventHeadsItem{{Repository: f.Repository, Commit: f.Merge}}}
if err != nil || !take || !reflect.DeepEqual(e, want) {
t.Fatalf("a branch the view keeps none of answered %+v, %t (%v); want %+v taken", e, take, err, want)
}
if taken, err := ix.TakeInSource(ctx, e); err != nil || len(taken.Entries) != 0 || taken.Revision != 1 {
t.Fatalf("the first revision read from the gate said %+v (%v), want revision 1 with no mover", taken, err)
}
if _, take, err := ix.SourceToTake(ctx, "main", answer); take || err != nil {
t.Errorf("a revision the view keeps is taken again (%v)", err)
}
if _, take, _ := ix.SourceToTake(ctx, "nox-mesh", meshdelivery.SourceAnswer{Branch: "nox-mesh"}); take {
t.Error("a branch the gate keeps no revision of is taken in")
}
answer.Revision, answer.Call = 2, "1123456789abcdef"
if e, take, err := ix.SourceToTake(ctx, "main", answer); err != nil || !take || e.Start || e.Source != 2 {
t.Errorf("a later revision answered %+v, %t (%v); want revision 2 taken, start false", e, take, err)
}
if len(*said) != 1 {
t.Errorf("said %d take-in.taken, want 1", len(*said))
}
}
@@ -182,7 +182,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
// Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths:
// ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a
// commit is in another (issue 1900).
if want := []string{"seat:git.trees", "seat:git.tags", "seat:go-package-registry.versions", "seat:npm-package-registry.versions", "seat:git.file", "seat:git.contains", "seat:git.branch", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps",
if want := []string{"seat:git.trees", "seat:git.tags", "seat:go-package-registry.versions", "seat:npm-package-registry.versions", "seat:git.file", "seat:git.contains", "seat:git.branch", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps",
"seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) {
t.Errorf("invokes = %v, want %v", m.Invokes, want)
}
@@ -152,6 +152,7 @@
"seat:git.file",
"seat:git.contains",
"seat:git.branch",
"seat:mesh-delivery.source",
"seat:mesh-module-assigner.assignments",
"seat:mesh-module-assigner.placement",
"seat:mesh-hand-acts.hand-act",