Compare commits

...
6 Commits
Author SHA1 Message Date
mesh-tools 403fb3068b Merge pull request 'The indexer makes its own per-branch durables and drops the earlier model's revisions at a take-in (design 47 H, H-A-134, H-A-135)' (#265) from fix/47-H1-indexer-durables into main 2026-11-20 11:21:50 +00:00
jschoubben e54bf7f346 indexer: a branch's durable it makes waits 30 s, delivers 5 times, starts from now
As the grants part gives a module's consumer on a seat event's stream kept by age:
without a start from now a made durable would replay every past source-moved of the
branch past the start's guard; the current revision is read at the start already.

Builds: design 47 H §743, H-A-134 (hq 8d30cc42), H-A-129, H-A-130

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
2026-11-20 12:19:50 +01:00
jschoubben f95a164941 indexer: drop the earlier model's trunk and branch revisions at a take-in
catalog-revisions kept trunk.<n> and branch.<branch>.<n> objects beside the
branch-keyed revisions, the newest trunk one marked current, so a reader heard
revision 528 while its picture held 1. A branch's take-in now drops every such
object no record pins, says the count in the journal, and the revisions tool marks
each branch's newest revision current.

Builds: design 47 H §743, H-A-135

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
2026-11-20 12:12:23 +01:00
jschoubben 0cf419619e indexer: make a watched branch's source-moved durable when the bus holds none
The grants part composes no per-branch durable on the source-moved stream, so the
indexer never heard a source revision after its start. Its bind now makes the
durable itself, as a module's seat-event consumer is made (SeatEventConsumerConfig,
named <node>_<indexer>_<branch>, filtered to the branch, one message at a time),
and binds an existing one as it stands.

Builds: design 47 H §743, H-A-134

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
2026-11-20 12:10:46 +01:00
mesh-tools 8b6f6b22d4 Merge pull request 'The indexer reads a branch's source by the verb when its consumer is not bound (design 47 H, H-A-129, H-A-130)' (#264) from fix/47-H1-indexer-bootstrap into main 2026-11-20 09:46:08 +00:00
jschoubben 9cddb5921a indexer: take a watched branch's source revision in from the gate's source when its consumer cannot be bound
At a clean state the grants part makes the per-branch consumers of source-moved only from the modules view, which the
indexer answers only after its first take-in, so nothing ever took the trunk in. At its start, and again at each
failed bind, the indexer now reads the branch's current source revision through mesh-delivery.source and takes it in
as a heard source-moved of it would, when its view keeps no revision of the branch at or above it: start true exactly
when it keeps none, every repository the answer names a head. No timer; binding on registration-changed is unchanged.

Builds: design 47 H §743, §768, H-A-129, H-A-130

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
2026-11-20 10:39:45 +01:00
10 changed files with 459 additions and 37 deletions
@@ -357,7 +357,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
registry.Pruned = ix.PruneBranchBuilds
ix.Registry = registry
// The gate's source revisions, on one durable per watched branch (design 47 H §743, H-A-90), bound below.
sources := &sourceMoved{ctx: ctx, ix: ix, branches: watchedBranches(registry), bind: bindBranch(js, cred, logger.Printf), logf: logger.Printf}
sources := &sourceMoved{ctx: ctx, ix: ix, branches: watchedBranches(registry), bind: bindBranch(js, cred, logger.Printf), source: sourceVia(calls.Call), logf: logger.Printf}
defer sources.stop()
named.Changed = ix.RemoveUnlisted
// The first fill of the versions (design 47 G §372): at the start, and again at each catalog revision kept until it is made.
@@ -7,6 +7,7 @@ import (
"fmt"
"slices"
"sync"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
@@ -20,17 +21,22 @@ import (
// the event's stream, named <node>_<indexer>_<branch>, the branch escaped as a key's name is (H-A-90), each delivering one message
// at a time (the grants part makes it so, H-A-30), acknowledged once its catalog revision and take-in.taken are written. The
// watched branches are the registrations'; a newly watched branch's consumer is bound on the indexer's own
// registration-changed, not only at its start.
// registration-changed, not only at its start. At its start, and again whenever a branch's consumer cannot be bound, the
// branch's current source revision is read through the seat mesh-delivery's verb source and taken in as a heard source-moved
// of it would be, when the view keeps no revision of the branch at or above it (H-A-129, H-A-130): at a clean state the
// consumers are made only once the indexer answers its modules view, which it does only after its first take-in.
type sourceMoved struct {
ctx context.Context
ix *index.Indexer
branches func(context.Context) ([]string, error) // the watched branches the registrations name
bind func(ctx context.Context, branch string, each func(jetstream.Msg)) (stop func(), err error)
source func(ctx context.Context, branch string) (meshdelivery.SourceAnswer, error) // the gate's source revision of a branch
logf func(string, ...any)
mu sync.Mutex
bound map[string]func() // by branch: the consumer's stop
again sync.Mutex // one take-in again at a time
mu sync.Mutex
bound map[string]func() // by branch: the consumer's stop
started bool // the start's refresh ran: every branch's source was read once
again sync.Mutex // one take-in again at a time
}
// watchedBranches are the branches every registration watches, each once, sorted.
@@ -53,10 +59,40 @@ func watchedBranches(r *index.Registry) func(context.Context) ([]string, error)
}
}
// bindBranch binds the indexer's durable of one watched branch on the stream of source-moved through mesh-sdk.
// sourceVia reads a watched branch's current source revision through the seat mesh-delivery's verb source, as the
// indexer's own account (H-A-129): a call where an answer is needed. An answer naming why it could not be read is that
// failure. The branch is given by its name, as the gate reads it (it escapes the name for its key itself).
func sourceVia(call toolCaller) func(context.Context, string) (meshdelivery.SourceAnswer, error) {
verb := "seat:" + meshdelivery.Seat + "." + meshdelivery.VerbSource
return func(_ context.Context, branch string) (meshdelivery.SourceAnswer, error) {
var a meshdelivery.SourceAnswer
raw, err := call(verb, meshdelivery.SourceInput{Branch: branch})
if err != nil {
return a, err
}
if err := json.Unmarshal(raw, &a); err != nil {
return a, fmt.Errorf("the answer of %s is not its seat's answer: %w", verb, err)
}
if a.Error != "" {
return a, fmt.Errorf("%s: %s", verb, a.Error)
}
return a, nil
}
}
// bindBranch binds the indexer's durable of one watched branch on the stream of source-moved through mesh-sdk. One the bus does
// not hold the indexer makes itself, as its own kept consumer, and then binds (design 47 H §743, H-A-134: the grants part
// composes none until H4b); one the bus holds is bound as it stands, never made again.
func bindBranch(js jetstream.JetStream, cred process.Account, logf func(string, ...any)) func(context.Context, string, func(jetstream.Msg)) (func(), error) {
return func(ctx context.Context, branch string, each func(jetstream.Msg)) (func(), error) {
cons, err := cred.NamedEventConsumer(ctx, js, meshdelivery.StreamSourceMoved, keys.EscapeName(branch))
if errors.Is(err, jetstream.ErrConsumerNotFound) {
cfg := branchConsumerConfig(cred, branch)
if cons, err = js.CreateConsumer(ctx, meshdelivery.StreamSourceMoved, cfg); err != nil {
return nil, fmt.Errorf("the consumer %s on %s is not there and is not made: %w", cfg.Durable, meshdelivery.StreamSourceMoved, err)
}
logf("source-moved: the consumer %s on %s was not there: the indexer made it (H-A-134)", cfg.Durable, meshdelivery.StreamSourceMoved)
}
if err != nil {
return nil, err
}
@@ -68,31 +104,107 @@ func bindBranch(js jetstream.JetStream, cred process.Account, logf func(string,
}
}
// branchConsumerConfig is the indexer's durable of one watched branch as the mesh gives a module's seat-event consumer
// (mesh-sdk's SeatEventConsumerConfig), named <node>_<indexer>_<branch> as NamedEventConsumer binds it, filtered to the
// branch's source-moved and delivering one message at a time, with the ack wait of 30 s, the at most 5 deliveries and the
// start from now the grants part gives a module's consumer on a seat event's stream kept by age (mesh-grants buscompose
// heardConsumer): the branch's current source revision is read at the start through the gate's source (H-A-129, H-A-130), so
// its past source-moved are never taken in again (design 47 H §743, H-A-90, H-A-134).
func branchConsumerConfig(cred process.Account, branch string) jetstream.ConsumerConfig {
esc := keys.EscapeName(branch)
cfg := cred.SeatEventConsumerConfig([]string{meshdelivery.SubjectSourceMoved(esc, "*")}, false)
cfg.Durable = cred.EventConsumerName() + "_" + esc
cfg.MaxAckPending = 1
cfg.AckWait, cfg.MaxDeliver = branchAckWait, branchMaxDeliver
cfg.DeliverPolicy = jetstream.DeliverNewPolicy
return cfg
}
// The ack wait and the deliveries of a branch's durable, as the grants part gives a module's consumers (H-A-134, H-A-133).
const (
branchAckWait = 30 * time.Second
branchMaxDeliver = 5
)
// refresh binds the consumer of every watched branch not bound yet: at the start and on each registration-changed. A consumer
// not there yet is said and bound at the next registration-changed or start.
// not there yet is said and bound at the next registration-changed or start. The start's refresh reads every watched branch's
// source revision, a later one each branch whose consumer could not be bound (H-A-129), after the binds.
func (s *sourceMoved) refresh() {
branches, err := s.branches(s.ctx)
if err != nil {
s.logf("source-moved: the watched branches are not read, so no new branch's consumer is bound: %v", err)
return
}
for _, branch := range s.bindAll(branches) {
s.readSource(branch)
}
}
// bindAll binds the branches not bound yet and answers those whose source revision is to be read now: at the start every
// watched branch, later each one whose consumer could not be bound.
func (s *sourceMoved) bindAll(branches []string) (read []string) {
s.mu.Lock()
defer s.mu.Unlock()
if s.bound == nil {
s.bound = map[string]func(){}
}
start := !s.started
s.started = true
for _, b := range branches {
if _, ok := s.bound[b]; ok {
if start {
read = append(read, b)
}
continue
}
branch := b
stop, err := s.bind(s.ctx, branch, func(m jetstream.Msg) { s.heard(branch, m) })
if err != nil {
s.logf("source-moved: the consumer of the branch %s is not bound, so its source revisions are not taken in until the next registration-changed or start: %v", branch, err)
s.logf("source-moved: the consumer of the branch %s is not bound, so its later source revisions are not heard until the next registration-changed or start; its current one is read through the gate's source: %v", branch, err)
read = append(read, branch)
continue
}
s.bound[branch] = stop
s.logf("source-moved: the consumer of the branch %s is bound", branch)
if start {
read = append(read, branch)
}
}
return read
}
// readSource reads a branch's current source revision through the gate's verb source and takes it in as a heard
// source-moved of that revision would be (takeIn), when the view keeps no revision of the branch at or above it (H-A-129,
// H-A-130: index.SourceToTake).
func (s *sourceMoved) readSource(branch string) {
if s.source == nil {
return
}
a, err := s.source(s.ctx, branch)
if err != nil {
s.logf("source-moved: the gate's source revision of %s is not read, so it is taken in only once its source-moved is heard: %v", branch, err)
return
}
if a.Revision <= 0 {
s.logf("source-moved: the gate keeps no source revision of %s yet", branch)
return
}
s.again.Lock()
defer s.again.Unlock()
e, take, err := s.ix.SourceToTake(s.ctx, branch, a)
if err != nil {
s.logf("source-moved: the catalog revisions of %s are not read, so the gate's source revision %d of it is not taken in now: %v", branch, a.Revision, err)
return
}
if !take {
return
}
s.logf("source-moved: source revision %d of %s is read through the gate's source and taken in now (start %t)", a.Revision, branch, e.Start)
switch err := s.takeIn(branch, e); {
case errors.Is(err, index.ErrOldForm):
s.logf("source-moved: the gate's source revision %d of %s is refused, the old form: %v", e.Source, branch, err)
case err != nil:
s.logf("source-moved: the gate's source revision %d of %s is not taken in now: %v", e.Source, branch, err)
}
}
@@ -109,10 +221,7 @@ func (s *sourceMoved) heard(branch string, m jetstream.Msg) {
}
s.again.Lock()
defer s.again.Unlock()
behind, err := s.ix.HoldBehind(s.ctx, e)
if !behind && err == nil {
_, err = s.ix.TakeInSource(s.ctx, e)
}
err := s.takeIn(branch, e)
switch {
case errors.Is(err, index.ErrOldForm):
s.logf("source-moved of %s on %s is refused, the old form: %v", branch, m.Subject(), err)
@@ -123,14 +232,25 @@ func (s *sourceMoved) heard(branch string, m jetstream.Msg) {
return
case err != nil:
s.logf("source-moved: source revision %d of %s is held, not taken in: %v", e.Source, branch, err)
case behind:
s.logf("source-moved: source revision %d of %s is held behind the branch's earlier held ones", e.Source, branch)
}
if err := m.Ack(); err != nil {
s.logf("source-moved: source revision %d of %s is taken in or held and its acknowledgement failed: %v", e.Source, branch, err)
}
}
// takeIn takes one source-moved of a branch in, or holds it behind the branch's earlier held ones, so its source revisions
// are taken in in their order; the caller holds s.again. A heard one and one read through the gate's source go this one way.
func (s *sourceMoved) takeIn(branch string, e meshdelivery.SourceMovedEvent) error {
behind, err := s.ix.HoldBehind(s.ctx, e)
if !behind && err == nil {
_, err = s.ix.TakeInSource(s.ctx, e)
}
if behind && err == nil {
s.logf("source-moved: source revision %d of %s is held behind the branch's earlier held ones", e.Source, branch)
}
return err
}
// takeAgain takes every held source revision in again from the state held, each branch in its order, on the indexer's own room
// (H-A-50) and at its start (H-A-114).
func (s *sourceMoved) takeAgain() {
@@ -2,14 +2,20 @@ package main
import (
"context"
"errors"
"fmt"
"reflect"
"slices"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-indexer/internal/index"
)
@@ -88,3 +94,143 @@ func TestAHeldSourceMovedIsSkippedByHand(t *testing.T) {
t.Errorf("held %+v after the skip, want source 5 alone", hs)
}
}
// revisionObjects is the catalog revisions' object bucket in memory.
type revisionObjects map[string][]byte
func (o revisionObjects) Put(_ context.Context, name string, data []byte) error {
o[name] = data
return nil
}
func (o revisionObjects) Get(_ context.Context, name string) ([]byte, error) {
if d, ok := o[name]; ok {
return d, nil
}
return nil, index.ErrNoObject
}
func (o revisionObjects) Delete(_ context.Context, name string) error { delete(o, name); return nil }
func (o revisionObjects) Names(context.Context) ([]string, error) {
var out []string
for n := range o {
out = append(out, n)
}
return out, nil
}
func (o revisionObjects) Fill(context.Context) (int64, int64, error) { return 0, 1 << 30, nil }
// Design 47 H §743, §768, H-A-129: at a clean state no branch's consumer is there to bind; the start reads each watched
// branch's source revision through the gate's verb source and takes it in, and a failed bind at a registration-changed reads
// it again without taking it in twice. A start whose view keeps the revision reads it and takes nothing in.
func TestAStartWithNoConsumerTakesTheGatesSourceRevisionIn(t *testing.T) {
ctx := context.Background()
var taken []string
objects := revisionObjects{}
ix := &index.Indexer{Module: testIndexer, Revisions: objects, Modules: index.MemoryStore(), Graph: index.MemoryStore(),
Held: &memStore{vals: map[string][]byte{}}, Logf: func(string, ...any) {},
SayTakeIn: func(_ context.Context, event, id string, _ any) error {
taken = append(taken, event+" "+id)
return nil
}}
var asked []string
newSources := func() *sourceMoved {
return &sourceMoved{ctx: ctx, ix: ix,
branches: func(context.Context) ([]string, error) { return []string{"main"}, nil },
bind: func(context.Context, string, func(jetstream.Msg)) (func(), error) {
return nil, errors.New("nats: API error: code=404 err_code=10014 description=consumer not found")
},
source: func(_ context.Context, branch string) (meshdelivery.SourceAnswer, error) {
asked = append(asked, branch)
return meshdelivery.SourceAnswer{Branch: branch, Revision: 1, Call: "0123456789abcdef"}, nil
},
logf: func(string, ...any) {}}
}
s := newSources()
s.refresh() // the start
if _, ok := objects[index.BranchObject("main", 1)]; !ok || !reflect.DeepEqual(taken, []string{"take-in.taken " + index.TakenID("main", 1)}) {
t.Fatalf("the start kept %v and said %v; want main.1 taken in from the gate's source", objects, taken)
}
s.refresh() // a registration-changed, the consumer still not there
newSources().refresh() // a start whose view keeps the revision
if !reflect.DeepEqual(asked, []string{"main", "main", "main"}) || len(taken) != 1 {
t.Errorf("asked the gate for %v and said %v; want main asked at each and taken in once", asked, taken)
}
}
// fakeBranchJS is the bus as bindBranch uses it: the source-moved stream's consumers by name, and the consumers made.
type fakeBranchJS struct {
jetstream.JetStream
held map[string]*fakeBranchConsumer
made []jetstream.ConsumerConfig
}
func (f *fakeBranchJS) Consumer(_ context.Context, stream, name string) (jetstream.Consumer, error) {
if c, ok := f.held[stream+"/"+name]; ok {
return c, nil
}
return nil, jetstream.ErrConsumerNotFound
}
func (f *fakeBranchJS) CreateConsumer(_ context.Context, stream string, cfg jetstream.ConsumerConfig) (jetstream.Consumer, error) {
f.made = append(f.made, cfg)
c := &fakeBranchConsumer{}
f.held[stream+"/"+cfg.Durable] = c
return c, nil
}
// fakeBranchConsumer hands one source-moved to the hearing as soon as it is consumed.
type fakeBranchConsumer struct {
jetstream.Consumer
heard int
}
type fakeConsumeContext struct{ jetstream.ConsumeContext }
func (fakeConsumeContext) Stop() {}
func (c *fakeBranchConsumer) Consume(handle jetstream.MessageHandler, _ ...jetstream.PullConsumeOpt) (jetstream.ConsumeContext, error) {
c.heard++
handle(msgOf{f: &fakeMsg{subject: "mesh.seat.mesh-delivery.event.source-moved.main.2"}})
return fakeConsumeContext{}, nil
}
// Design 47 H §743, H-A-134: a branch's durable the bus does not hold the indexer makes, as a module's seat-event consumer is
// made (SeatEventConsumerConfig), named <node>_<indexer>_<branch>, filtered to the branch, one message at a time, and then
// hears on it; one the bus holds is bound, never made again.
func TestABranchDurableNotThereIsMadeAndHeard(t *testing.T) {
cred := process.Account{Node: "n1", Module: "mesh-delivery-indexer"}
for _, tc := range []struct {
name string
held bool
}{{"not there: made", false}, {"there: bound", true}} {
t.Run(tc.name, func(t *testing.T) {
js := &fakeBranchJS{held: map[string]*fakeBranchConsumer{}}
name := "n1_mesh-delivery-indexer_" + keys.EscapeName("main")
var existing *fakeBranchConsumer
if tc.held {
existing = &fakeBranchConsumer{}
js.held[meshdelivery.StreamSourceMoved+"/"+name] = existing
}
var heard []string
stop, err := bindBranch(js, cred, func(string, ...any) {})(context.Background(), "main", func(m jetstream.Msg) { heard = append(heard, m.Subject()) })
if err != nil {
t.Fatal(err)
}
stop()
if len(heard) != 1 {
t.Fatalf("heard %v, want the one source-moved the consumer hands", heard)
}
if tc.held {
if len(js.made) != 0 || existing.heard != 1 {
t.Fatalf("made %v and heard the held one %d times; want it bound, not made again", js.made, existing.heard)
}
return
}
want := cred.SeatEventConsumerConfig([]string{meshdelivery.SubjectSourceMoved(keys.EscapeName("main"), "*")}, false)
want.Durable, want.MaxAckPending = name, 1
want.AckWait, want.MaxDeliver, want.DeliverPolicy = 30*time.Second, 5, jetstream.DeliverNewPolicy
if len(js.made) != 1 || !reflect.DeepEqual(js.made[0], want) {
t.Fatalf("made %+v, want once %+v", js.made, want)
}
})
}
}
@@ -337,6 +337,11 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int
} else if len(removed) > 0 {
ix.logf("pins: %d revisions removed (%s), no record pins them and none is current", len(removed), strings.Join(removed, ", "))
}
if dropped, err := ix.dropEarlierFormsLocked(ctx); err != nil {
ix.logf("catalog graph: the revisions of the earlier forms (trunk.*, branch.*) are not dropped: %v", err)
} else if len(dropped) > 0 {
ix.logf("catalog graph: %d revisions of the earlier forms dropped (%s), no record pins them; each branch's newest revision is its current one", len(dropped), strings.Join(dropped, ", "))
}
if _, err := ix.removeUnlisted(ctx); err != nil {
ix.logf("catalog graph: the entries nothing lists or names any more are not removed: %v", err) // at the first revision after none does (§675)
}
@@ -253,25 +253,10 @@ func (ix *Indexer) judgePinsLocked(ctx context.Context) ([]string, error) {
return nil, err
}
current := CurrentRevisions(names)
pinned := map[string]bool{}
pinKeys, err := ix.Pins.Keys(ctx)
pinKeys, pinned, err := ix.pinnedObjects(ctx)
if err != nil {
return nil, err
}
for _, k := range pinKeys {
object, ok := pinObject(k)
if !ok {
continue
}
v, _, found, err := ix.Pins.Get(ctx, k)
if err != nil {
return nil, err
}
var pv PinValue
if found && json.Unmarshal(v, &pv) == nil && len(pv.Records) > 0 {
pinned[object] = true
}
}
var removed []string
for _, n := range names {
if _, _, ok := RevisionGroup(n); !ok || pinned[n] {
@@ -301,6 +286,72 @@ func (ix *Indexer) judgePinsLocked(ctx context.Context) ([]string, error) {
return removed, nil
}
// pinnedObjects are the state pins' keys and the revision objects a record pins.
func (ix *Indexer) pinnedObjects(ctx context.Context) ([]string, map[string]bool, error) {
pinned := map[string]bool{}
pinKeys, err := ix.Pins.Keys(ctx)
if err != nil {
return nil, nil, err
}
for _, k := range pinKeys {
object, ok := pinObject(k)
if !ok {
continue
}
v, _, found, err := ix.Pins.Get(ctx, k)
if err != nil {
return nil, nil, err
}
var pv PinValue
if found && json.Unmarshal(v, &pv) == nil && len(pv.Records) > 0 {
pinned[object] = true
}
}
return pinKeys, pinned, nil
}
// earlierForm says whether a revision object is of the earlier model's forms, trunk.<n> or branch.<branch>.<n>, beside the
// branch-keyed <branch>.<n> of design 47 H §743 (H-A-135).
func earlierForm(name string) bool {
if _, _, ok := RevisionGroup(name); !ok {
return false
}
return strings.HasPrefix(name, "trunk.") || strings.HasPrefix(name, "branch.")
}
// dropEarlierFormsLocked drops every revision object of the earlier forms (earlierForm) no record pins, at a branch's take-in
// (design 47 H §743, H-A-135): the branch's newest revision is its current one, and an earlier trunk revision kept beside it
// would be the newest trunk revision a reader hears. It drops none while the pins do not count yet; the caller holds keepMu.
func (ix *Indexer) dropEarlierFormsLocked(ctx context.Context) ([]string, error) {
if ix.PinsPending != nil {
if pending, err := ix.PinsPending(ctx); err != nil || pending != 0 {
return nil, err
}
}
names, err := ix.Revisions.Names(ctx)
if err != nil {
return nil, err
}
pinned := map[string]bool{}
if ix.Pins != nil {
if _, pinned, err = ix.pinnedObjects(ctx); err != nil {
return nil, err
}
}
var dropped []string
for _, n := range names {
if !earlierForm(n) || pinned[n] {
continue
}
if err := ix.Revisions.Delete(ctx, n); err != nil {
return dropped, fmt.Errorf("%s: %w", n, err)
}
dropped = append(dropped, n)
}
sort.Strings(dropped)
return dropped, nil
}
// PinDrop drops one pin of the state pins by hand, by its key <branch>.<revision> (indexer_pin_drop, design 47 H §765, H-A-57):
// every record's, for a pin that names no running pipeline; the finding pin-missing of the key is cleared, and the removal
// rule then applies.
@@ -61,7 +61,8 @@ type RevisionSummary struct {
Heads string `json:"heads,omitempty"`
At *time.Time `json:"at,omitempty"`
Entries int `json:"entries"`
// Current says it is the trunk's newest revision, which is kept whatever pins it.
// Current says it is the newest revision of its watched branch (CurrentRevisions, §579; H §743, H-A-135: no longer the
// newest trunk-kind revision over every branch, and never one of the earlier forms), which is kept whatever pins it.
Current bool `json:"current,omitempty"`
}
@@ -84,7 +85,6 @@ func ListRevisions(ctx context.Context, o Objects, candidates bool) (RevisionsAn
return out, fmt.Errorf("the objects %s cannot be listed: %w", CatalogRevisionsObjects, err)
}
listed := map[string]bool{}
var newest int64
entries := []string{}
for _, n := range names {
if strings.HasPrefix(n, "entry.") {
@@ -103,17 +103,16 @@ func ListRevisions(ctx context.Context, o Objects, candidates bool) (RevisionsAn
for _, ref := range rev.Entries {
listed[EntryObject(ref.Digest)] = true
}
if rev.Kind == "trunk" && rev.Revision > newest {
newest = rev.Revision
}
if candidates && rev.Kind != "candidate" {
continue
}
out.Revisions = append(out.Revisions, RevisionSummary{Name: n, Kind: rev.Kind, Revision: rev.Revision, Source: rev.Source,
Group: rev.Group, Heads: rev.Heads, At: rev.At, Entries: len(rev.Entries)})
}
current := CurrentRevisions(names)
for i := range out.Revisions {
out.Revisions[i].Current = out.Revisions[i].Kind == "trunk" && out.Revisions[i].Revision == newest
_, cur := current[out.Revisions[i].Name]
out.Revisions[i].Current = cur && out.Revisions[i].Kind == "trunk" && !earlierForm(out.Revisions[i].Name)
}
slices.SortFunc(out.Revisions, func(a, b RevisionSummary) int { return strings.Compare(a.Name, b.Name) })
out.Entries = len(entries)
@@ -142,3 +142,37 @@ func (ix *Indexer) sayTaken(ctx context.Context, t meshcatalog.TakeInTaken) erro
func PendingID(branch string, source int64) string {
return takein.TakeInPendingEvent + "." + keys.EscapeName(branch) + "." + strconv.FormatInt(source, 10)
}
// SourceToTake is the source-moved a read of the gate's source revision of a branch, through the seat mesh-delivery's verb
// source, takes in as a heard one of that revision would be (design 47 H H-A-129), and false when the view keeps a revision of
// the branch at or above it, so a revision is never taken in twice. It is the branch's first take-in, start true, exactly
// when the view keeps no revision of the branch; otherwise its movers are judged against the newest kept (H-A-130). Every
// repository the answer names is a head: the movers are the fingerprint changes alone.
func (ix *Indexer) SourceToTake(ctx context.Context, branch string, a meshdelivery.SourceAnswer) (meshdelivery.SourceMovedEvent, bool, error) {
e := meshdelivery.SourceMovedEvent{V: 1, Branch: branch, Source: a.Revision, Call: a.Call,
Heads: make([]meshdelivery.SourceMovedEventHeadsItem, 0, len(a.Repositories))}
if a.Revision <= 0 {
return e, false, nil
}
if ix.Revisions == nil {
return e, false, errors.New("the indexer keeps no catalog revisions")
}
names, err := ix.Revisions.Names(ctx)
if err != nil {
return e, false, fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err)
}
var kept int64
for _, n := range names {
if r, ok := BranchRevisionOf(branch, n); ok && r > kept {
kept = r
}
}
if kept >= a.Revision {
return e, false, nil
}
e.Start = kept == 0
for _, r := range a.Repositories {
e.Heads = append(e.Heads, meshdelivery.SourceMovedEventHeadsItem{Repository: r.Repository, Commit: r.Head})
}
return e, true, nil
}
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"testing"
@@ -121,6 +122,39 @@ func TestAFirstSourceRevisionIsTakenInWithNoMover(t *testing.T) {
}
}
// Design 47 H §743, H-A-135: at a branch's take-in the earlier model's revisions (trunk.<n>, branch.<branch>.<n>) no record
// pins are dropped, one a record pins stays, and the branch's newest revision is the current one, no longer the newest trunk.
func TestATakeInDropsTheEarlierFormsNoRecordPins(t *testing.T) {
ctx := context.Background()
ix, objects, _, f := takeInIndexer(t)
for _, n := range []string{"trunk.527", "trunk.528", "branch.master.1"} {
_, rev, _ := RevisionGroup(n)
raw, _ := json.Marshal(meshcatalog.CatalogRevision{Revision: rev, Kind: "trunk", Source: 1000 + rev})
if err := objects.Put(ctx, n, raw); err != nil {
t.Fatal(err)
}
}
if err := ix.pin(ctx, "trunk.527", "pipeline.old"); err != nil {
t.Fatal(err)
}
start := meshdelivery.SourceMovedEvent{V: 1, Branch: "main", Source: 1, Call: "seed.operator.20261120", Start: true,
Heads: []meshdelivery.SourceMovedEventHeadsItem{{Repository: f.Repository, Commit: f.Merge}}}
if _, err := ix.TakeInSource(ctx, start); err != nil {
t.Fatal(err)
}
a, err := ListRevisions(ctx, objects, false)
if err != nil {
t.Fatal(err)
}
var got []string
for _, r := range a.Revisions {
got = append(got, fmt.Sprintf("%s current=%t", r.Name, r.Current))
}
if want := []string{"main.1 current=true", "trunk.527 current=false"}; !reflect.DeepEqual(got, want) {
t.Errorf("kept %v, want %v: the unpinned earlier forms dropped, the pinned one kept, main.1 current", got, want)
}
}
// Design 47 H §769: the take-in of a source revision moving the catalogue repository reads `predefined` from that commit's
// trees, and one changing a predefined key rewrites it.
func TestATakeInOfTheCatalogsCommitRewritesPredefined(t *testing.T) {
@@ -222,3 +256,35 @@ func TestATakeInOnAFullStoreIsHeldAndSaidOnce(t *testing.T) {
t.Errorf("held = %+v after the skip, want none", held)
}
}
// Design 47 H §743, §768, H-A-129, H-A-130: the gate's source revision read through its verb source is taken in as a heard
// source-moved of it would be: start true when the view keeps no revision of the branch, every repository it names a head;
// one the view keeps at or above is not taken again; a later one is judged against the newest kept, start false.
func TestASourceRevisionReadFromTheGateIsTakenInOnce(t *testing.T) {
ctx := context.Background()
ix, _, said, f := takeInIndexer(t)
answer := meshdelivery.SourceAnswer{Branch: "main", Revision: 1, Call: "0123456789abcdef",
Repositories: []meshdelivery.SourceHead{{Repository: f.Repository, Trunk: "main", Head: f.Merge}}}
e, take, err := ix.SourceToTake(ctx, "main", answer)
want := meshdelivery.SourceMovedEvent{V: 1, Branch: "main", Source: 1, Call: answer.Call, Start: true,
Heads: []meshdelivery.SourceMovedEventHeadsItem{{Repository: f.Repository, Commit: f.Merge}}}
if err != nil || !take || !reflect.DeepEqual(e, want) {
t.Fatalf("a branch the view keeps none of answered %+v, %t (%v); want %+v taken", e, take, err, want)
}
if taken, err := ix.TakeInSource(ctx, e); err != nil || len(taken.Entries) != 0 || taken.Revision != 1 {
t.Fatalf("the first revision read from the gate said %+v (%v), want revision 1 with no mover", taken, err)
}
if _, take, err := ix.SourceToTake(ctx, "main", answer); take || err != nil {
t.Errorf("a revision the view keeps is taken again (%v)", err)
}
if _, take, _ := ix.SourceToTake(ctx, "nox-mesh", meshdelivery.SourceAnswer{Branch: "nox-mesh"}); take {
t.Error("a branch the gate keeps no revision of is taken in")
}
answer.Revision, answer.Call = 2, "1123456789abcdef"
if e, take, err := ix.SourceToTake(ctx, "main", answer); err != nil || !take || e.Start || e.Source != 2 {
t.Errorf("a later revision answered %+v, %t (%v); want revision 2 taken, start false", e, take, err)
}
if len(*said) != 1 {
t.Errorf("said %d take-in.taken, want 1", len(*said))
}
}
@@ -182,7 +182,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
// Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths:
// ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a
// commit is in another (issue 1900).
if want := []string{"seat:git.trees", "seat:git.tags", "seat:go-package-registry.versions", "seat:npm-package-registry.versions", "seat:git.file", "seat:git.contains", "seat:git.branch", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps",
if want := []string{"seat:git.trees", "seat:git.tags", "seat:go-package-registry.versions", "seat:npm-package-registry.versions", "seat:git.file", "seat:git.contains", "seat:git.branch", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps",
"seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) {
t.Errorf("invokes = %v, want %v", m.Invokes, want)
}
@@ -152,6 +152,7 @@
"seat:git.file",
"seat:git.contains",
"seat:git.branch",
"seat:mesh-delivery.source",
"seat:mesh-module-assigner.assignments",
"seat:mesh-module-assigner.placement",
"seat:mesh-hand-acts.hand-act",