Compare commits
6
Commits
go/v0.1.74
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
403fb3068b | ||
|
|
e54bf7f346 | ||
|
|
f95a164941 | ||
|
|
0cf419619e | ||
|
|
8b6f6b22d4 | ||
|
|
9cddb5921a |
@@ -357,7 +357,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
registry.Pruned = ix.PruneBranchBuilds
|
||||
ix.Registry = registry
|
||||
// The gate's source revisions, on one durable per watched branch (design 47 H §743, H-A-90), bound below.
|
||||
sources := &sourceMoved{ctx: ctx, ix: ix, branches: watchedBranches(registry), bind: bindBranch(js, cred, logger.Printf), logf: logger.Printf}
|
||||
sources := &sourceMoved{ctx: ctx, ix: ix, branches: watchedBranches(registry), bind: bindBranch(js, cred, logger.Printf), source: sourceVia(calls.Call), logf: logger.Printf}
|
||||
defer sources.stop()
|
||||
named.Changed = ix.RemoveUnlisted
|
||||
// The first fill of the versions (design 47 G §372): at the start, and again at each catalog revision kept until it is made.
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
@@ -20,17 +21,22 @@ import (
|
||||
// the event's stream, named <node>_<indexer>_<branch>, the branch escaped as a key's name is (H-A-90), each delivering one message
|
||||
// at a time (the grants part makes it so, H-A-30), acknowledged once its catalog revision and take-in.taken are written. The
|
||||
// watched branches are the registrations'; a newly watched branch's consumer is bound on the indexer's own
|
||||
// registration-changed, not only at its start.
|
||||
// registration-changed, not only at its start. At its start, and again whenever a branch's consumer cannot be bound, the
|
||||
// branch's current source revision is read through the seat mesh-delivery's verb source and taken in as a heard source-moved
|
||||
// of it would be, when the view keeps no revision of the branch at or above it (H-A-129, H-A-130): at a clean state the
|
||||
// consumers are made only once the indexer answers its modules view, which it does only after its first take-in.
|
||||
type sourceMoved struct {
|
||||
ctx context.Context
|
||||
ix *index.Indexer
|
||||
branches func(context.Context) ([]string, error) // the watched branches the registrations name
|
||||
bind func(ctx context.Context, branch string, each func(jetstream.Msg)) (stop func(), err error)
|
||||
source func(ctx context.Context, branch string) (meshdelivery.SourceAnswer, error) // the gate's source revision of a branch
|
||||
logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
bound map[string]func() // by branch: the consumer's stop
|
||||
again sync.Mutex // one take-in again at a time
|
||||
mu sync.Mutex
|
||||
bound map[string]func() // by branch: the consumer's stop
|
||||
started bool // the start's refresh ran: every branch's source was read once
|
||||
again sync.Mutex // one take-in again at a time
|
||||
}
|
||||
|
||||
// watchedBranches are the branches every registration watches, each once, sorted.
|
||||
@@ -53,10 +59,40 @@ func watchedBranches(r *index.Registry) func(context.Context) ([]string, error)
|
||||
}
|
||||
}
|
||||
|
||||
// bindBranch binds the indexer's durable of one watched branch on the stream of source-moved through mesh-sdk.
|
||||
// sourceVia reads a watched branch's current source revision through the seat mesh-delivery's verb source, as the
|
||||
// indexer's own account (H-A-129): a call where an answer is needed. An answer naming why it could not be read is that
|
||||
// failure. The branch is given by its name, as the gate reads it (it escapes the name for its key itself).
|
||||
func sourceVia(call toolCaller) func(context.Context, string) (meshdelivery.SourceAnswer, error) {
|
||||
verb := "seat:" + meshdelivery.Seat + "." + meshdelivery.VerbSource
|
||||
return func(_ context.Context, branch string) (meshdelivery.SourceAnswer, error) {
|
||||
var a meshdelivery.SourceAnswer
|
||||
raw, err := call(verb, meshdelivery.SourceInput{Branch: branch})
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return a, fmt.Errorf("the answer of %s is not its seat's answer: %w", verb, err)
|
||||
}
|
||||
if a.Error != "" {
|
||||
return a, fmt.Errorf("%s: %s", verb, a.Error)
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
}
|
||||
|
||||
// bindBranch binds the indexer's durable of one watched branch on the stream of source-moved through mesh-sdk. One the bus does
|
||||
// not hold the indexer makes itself, as its own kept consumer, and then binds (design 47 H §743, H-A-134: the grants part
|
||||
// composes none until H4b); one the bus holds is bound as it stands, never made again.
|
||||
func bindBranch(js jetstream.JetStream, cred process.Account, logf func(string, ...any)) func(context.Context, string, func(jetstream.Msg)) (func(), error) {
|
||||
return func(ctx context.Context, branch string, each func(jetstream.Msg)) (func(), error) {
|
||||
cons, err := cred.NamedEventConsumer(ctx, js, meshdelivery.StreamSourceMoved, keys.EscapeName(branch))
|
||||
if errors.Is(err, jetstream.ErrConsumerNotFound) {
|
||||
cfg := branchConsumerConfig(cred, branch)
|
||||
if cons, err = js.CreateConsumer(ctx, meshdelivery.StreamSourceMoved, cfg); err != nil {
|
||||
return nil, fmt.Errorf("the consumer %s on %s is not there and is not made: %w", cfg.Durable, meshdelivery.StreamSourceMoved, err)
|
||||
}
|
||||
logf("source-moved: the consumer %s on %s was not there: the indexer made it (H-A-134)", cfg.Durable, meshdelivery.StreamSourceMoved)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -68,31 +104,107 @@ func bindBranch(js jetstream.JetStream, cred process.Account, logf func(string,
|
||||
}
|
||||
}
|
||||
|
||||
// branchConsumerConfig is the indexer's durable of one watched branch as the mesh gives a module's seat-event consumer
|
||||
// (mesh-sdk's SeatEventConsumerConfig), named <node>_<indexer>_<branch> as NamedEventConsumer binds it, filtered to the
|
||||
// branch's source-moved and delivering one message at a time, with the ack wait of 30 s, the at most 5 deliveries and the
|
||||
// start from now the grants part gives a module's consumer on a seat event's stream kept by age (mesh-grants buscompose
|
||||
// heardConsumer): the branch's current source revision is read at the start through the gate's source (H-A-129, H-A-130), so
|
||||
// its past source-moved are never taken in again (design 47 H §743, H-A-90, H-A-134).
|
||||
func branchConsumerConfig(cred process.Account, branch string) jetstream.ConsumerConfig {
|
||||
esc := keys.EscapeName(branch)
|
||||
cfg := cred.SeatEventConsumerConfig([]string{meshdelivery.SubjectSourceMoved(esc, "*")}, false)
|
||||
cfg.Durable = cred.EventConsumerName() + "_" + esc
|
||||
cfg.MaxAckPending = 1
|
||||
cfg.AckWait, cfg.MaxDeliver = branchAckWait, branchMaxDeliver
|
||||
cfg.DeliverPolicy = jetstream.DeliverNewPolicy
|
||||
return cfg
|
||||
}
|
||||
|
||||
// The ack wait and the deliveries of a branch's durable, as the grants part gives a module's consumers (H-A-134, H-A-133).
|
||||
const (
|
||||
branchAckWait = 30 * time.Second
|
||||
branchMaxDeliver = 5
|
||||
)
|
||||
|
||||
// refresh binds the consumer of every watched branch not bound yet: at the start and on each registration-changed. A consumer
|
||||
// not there yet is said and bound at the next registration-changed or start.
|
||||
// not there yet is said and bound at the next registration-changed or start. The start's refresh reads every watched branch's
|
||||
// source revision, a later one each branch whose consumer could not be bound (H-A-129), after the binds.
|
||||
func (s *sourceMoved) refresh() {
|
||||
branches, err := s.branches(s.ctx)
|
||||
if err != nil {
|
||||
s.logf("source-moved: the watched branches are not read, so no new branch's consumer is bound: %v", err)
|
||||
return
|
||||
}
|
||||
for _, branch := range s.bindAll(branches) {
|
||||
s.readSource(branch)
|
||||
}
|
||||
}
|
||||
|
||||
// bindAll binds the branches not bound yet and answers those whose source revision is to be read now: at the start every
|
||||
// watched branch, later each one whose consumer could not be bound.
|
||||
func (s *sourceMoved) bindAll(branches []string) (read []string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.bound == nil {
|
||||
s.bound = map[string]func(){}
|
||||
}
|
||||
start := !s.started
|
||||
s.started = true
|
||||
for _, b := range branches {
|
||||
if _, ok := s.bound[b]; ok {
|
||||
if start {
|
||||
read = append(read, b)
|
||||
}
|
||||
continue
|
||||
}
|
||||
branch := b
|
||||
stop, err := s.bind(s.ctx, branch, func(m jetstream.Msg) { s.heard(branch, m) })
|
||||
if err != nil {
|
||||
s.logf("source-moved: the consumer of the branch %s is not bound, so its source revisions are not taken in until the next registration-changed or start: %v", branch, err)
|
||||
s.logf("source-moved: the consumer of the branch %s is not bound, so its later source revisions are not heard until the next registration-changed or start; its current one is read through the gate's source: %v", branch, err)
|
||||
read = append(read, branch)
|
||||
continue
|
||||
}
|
||||
s.bound[branch] = stop
|
||||
s.logf("source-moved: the consumer of the branch %s is bound", branch)
|
||||
if start {
|
||||
read = append(read, branch)
|
||||
}
|
||||
}
|
||||
return read
|
||||
}
|
||||
|
||||
// readSource reads a branch's current source revision through the gate's verb source and takes it in as a heard
|
||||
// source-moved of that revision would be (takeIn), when the view keeps no revision of the branch at or above it (H-A-129,
|
||||
// H-A-130: index.SourceToTake).
|
||||
func (s *sourceMoved) readSource(branch string) {
|
||||
if s.source == nil {
|
||||
return
|
||||
}
|
||||
a, err := s.source(s.ctx, branch)
|
||||
if err != nil {
|
||||
s.logf("source-moved: the gate's source revision of %s is not read, so it is taken in only once its source-moved is heard: %v", branch, err)
|
||||
return
|
||||
}
|
||||
if a.Revision <= 0 {
|
||||
s.logf("source-moved: the gate keeps no source revision of %s yet", branch)
|
||||
return
|
||||
}
|
||||
s.again.Lock()
|
||||
defer s.again.Unlock()
|
||||
e, take, err := s.ix.SourceToTake(s.ctx, branch, a)
|
||||
if err != nil {
|
||||
s.logf("source-moved: the catalog revisions of %s are not read, so the gate's source revision %d of it is not taken in now: %v", branch, a.Revision, err)
|
||||
return
|
||||
}
|
||||
if !take {
|
||||
return
|
||||
}
|
||||
s.logf("source-moved: source revision %d of %s is read through the gate's source and taken in now (start %t)", a.Revision, branch, e.Start)
|
||||
switch err := s.takeIn(branch, e); {
|
||||
case errors.Is(err, index.ErrOldForm):
|
||||
s.logf("source-moved: the gate's source revision %d of %s is refused, the old form: %v", e.Source, branch, err)
|
||||
case err != nil:
|
||||
s.logf("source-moved: the gate's source revision %d of %s is not taken in now: %v", e.Source, branch, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -109,10 +221,7 @@ func (s *sourceMoved) heard(branch string, m jetstream.Msg) {
|
||||
}
|
||||
s.again.Lock()
|
||||
defer s.again.Unlock()
|
||||
behind, err := s.ix.HoldBehind(s.ctx, e)
|
||||
if !behind && err == nil {
|
||||
_, err = s.ix.TakeInSource(s.ctx, e)
|
||||
}
|
||||
err := s.takeIn(branch, e)
|
||||
switch {
|
||||
case errors.Is(err, index.ErrOldForm):
|
||||
s.logf("source-moved of %s on %s is refused, the old form: %v", branch, m.Subject(), err)
|
||||
@@ -123,14 +232,25 @@ func (s *sourceMoved) heard(branch string, m jetstream.Msg) {
|
||||
return
|
||||
case err != nil:
|
||||
s.logf("source-moved: source revision %d of %s is held, not taken in: %v", e.Source, branch, err)
|
||||
case behind:
|
||||
s.logf("source-moved: source revision %d of %s is held behind the branch's earlier held ones", e.Source, branch)
|
||||
}
|
||||
if err := m.Ack(); err != nil {
|
||||
s.logf("source-moved: source revision %d of %s is taken in or held and its acknowledgement failed: %v", e.Source, branch, err)
|
||||
}
|
||||
}
|
||||
|
||||
// takeIn takes one source-moved of a branch in, or holds it behind the branch's earlier held ones, so its source revisions
|
||||
// are taken in in their order; the caller holds s.again. A heard one and one read through the gate's source go this one way.
|
||||
func (s *sourceMoved) takeIn(branch string, e meshdelivery.SourceMovedEvent) error {
|
||||
behind, err := s.ix.HoldBehind(s.ctx, e)
|
||||
if !behind && err == nil {
|
||||
_, err = s.ix.TakeInSource(s.ctx, e)
|
||||
}
|
||||
if behind && err == nil {
|
||||
s.logf("source-moved: source revision %d of %s is held behind the branch's earlier held ones", e.Source, branch)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// takeAgain takes every held source revision in again from the state held, each branch in its order, on the indexer's own room
|
||||
// (H-A-50) and at its start (H-A-114).
|
||||
func (s *sourceMoved) takeAgain() {
|
||||
|
||||
@@ -2,14 +2,20 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
@@ -88,3 +94,143 @@ func TestAHeldSourceMovedIsSkippedByHand(t *testing.T) {
|
||||
t.Errorf("held %+v after the skip, want source 5 alone", hs)
|
||||
}
|
||||
}
|
||||
|
||||
// revisionObjects is the catalog revisions' object bucket in memory.
|
||||
type revisionObjects map[string][]byte
|
||||
|
||||
func (o revisionObjects) Put(_ context.Context, name string, data []byte) error {
|
||||
o[name] = data
|
||||
return nil
|
||||
}
|
||||
func (o revisionObjects) Get(_ context.Context, name string) ([]byte, error) {
|
||||
if d, ok := o[name]; ok {
|
||||
return d, nil
|
||||
}
|
||||
return nil, index.ErrNoObject
|
||||
}
|
||||
func (o revisionObjects) Delete(_ context.Context, name string) error { delete(o, name); return nil }
|
||||
func (o revisionObjects) Names(context.Context) ([]string, error) {
|
||||
var out []string
|
||||
for n := range o {
|
||||
out = append(out, n)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (o revisionObjects) Fill(context.Context) (int64, int64, error) { return 0, 1 << 30, nil }
|
||||
|
||||
// Design 47 H §743, §768, H-A-129: at a clean state no branch's consumer is there to bind; the start reads each watched
|
||||
// branch's source revision through the gate's verb source and takes it in, and a failed bind at a registration-changed reads
|
||||
// it again without taking it in twice. A start whose view keeps the revision reads it and takes nothing in.
|
||||
func TestAStartWithNoConsumerTakesTheGatesSourceRevisionIn(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
var taken []string
|
||||
objects := revisionObjects{}
|
||||
ix := &index.Indexer{Module: testIndexer, Revisions: objects, Modules: index.MemoryStore(), Graph: index.MemoryStore(),
|
||||
Held: &memStore{vals: map[string][]byte{}}, Logf: func(string, ...any) {},
|
||||
SayTakeIn: func(_ context.Context, event, id string, _ any) error {
|
||||
taken = append(taken, event+" "+id)
|
||||
return nil
|
||||
}}
|
||||
var asked []string
|
||||
newSources := func() *sourceMoved {
|
||||
return &sourceMoved{ctx: ctx, ix: ix,
|
||||
branches: func(context.Context) ([]string, error) { return []string{"main"}, nil },
|
||||
bind: func(context.Context, string, func(jetstream.Msg)) (func(), error) {
|
||||
return nil, errors.New("nats: API error: code=404 err_code=10014 description=consumer not found")
|
||||
},
|
||||
source: func(_ context.Context, branch string) (meshdelivery.SourceAnswer, error) {
|
||||
asked = append(asked, branch)
|
||||
return meshdelivery.SourceAnswer{Branch: branch, Revision: 1, Call: "0123456789abcdef"}, nil
|
||||
},
|
||||
logf: func(string, ...any) {}}
|
||||
}
|
||||
s := newSources()
|
||||
s.refresh() // the start
|
||||
if _, ok := objects[index.BranchObject("main", 1)]; !ok || !reflect.DeepEqual(taken, []string{"take-in.taken " + index.TakenID("main", 1)}) {
|
||||
t.Fatalf("the start kept %v and said %v; want main.1 taken in from the gate's source", objects, taken)
|
||||
}
|
||||
s.refresh() // a registration-changed, the consumer still not there
|
||||
newSources().refresh() // a start whose view keeps the revision
|
||||
if !reflect.DeepEqual(asked, []string{"main", "main", "main"}) || len(taken) != 1 {
|
||||
t.Errorf("asked the gate for %v and said %v; want main asked at each and taken in once", asked, taken)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeBranchJS is the bus as bindBranch uses it: the source-moved stream's consumers by name, and the consumers made.
|
||||
type fakeBranchJS struct {
|
||||
jetstream.JetStream
|
||||
held map[string]*fakeBranchConsumer
|
||||
made []jetstream.ConsumerConfig
|
||||
}
|
||||
|
||||
func (f *fakeBranchJS) Consumer(_ context.Context, stream, name string) (jetstream.Consumer, error) {
|
||||
if c, ok := f.held[stream+"/"+name]; ok {
|
||||
return c, nil
|
||||
}
|
||||
return nil, jetstream.ErrConsumerNotFound
|
||||
}
|
||||
|
||||
func (f *fakeBranchJS) CreateConsumer(_ context.Context, stream string, cfg jetstream.ConsumerConfig) (jetstream.Consumer, error) {
|
||||
f.made = append(f.made, cfg)
|
||||
c := &fakeBranchConsumer{}
|
||||
f.held[stream+"/"+cfg.Durable] = c
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// fakeBranchConsumer hands one source-moved to the hearing as soon as it is consumed.
|
||||
type fakeBranchConsumer struct {
|
||||
jetstream.Consumer
|
||||
heard int
|
||||
}
|
||||
|
||||
type fakeConsumeContext struct{ jetstream.ConsumeContext }
|
||||
|
||||
func (fakeConsumeContext) Stop() {}
|
||||
|
||||
func (c *fakeBranchConsumer) Consume(handle jetstream.MessageHandler, _ ...jetstream.PullConsumeOpt) (jetstream.ConsumeContext, error) {
|
||||
c.heard++
|
||||
handle(msgOf{f: &fakeMsg{subject: "mesh.seat.mesh-delivery.event.source-moved.main.2"}})
|
||||
return fakeConsumeContext{}, nil
|
||||
}
|
||||
|
||||
// Design 47 H §743, H-A-134: a branch's durable the bus does not hold the indexer makes, as a module's seat-event consumer is
|
||||
// made (SeatEventConsumerConfig), named <node>_<indexer>_<branch>, filtered to the branch, one message at a time, and then
|
||||
// hears on it; one the bus holds is bound, never made again.
|
||||
func TestABranchDurableNotThereIsMadeAndHeard(t *testing.T) {
|
||||
cred := process.Account{Node: "n1", Module: "mesh-delivery-indexer"}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
held bool
|
||||
}{{"not there: made", false}, {"there: bound", true}} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
js := &fakeBranchJS{held: map[string]*fakeBranchConsumer{}}
|
||||
name := "n1_mesh-delivery-indexer_" + keys.EscapeName("main")
|
||||
var existing *fakeBranchConsumer
|
||||
if tc.held {
|
||||
existing = &fakeBranchConsumer{}
|
||||
js.held[meshdelivery.StreamSourceMoved+"/"+name] = existing
|
||||
}
|
||||
var heard []string
|
||||
stop, err := bindBranch(js, cred, func(string, ...any) {})(context.Background(), "main", func(m jetstream.Msg) { heard = append(heard, m.Subject()) })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stop()
|
||||
if len(heard) != 1 {
|
||||
t.Fatalf("heard %v, want the one source-moved the consumer hands", heard)
|
||||
}
|
||||
if tc.held {
|
||||
if len(js.made) != 0 || existing.heard != 1 {
|
||||
t.Fatalf("made %v and heard the held one %d times; want it bound, not made again", js.made, existing.heard)
|
||||
}
|
||||
return
|
||||
}
|
||||
want := cred.SeatEventConsumerConfig([]string{meshdelivery.SubjectSourceMoved(keys.EscapeName("main"), "*")}, false)
|
||||
want.Durable, want.MaxAckPending = name, 1
|
||||
want.AckWait, want.MaxDeliver, want.DeliverPolicy = 30*time.Second, 5, jetstream.DeliverNewPolicy
|
||||
if len(js.made) != 1 || !reflect.DeepEqual(js.made[0], want) {
|
||||
t.Fatalf("made %+v, want once %+v", js.made, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -337,6 +337,11 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int
|
||||
} else if len(removed) > 0 {
|
||||
ix.logf("pins: %d revisions removed (%s), no record pins them and none is current", len(removed), strings.Join(removed, ", "))
|
||||
}
|
||||
if dropped, err := ix.dropEarlierFormsLocked(ctx); err != nil {
|
||||
ix.logf("catalog graph: the revisions of the earlier forms (trunk.*, branch.*) are not dropped: %v", err)
|
||||
} else if len(dropped) > 0 {
|
||||
ix.logf("catalog graph: %d revisions of the earlier forms dropped (%s), no record pins them; each branch's newest revision is its current one", len(dropped), strings.Join(dropped, ", "))
|
||||
}
|
||||
if _, err := ix.removeUnlisted(ctx); err != nil {
|
||||
ix.logf("catalog graph: the entries nothing lists or names any more are not removed: %v", err) // at the first revision after none does (§675)
|
||||
}
|
||||
|
||||
@@ -253,25 +253,10 @@ func (ix *Indexer) judgePinsLocked(ctx context.Context) ([]string, error) {
|
||||
return nil, err
|
||||
}
|
||||
current := CurrentRevisions(names)
|
||||
pinned := map[string]bool{}
|
||||
pinKeys, err := ix.Pins.Keys(ctx)
|
||||
pinKeys, pinned, err := ix.pinnedObjects(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, k := range pinKeys {
|
||||
object, ok := pinObject(k)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
v, _, found, err := ix.Pins.Get(ctx, k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var pv PinValue
|
||||
if found && json.Unmarshal(v, &pv) == nil && len(pv.Records) > 0 {
|
||||
pinned[object] = true
|
||||
}
|
||||
}
|
||||
var removed []string
|
||||
for _, n := range names {
|
||||
if _, _, ok := RevisionGroup(n); !ok || pinned[n] {
|
||||
@@ -301,6 +286,72 @@ func (ix *Indexer) judgePinsLocked(ctx context.Context) ([]string, error) {
|
||||
return removed, nil
|
||||
}
|
||||
|
||||
// pinnedObjects are the state pins' keys and the revision objects a record pins.
|
||||
func (ix *Indexer) pinnedObjects(ctx context.Context) ([]string, map[string]bool, error) {
|
||||
pinned := map[string]bool{}
|
||||
pinKeys, err := ix.Pins.Keys(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
for _, k := range pinKeys {
|
||||
object, ok := pinObject(k)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
v, _, found, err := ix.Pins.Get(ctx, k)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var pv PinValue
|
||||
if found && json.Unmarshal(v, &pv) == nil && len(pv.Records) > 0 {
|
||||
pinned[object] = true
|
||||
}
|
||||
}
|
||||
return pinKeys, pinned, nil
|
||||
}
|
||||
|
||||
// earlierForm says whether a revision object is of the earlier model's forms, trunk.<n> or branch.<branch>.<n>, beside the
|
||||
// branch-keyed <branch>.<n> of design 47 H §743 (H-A-135).
|
||||
func earlierForm(name string) bool {
|
||||
if _, _, ok := RevisionGroup(name); !ok {
|
||||
return false
|
||||
}
|
||||
return strings.HasPrefix(name, "trunk.") || strings.HasPrefix(name, "branch.")
|
||||
}
|
||||
|
||||
// dropEarlierFormsLocked drops every revision object of the earlier forms (earlierForm) no record pins, at a branch's take-in
|
||||
// (design 47 H §743, H-A-135): the branch's newest revision is its current one, and an earlier trunk revision kept beside it
|
||||
// would be the newest trunk revision a reader hears. It drops none while the pins do not count yet; the caller holds keepMu.
|
||||
func (ix *Indexer) dropEarlierFormsLocked(ctx context.Context) ([]string, error) {
|
||||
if ix.PinsPending != nil {
|
||||
if pending, err := ix.PinsPending(ctx); err != nil || pending != 0 {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
names, err := ix.Revisions.Names(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pinned := map[string]bool{}
|
||||
if ix.Pins != nil {
|
||||
if _, pinned, err = ix.pinnedObjects(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var dropped []string
|
||||
for _, n := range names {
|
||||
if !earlierForm(n) || pinned[n] {
|
||||
continue
|
||||
}
|
||||
if err := ix.Revisions.Delete(ctx, n); err != nil {
|
||||
return dropped, fmt.Errorf("%s: %w", n, err)
|
||||
}
|
||||
dropped = append(dropped, n)
|
||||
}
|
||||
sort.Strings(dropped)
|
||||
return dropped, nil
|
||||
}
|
||||
|
||||
// PinDrop drops one pin of the state pins by hand, by its key <branch>.<revision> (indexer_pin_drop, design 47 H §765, H-A-57):
|
||||
// every record's, for a pin that names no running pipeline; the finding pin-missing of the key is cleared, and the removal
|
||||
// rule then applies.
|
||||
|
||||
@@ -61,7 +61,8 @@ type RevisionSummary struct {
|
||||
Heads string `json:"heads,omitempty"`
|
||||
At *time.Time `json:"at,omitempty"`
|
||||
Entries int `json:"entries"`
|
||||
// Current says it is the trunk's newest revision, which is kept whatever pins it.
|
||||
// Current says it is the newest revision of its watched branch (CurrentRevisions, §579; H §743, H-A-135: no longer the
|
||||
// newest trunk-kind revision over every branch, and never one of the earlier forms), which is kept whatever pins it.
|
||||
Current bool `json:"current,omitempty"`
|
||||
}
|
||||
|
||||
@@ -84,7 +85,6 @@ func ListRevisions(ctx context.Context, o Objects, candidates bool) (RevisionsAn
|
||||
return out, fmt.Errorf("the objects %s cannot be listed: %w", CatalogRevisionsObjects, err)
|
||||
}
|
||||
listed := map[string]bool{}
|
||||
var newest int64
|
||||
entries := []string{}
|
||||
for _, n := range names {
|
||||
if strings.HasPrefix(n, "entry.") {
|
||||
@@ -103,17 +103,16 @@ func ListRevisions(ctx context.Context, o Objects, candidates bool) (RevisionsAn
|
||||
for _, ref := range rev.Entries {
|
||||
listed[EntryObject(ref.Digest)] = true
|
||||
}
|
||||
if rev.Kind == "trunk" && rev.Revision > newest {
|
||||
newest = rev.Revision
|
||||
}
|
||||
if candidates && rev.Kind != "candidate" {
|
||||
continue
|
||||
}
|
||||
out.Revisions = append(out.Revisions, RevisionSummary{Name: n, Kind: rev.Kind, Revision: rev.Revision, Source: rev.Source,
|
||||
Group: rev.Group, Heads: rev.Heads, At: rev.At, Entries: len(rev.Entries)})
|
||||
}
|
||||
current := CurrentRevisions(names)
|
||||
for i := range out.Revisions {
|
||||
out.Revisions[i].Current = out.Revisions[i].Kind == "trunk" && out.Revisions[i].Revision == newest
|
||||
_, cur := current[out.Revisions[i].Name]
|
||||
out.Revisions[i].Current = cur && out.Revisions[i].Kind == "trunk" && !earlierForm(out.Revisions[i].Name)
|
||||
}
|
||||
slices.SortFunc(out.Revisions, func(a, b RevisionSummary) int { return strings.Compare(a.Name, b.Name) })
|
||||
out.Entries = len(entries)
|
||||
|
||||
@@ -142,3 +142,37 @@ func (ix *Indexer) sayTaken(ctx context.Context, t meshcatalog.TakeInTaken) erro
|
||||
func PendingID(branch string, source int64) string {
|
||||
return takein.TakeInPendingEvent + "." + keys.EscapeName(branch) + "." + strconv.FormatInt(source, 10)
|
||||
}
|
||||
|
||||
// SourceToTake is the source-moved a read of the gate's source revision of a branch, through the seat mesh-delivery's verb
|
||||
// source, takes in as a heard one of that revision would be (design 47 H H-A-129), and false when the view keeps a revision of
|
||||
// the branch at or above it, so a revision is never taken in twice. It is the branch's first take-in, start true, exactly
|
||||
// when the view keeps no revision of the branch; otherwise its movers are judged against the newest kept (H-A-130). Every
|
||||
// repository the answer names is a head: the movers are the fingerprint changes alone.
|
||||
func (ix *Indexer) SourceToTake(ctx context.Context, branch string, a meshdelivery.SourceAnswer) (meshdelivery.SourceMovedEvent, bool, error) {
|
||||
e := meshdelivery.SourceMovedEvent{V: 1, Branch: branch, Source: a.Revision, Call: a.Call,
|
||||
Heads: make([]meshdelivery.SourceMovedEventHeadsItem, 0, len(a.Repositories))}
|
||||
if a.Revision <= 0 {
|
||||
return e, false, nil
|
||||
}
|
||||
if ix.Revisions == nil {
|
||||
return e, false, errors.New("the indexer keeps no catalog revisions")
|
||||
}
|
||||
names, err := ix.Revisions.Names(ctx)
|
||||
if err != nil {
|
||||
return e, false, fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err)
|
||||
}
|
||||
var kept int64
|
||||
for _, n := range names {
|
||||
if r, ok := BranchRevisionOf(branch, n); ok && r > kept {
|
||||
kept = r
|
||||
}
|
||||
}
|
||||
if kept >= a.Revision {
|
||||
return e, false, nil
|
||||
}
|
||||
e.Start = kept == 0
|
||||
for _, r := range a.Repositories {
|
||||
e.Heads = append(e.Heads, meshdelivery.SourceMovedEventHeadsItem{Repository: r.Repository, Commit: r.Head})
|
||||
}
|
||||
return e, true, nil
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
@@ -121,6 +122,39 @@ func TestAFirstSourceRevisionIsTakenInWithNoMover(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H §743, H-A-135: at a branch's take-in the earlier model's revisions (trunk.<n>, branch.<branch>.<n>) no record
|
||||
// pins are dropped, one a record pins stays, and the branch's newest revision is the current one, no longer the newest trunk.
|
||||
func TestATakeInDropsTheEarlierFormsNoRecordPins(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix, objects, _, f := takeInIndexer(t)
|
||||
for _, n := range []string{"trunk.527", "trunk.528", "branch.master.1"} {
|
||||
_, rev, _ := RevisionGroup(n)
|
||||
raw, _ := json.Marshal(meshcatalog.CatalogRevision{Revision: rev, Kind: "trunk", Source: 1000 + rev})
|
||||
if err := objects.Put(ctx, n, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := ix.pin(ctx, "trunk.527", "pipeline.old"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
start := meshdelivery.SourceMovedEvent{V: 1, Branch: "main", Source: 1, Call: "seed.operator.20261120", Start: true,
|
||||
Heads: []meshdelivery.SourceMovedEventHeadsItem{{Repository: f.Repository, Commit: f.Merge}}}
|
||||
if _, err := ix.TakeInSource(ctx, start); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, err := ListRevisions(ctx, objects, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got []string
|
||||
for _, r := range a.Revisions {
|
||||
got = append(got, fmt.Sprintf("%s current=%t", r.Name, r.Current))
|
||||
}
|
||||
if want := []string{"main.1 current=true", "trunk.527 current=false"}; !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("kept %v, want %v: the unpinned earlier forms dropped, the pinned one kept, main.1 current", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H §769: the take-in of a source revision moving the catalogue repository reads `predefined` from that commit's
|
||||
// trees, and one changing a predefined key rewrites it.
|
||||
func TestATakeInOfTheCatalogsCommitRewritesPredefined(t *testing.T) {
|
||||
@@ -222,3 +256,35 @@ func TestATakeInOnAFullStoreIsHeldAndSaidOnce(t *testing.T) {
|
||||
t.Errorf("held = %+v after the skip, want none", held)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H §743, §768, H-A-129, H-A-130: the gate's source revision read through its verb source is taken in as a heard
|
||||
// source-moved of it would be: start true when the view keeps no revision of the branch, every repository it names a head;
|
||||
// one the view keeps at or above is not taken again; a later one is judged against the newest kept, start false.
|
||||
func TestASourceRevisionReadFromTheGateIsTakenInOnce(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix, _, said, f := takeInIndexer(t)
|
||||
answer := meshdelivery.SourceAnswer{Branch: "main", Revision: 1, Call: "0123456789abcdef",
|
||||
Repositories: []meshdelivery.SourceHead{{Repository: f.Repository, Trunk: "main", Head: f.Merge}}}
|
||||
e, take, err := ix.SourceToTake(ctx, "main", answer)
|
||||
want := meshdelivery.SourceMovedEvent{V: 1, Branch: "main", Source: 1, Call: answer.Call, Start: true,
|
||||
Heads: []meshdelivery.SourceMovedEventHeadsItem{{Repository: f.Repository, Commit: f.Merge}}}
|
||||
if err != nil || !take || !reflect.DeepEqual(e, want) {
|
||||
t.Fatalf("a branch the view keeps none of answered %+v, %t (%v); want %+v taken", e, take, err, want)
|
||||
}
|
||||
if taken, err := ix.TakeInSource(ctx, e); err != nil || len(taken.Entries) != 0 || taken.Revision != 1 {
|
||||
t.Fatalf("the first revision read from the gate said %+v (%v), want revision 1 with no mover", taken, err)
|
||||
}
|
||||
if _, take, err := ix.SourceToTake(ctx, "main", answer); take || err != nil {
|
||||
t.Errorf("a revision the view keeps is taken again (%v)", err)
|
||||
}
|
||||
if _, take, _ := ix.SourceToTake(ctx, "nox-mesh", meshdelivery.SourceAnswer{Branch: "nox-mesh"}); take {
|
||||
t.Error("a branch the gate keeps no revision of is taken in")
|
||||
}
|
||||
answer.Revision, answer.Call = 2, "1123456789abcdef"
|
||||
if e, take, err := ix.SourceToTake(ctx, "main", answer); err != nil || !take || e.Start || e.Source != 2 {
|
||||
t.Errorf("a later revision answered %+v, %t (%v); want revision 2 taken, start false", e, take, err)
|
||||
}
|
||||
if len(*said) != 1 {
|
||||
t.Errorf("said %d take-in.taken, want 1", len(*said))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -182,7 +182,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths:
|
||||
// ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a
|
||||
// commit is in another (issue 1900).
|
||||
if want := []string{"seat:git.trees", "seat:git.tags", "seat:go-package-registry.versions", "seat:npm-package-registry.versions", "seat:git.file", "seat:git.contains", "seat:git.branch", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps",
|
||||
if want := []string{"seat:git.trees", "seat:git.tags", "seat:go-package-registry.versions", "seat:npm-package-registry.versions", "seat:git.file", "seat:git.contains", "seat:git.branch", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps",
|
||||
"seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
|
||||
@@ -152,6 +152,7 @@
|
||||
"seat:git.file",
|
||||
"seat:git.contains",
|
||||
"seat:git.branch",
|
||||
"seat:mesh-delivery.source",
|
||||
"seat:mesh-module-assigner.assignments",
|
||||
"seat:mesh-module-assigner.placement",
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
|
||||
Reference in New Issue
Block a user