I5: a take-in that cannot be written is held, said once and skippable
A source-moved the indexer cannot take in (the catalog store full, a read unanswered) stays
unacknowledged on its branch's consumer, so that branch's next waits while another branch's
go on; it is written in the new state held, one key per branch {branch, source, why, since}
(H-A-97), and said once per source as take-in.pending {branch, source, why}. A full store
sets heldFull, so RoomAgain says room once it has room, and the indexer's own room (H-A-50),
heard on its events consumer, takes every held one again, never a timer. indexer_pending
reads the state held; indexer_take_in_pending takes a branch's held one again now and
indexer_take_in_skip, with branch, source and why, acknowledges it unwritten, says
take-in.taken with no entries, skipped, by and why, and is recorded in the hand-act log;
both are asks in the state takeins. The held key goes at the take-in or the skip.
Builds: design 47 H §744; H-A-12, H-A-50, H-A-97
Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
This commit is contained in:
@@ -256,6 +256,10 @@ func main() {
|
||||
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.UnpinTool, Verb: "unpin"})
|
||||
return err
|
||||
})...)
|
||||
tools = append(tools, index.HeldTools(keptRead{stdio.State(index.HeldState)}, asks, time.Now, func(what, why string) error {
|
||||
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.TakeInSkipTool, Verb: "take-in-skip"})
|
||||
return err
|
||||
})...)
|
||||
tools = append(tools, index.RunsTools(keptRead{stdio.State(index.RunsState)}, keptRead{stdio.State(index.CandidatesState)}, asks, time.Now, func(what, why string) error {
|
||||
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.RunDropTool, Verb: "run-drop"})
|
||||
return err
|
||||
|
||||
@@ -344,6 +344,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
Versions: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.VersionsState), "no released version is kept, so no package-published makes a version", logger.Printf)},
|
||||
BuildEdges: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.BuildEdgesState), "no build of another watched branch keeps its edges", logger.Printf)},
|
||||
Pins: pinsStore, PinsPending: pendingOf(js, cred),
|
||||
Held: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.HeldState), "no held take-in is kept, so indexer_pending reads none", logger.Printf)},
|
||||
Inputs: inputsStore,
|
||||
InputObjects: inObjects,
|
||||
SayVersionReleased: sayVersionReleased(conn, cred), SayVersionsFilled: sayVersionsFilled(conn, cred),
|
||||
@@ -356,6 +357,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
routerRecordOf(cred.Module, calls.Call, calls.Ask), ix, logger.Printf)
|
||||
registry.Pruned = ix.PruneBranchBuilds
|
||||
ix.Registry = registry
|
||||
// The gate's source revisions, on one durable per watched branch (design 47 H §743, H-A-90), bound below.
|
||||
sources := &sourceMoved{ctx: ctx, ix: ix, branches: watchedBranches(registry), bind: bindBranch(js, cred, logger.Printf), logf: logger.Printf}
|
||||
defer sources.stop()
|
||||
named.Changed = ix.RemoveUnlisted
|
||||
// The first fill of the versions (design 47 G §372): at the start, and again at each catalog revision kept until it is made.
|
||||
fill := &versionsFill{ix: ix, ctx: ctx, ask: versionsVia(calls.CallKeptWhileOpen, ix.ModuleInNewestView, logger.Printf), floor: index.FloorSources{Tags: tagsVia(calls.CallKept), Holder: holderVia(calls.CallKept)}, logf: logger.Printf}
|
||||
@@ -505,7 +509,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// merged message (index.RetryUnread).
|
||||
// A state not on the bus yet at this start is watched once it appears (issue 1790).
|
||||
process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.TakeInsState), "indexer_take_in is not answered; a module the forge could not read is still read again by itself", logger.Printf).Ready(func(kv jetstream.KeyValue) {
|
||||
eachTakeIn := func(e jetstream.KeyValueEntry) { takenIn(ctx, ix, kv, e, logger.Printf) }
|
||||
eachTakeIn := func(e jetstream.KeyValueEntry) { takenIn(ctx, actor{ix, sources}, kv, e, logger.Printf) }
|
||||
if tw, err := settle(ctx, kv, eachTakeIn); err != nil {
|
||||
process.NotWatched(logger.Printf, index.Bucket(cred.Module, index.TakeInsState), err, "indexer_take_in is not answered until the indexer starts again")
|
||||
} else {
|
||||
@@ -575,6 +579,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err := hub.Handle(decidedSubject(cred.Module), func(m jetstream.Msg) { heardDecided(ctx, registry, m, logger.Printf) }); err != nil {
|
||||
return err
|
||||
}
|
||||
// The indexer's own room wakes every take-in held on a full catalog store (design 47 H §744, H-A-50).
|
||||
if err := hub.Handle(meshcatalog.SubjectRoom, func(m jetstream.Msg) { _ = m.Ack(); go sources.takeAgain() }); err != nil {
|
||||
return err
|
||||
}
|
||||
for subject := range publishedSeats {
|
||||
if err := hub.Handle(subject, versions); err != nil {
|
||||
return err
|
||||
@@ -594,10 +602,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
})
|
||||
hearNamed(ctx, js, cred, notHeard, named, logger.Printf)
|
||||
hearRuns(ctx, js, cred, notHeard, ix, logger.Printf)
|
||||
// The gate's source revisions, on one durable per watched branch (design 47 H §743, H-A-90): bound now for every branch the
|
||||
// registrations watch, and for a newly watched one on the indexer's own registration-changed.
|
||||
sources := &sourceMoved{ctx: ctx, ix: ix, branches: watchedBranches(registry), bind: bindBranch(js, cred, logger.Printf), logf: logger.Printf}
|
||||
defer sources.stop()
|
||||
// The consumer of every branch the registrations watch, bound now, and of a newly watched one on the indexer's own
|
||||
// registration-changed.
|
||||
hear(meshcatalog.StreamRegistrationChanged, "a newly watched branch's source revisions are not taken in until the indexer starts again", func(m jetstream.Msg) {
|
||||
_ = m.Ack()
|
||||
sources.refresh()
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sync"
|
||||
|
||||
@@ -28,7 +29,16 @@ type sourceMoved struct {
|
||||
logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
bound map[string]func() // by branch: the consumer's stop
|
||||
bound map[string]func() // by branch: the consumer's stop
|
||||
held map[string]heldTake // by branch: the source-moved held unacknowledged (design 47 H §744)
|
||||
again sync.Mutex // one take-in again at a time
|
||||
}
|
||||
|
||||
// heldTake is a source-moved its branch's consumer gave and the indexer could not take in: unacknowledged, so its consumer
|
||||
// gives no later one of that branch, and taken in again on the indexer's own room or by hand, never on a timer.
|
||||
type heldTake struct {
|
||||
m jetstream.Msg
|
||||
e meshdelivery.SourceMovedEvent
|
||||
}
|
||||
|
||||
// watchedBranches are the branches every registration watches, each once, sorted.
|
||||
@@ -110,6 +120,12 @@ func (s *sourceMoved) heard(branch string, m jetstream.Msg) {
|
||||
return
|
||||
}
|
||||
s.logf("source-moved: source revision %d of %s is held, not taken in: %v", e.Source, branch, err)
|
||||
s.mu.Lock()
|
||||
if s.held == nil {
|
||||
s.held = map[string]heldTake{}
|
||||
}
|
||||
s.held[branch] = heldTake{m: m, e: e}
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if err := m.Ack(); err != nil {
|
||||
@@ -117,6 +133,83 @@ func (s *sourceMoved) heard(branch string, m jetstream.Msg) {
|
||||
}
|
||||
}
|
||||
|
||||
// takeAgain takes every held source-moved in again, on the indexer's own room (H-A-50): each taken in is acknowledged, so its
|
||||
// branch's consumer gives the next; one that still cannot be taken in stays held.
|
||||
func (s *sourceMoved) takeAgain() {
|
||||
s.again.Lock()
|
||||
defer s.again.Unlock()
|
||||
s.mu.Lock()
|
||||
held := make(map[string]heldTake, len(s.held))
|
||||
for b, h := range s.held {
|
||||
held[b] = h
|
||||
}
|
||||
s.mu.Unlock()
|
||||
for branch, h := range held {
|
||||
if _, err := s.ix.TakeInSource(s.ctx, h.e); err != nil {
|
||||
s.logf("source-moved: source revision %d of %s is still held: %v", h.e.Source, branch, err)
|
||||
continue
|
||||
}
|
||||
s.mu.Lock()
|
||||
delete(s.held, branch)
|
||||
s.mu.Unlock()
|
||||
if err := h.m.Ack(); err != nil {
|
||||
s.logf("source-moved: source revision %d of %s is taken in again and its acknowledgement failed: %v", h.e.Source, branch, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// actor is the indexer and its hearing of source-moved, as the asks of the state takeins act on them.
|
||||
type actor struct {
|
||||
*index.Indexer
|
||||
s *sourceMoved
|
||||
}
|
||||
|
||||
// HeldAgain takes a branch's held source-moved in again now (indexer_take_in_pending).
|
||||
func (a actor) HeldAgain(_ context.Context, branch string) index.TakenIn {
|
||||
r := index.TakenIn{Module: branch}
|
||||
a.s.mu.Lock()
|
||||
_, held := a.s.held[branch]
|
||||
a.s.mu.Unlock()
|
||||
if !held {
|
||||
r.Answer = "the branch " + branch + " holds no source-moved"
|
||||
return r
|
||||
}
|
||||
a.s.takeAgain()
|
||||
a.s.mu.Lock()
|
||||
_, still := a.s.held[branch]
|
||||
a.s.mu.Unlock()
|
||||
r.Stored, r.Held = !still, still
|
||||
r.Answer = "taken in"
|
||||
if still {
|
||||
r.Answer = "still held: indexer_pending says why"
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// HeldSkip acknowledges a branch's held source-moved unwritten (indexer_take_in_skip): take-in.taken says it skipped.
|
||||
func (a actor) HeldSkip(ctx context.Context, branch string, source int64, by, why string) index.TakenIn {
|
||||
r := index.TakenIn{Module: branch}
|
||||
a.s.mu.Lock()
|
||||
h, held := a.s.held[branch]
|
||||
a.s.mu.Unlock()
|
||||
if !held || h.e.Source != source {
|
||||
r.Answer = fmt.Sprintf("the branch %s holds no source revision %d", branch, source)
|
||||
return r
|
||||
}
|
||||
if err := a.Indexer.SkipTakeIn(ctx, h.e, by, why); err != nil {
|
||||
r.Answer = "not skipped: " + err.Error()
|
||||
return r
|
||||
}
|
||||
a.s.mu.Lock()
|
||||
delete(a.s.held, branch)
|
||||
a.s.mu.Unlock()
|
||||
if err := h.m.Ack(); err != nil {
|
||||
a.s.logf("source-moved: source revision %d of %s is skipped and its acknowledgement failed: %v", source, branch, err)
|
||||
}
|
||||
r.Stored, r.Answer = true, "skipped"
|
||||
return r
|
||||
}
|
||||
|
||||
// stop stops every consumer bound.
|
||||
func (s *sourceMoved) stop() {
|
||||
s.mu.Lock()
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -49,3 +50,34 @@ func TestEachWatchedBranchHasItsConsumerAndTheOldFormIsRefused(t *testing.T) {
|
||||
t.Errorf("the old form was terminated %d and acknowledged %d times, journal %q; want refused with a line naming it", old.termd, old.acked, lines)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H §744: a source-moved that cannot be taken in stays unacknowledged, held; indexer_take_in_skip acknowledges it and
|
||||
// take-in.taken says it skipped.
|
||||
func TestAHeldSourceMovedIsSkippedByHand(t *testing.T) {
|
||||
var said []string
|
||||
ix := &index.Indexer{SayTakeIn: func(_ context.Context, event, id string, _ any) error { said = append(said, event+" "+id); return nil }}
|
||||
handlers := map[string]func(jetstream.Msg){}
|
||||
s := &sourceMoved{ctx: context.Background(), ix: ix,
|
||||
branches: func(context.Context) ([]string, error) { return []string{"main"}, nil },
|
||||
bind: func(_ context.Context, branch string, each func(jetstream.Msg)) (func(), error) {
|
||||
handlers[branch] = each
|
||||
return func() {}, nil
|
||||
},
|
||||
logf: func(string, ...any) {}}
|
||||
s.refresh()
|
||||
m := &fakeMsg{subject: "mesh.seat.mesh-delivery.event.source-moved.main.4", data: []byte(`{"v":1,"branch":"main","source":4,"call":"0123456789abcdef","heads":[]}`)}
|
||||
handlers["main"](msgOf{f: m})
|
||||
if m.acked != 0 || m.termd != 0 {
|
||||
t.Fatalf("a source-moved the indexer cannot take in was acknowledged %d and terminated %d times, want held", m.acked, m.termd)
|
||||
}
|
||||
a := actor{ix, s}
|
||||
if r := a.HeldSkip(context.Background(), "main", 3, "operator", "stale"); r.Stored || m.acked != 0 {
|
||||
t.Errorf("a skip of another source answered %+v", r)
|
||||
}
|
||||
if r := a.HeldSkip(context.Background(), "main", 4, "operator", "stale"); !r.Stored || m.acked != 1 {
|
||||
t.Errorf("the skip answered %+v and acknowledged %d times, want it skipped and acknowledged once", r, m.acked)
|
||||
}
|
||||
if !slices.Contains(said, "take-in.taken "+index.TakenID("main", 4)) {
|
||||
t.Errorf("said %v, want take-in.taken of main 4", said)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,6 +23,9 @@ type takerIn interface {
|
||||
Unpin(ctx context.Context, revision, record, by, why string) index.TakenIn
|
||||
DropRunByHand(ctx context.Context, run, by, why string) index.TakenIn
|
||||
RegistryAct(ctx context.Context, a index.TakeInAsk) index.TakenIn
|
||||
// HeldAgain and HeldSkip act on a watched branch's held source-moved (design 47 H §744).
|
||||
HeldAgain(ctx context.Context, branch string) index.TakenIn
|
||||
HeldSkip(ctx context.Context, branch string, source int64, by, why string) index.TakenIn
|
||||
}
|
||||
|
||||
// takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the
|
||||
@@ -46,7 +49,13 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
|
||||
defer func() {
|
||||
logf("%s.%s: ended after %s", index.TakeInsState, e.Key(), time.Since(started).Round(time.Millisecond))
|
||||
}()
|
||||
if a.Registry != nil {
|
||||
if a.Held != "" && a.Skip {
|
||||
asked, r = "skip of a held take-in", ix.HeldSkip(ctx, a.Held, a.HeldSource, a.By, a.Why)
|
||||
a.Module = a.Held
|
||||
} else if a.Held != "" {
|
||||
asked, r = "take-in again of a held take-in", ix.HeldAgain(ctx, a.Held)
|
||||
a.Module = a.Held
|
||||
} else if a.Registry != nil {
|
||||
asked, r = "registrations' act "+a.Registry.Verb, ix.RegistryAct(ctx, a)
|
||||
} else if a.Recompute {
|
||||
asked, r = "recompute of the catalog graph", ix.RecomputeByHand(ctx, a.By, a.Why)
|
||||
|
||||
@@ -143,7 +143,6 @@ type Indexer struct {
|
||||
|
||||
mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach
|
||||
source sourceNow
|
||||
saidSource map[string]uint64 // the newest source revision a take-in.taken named, by watched branch (nextSource)
|
||||
toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go)
|
||||
buildModules map[string]BuildModule // by language: the build modules whose toolchains are taken in (buildmodules.go)
|
||||
floors sync.Map // the modules whose floor FillFloor reads now (tag_floor.go), one fill each
|
||||
@@ -194,6 +193,8 @@ type Indexer struct {
|
||||
// SayTakeIn publishes a take-in event (take-in.taken, take-in.pending; design 47 H §743, §744) with its body, under the
|
||||
// duplicate id given; nil says none.
|
||||
SayTakeIn func(ctx context.Context, event, id string, body any) error
|
||||
// Held is the state held: each watched branch's source-moved held unacknowledged (held.go, design 47 H §744); nil keeps none.
|
||||
Held Store
|
||||
// Runs is the state runs: the runs on request the indexer holds a candidate for (runcandidates.go, A-174); nil keeps none.
|
||||
Runs Store
|
||||
runMu sync.Mutex // one run event at a time (runcandidates.go)
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// HeldState is the indexer's state held (design 47 H §744, H-A-97): one key per watched branch, the branch escaped as a key's
|
||||
// name is, holding the source-moved its consumer gave and the indexer could not take in, written when it is held and deleted at
|
||||
// its take-in or skip. indexer_pending reads it.
|
||||
const HeldState = "held"
|
||||
|
||||
// The held take-in's tools (§744).
|
||||
const (
|
||||
PendingTool = "indexer_pending"
|
||||
TakeInPendingTool = "indexer_take_in_pending"
|
||||
TakeInSkipTool = "indexer_take_in_skip"
|
||||
)
|
||||
|
||||
// HeldTakeIn is a value of the state held: the branch's held source revision, why it is held and since when.
|
||||
type HeldTakeIn struct {
|
||||
Branch string `json:"branch"`
|
||||
Source int64 `json:"source"`
|
||||
Why string `json:"why"`
|
||||
Since time.Time `json:"since"`
|
||||
}
|
||||
|
||||
// hold writes the branch's held take-in and says take-in.pending, once per source: a source already held is neither written nor
|
||||
// said again, so its since stays.
|
||||
func (ix *Indexer) hold(ctx context.Context, e meshdelivery.SourceMovedEvent, why string) {
|
||||
if ix.Held != nil {
|
||||
key := keys.EscapeName(e.Branch)
|
||||
if v, _, found, err := ix.Held.Get(ctx, key); err == nil && found {
|
||||
var h HeldTakeIn
|
||||
if json.Unmarshal(v, &h) == nil && h.Source == e.Source {
|
||||
return
|
||||
}
|
||||
}
|
||||
data, err := json.Marshal(HeldTakeIn{Branch: e.Branch, Source: e.Source, Why: why, Since: ix.now().UTC()})
|
||||
if err == nil {
|
||||
err = write(ctx, ix.Held, key, func([]byte, bool) ([]byte, bool, error) { return data, false, nil })
|
||||
}
|
||||
if err != nil {
|
||||
ix.logf("take-in: %s.%s is not written: %v", HeldState, key, err)
|
||||
}
|
||||
}
|
||||
ix.logf("take-in: source revision %d of %s is held: %s", e.Source, e.Branch, why)
|
||||
if ix.SayTakeIn == nil {
|
||||
return
|
||||
}
|
||||
p := meshcatalog.TakeInPending{Branch: e.Branch, Source: e.Source, Why: why}
|
||||
if err := ix.SayTakeIn(ctx, takein.TakeInPendingEvent, PendingID(e.Branch, e.Source), p); err != nil {
|
||||
ix.logf("take-in: take-in.pending of %s is not said: %v", BranchObject(e.Branch, e.Source), err)
|
||||
}
|
||||
}
|
||||
|
||||
// unhold deletes the branch's held take-in, once its source revision is taken in or skipped.
|
||||
func (ix *Indexer) unhold(ctx context.Context, branch string) {
|
||||
if ix.Held == nil {
|
||||
return
|
||||
}
|
||||
key := keys.EscapeName(branch)
|
||||
if err := write(ctx, ix.Held, key, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }); err != nil {
|
||||
ix.logf("take-in: %s.%s is not deleted: %v", HeldState, key, err)
|
||||
}
|
||||
}
|
||||
|
||||
// SkipTakeIn ends a held source revision unwritten by hand (indexer_take_in_skip, §744): it says take-in.taken with no entries,
|
||||
// skipped, by and why, and deletes the branch's held take-in; the caller acknowledges the source-moved.
|
||||
func (ix *Indexer) SkipTakeIn(ctx context.Context, e meshdelivery.SourceMovedEvent, by, why string) error {
|
||||
if err := ix.sayTaken(ctx, meshcatalog.TakeInTaken{Branch: e.Branch, Source: e.Source, Call: e.Call, Skipped: true, By: by, Why: why}); err != nil {
|
||||
return err
|
||||
}
|
||||
ix.unhold(ctx, e.Branch)
|
||||
ix.logf("take-in: source revision %d of %s skipped by %s: %s", e.Source, e.Branch, by, why)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReadHeld is every branch's held take-in, by branch.
|
||||
func ReadHeld(ctx context.Context, st Store) ([]HeldTakeIn, error) {
|
||||
ks, err := st.Keys(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slices.Sort(ks)
|
||||
out := []HeldTakeIn{}
|
||||
for _, k := range ks {
|
||||
v, _, found, err := st.Get(ctx, k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var h HeldTakeIn
|
||||
if !found || json.Unmarshal(v, &h) != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, h)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// HeldTools are the held take-in's tools (§744): indexer_pending reads the state held; indexer_take_in_pending takes a branch's
|
||||
// held source-moved again now and indexer_take_in_skip acknowledges it unwritten, each an ask in the state takeins the indexer
|
||||
// acts on (H-A-97), the skip a hand act recorded through mesh-hand-acts before it is asked.
|
||||
func HeldTools(held Store, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time, record func(what, why string) error) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
return []stdio.Tool{
|
||||
{Name: PendingTool,
|
||||
Description: "Each watched branch's held take-in (design 47 H §744): the source revision of the gate's source-moved the indexer " +
|
||||
"could not take in (the catalog store full, a read of the forge unanswered), why and since when; it stays unacknowledged " +
|
||||
"until the indexer's room wakes it. indexer_take_in_pending takes it again now, indexer_take_in_skip ends it unwritten.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) { return ReadHeld(context.Background(), held) }},
|
||||
{Name: TakeInPendingTool,
|
||||
Description: "Take a watched branch's held source-moved in again now (design 47 H §744), as the indexer's room would: " +
|
||||
"answers whether it was taken in. indexer_pending names the held branches.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"branch"}, "properties": map[string]any{
|
||||
"branch": map[string]any{"type": "string", "description": "the watched branch, as indexer_pending names it"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is taken again now"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
branch := str(args, "branch")
|
||||
if branch == "" {
|
||||
return nil, errors.New("name the held branch (branch), as indexer_pending names it")
|
||||
}
|
||||
data, err := json.Marshal(TakeInAsk{V: 1, Held: branch, Why: str(args, "why"), By: stdio.CallerOf(args), At: now().UTC()})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a, err := ask("held."+keys.EscapeName(branch)+".again", data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.Result, nil
|
||||
}},
|
||||
{Name: TakeInSkipTool,
|
||||
Description: "End a watched branch's held source-moved unwritten (design 47 H §744): it is acknowledged, take-in.taken is said " +
|
||||
"with no entries and skipped, and the hand act is recorded through mesh-hand-acts before it is asked. indexer_pending " +
|
||||
"names the held branch and source.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"branch", "source", "why"}, "properties": map[string]any{
|
||||
"branch": map[string]any{"type": "string", "description": "the watched branch, as indexer_pending names it"},
|
||||
"source": map[string]any{"type": "integer", "description": "the held source revision"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is skipped, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
branch, why := str(args, "branch"), str(args, "why")
|
||||
source, _ := args["source"].(float64)
|
||||
if branch == "" || source <= 0 || why == "" {
|
||||
return nil, errors.New("name the held branch (branch) and source revision (source) and say why (why): a skip is a hand act, recorded through mesh-hand-acts")
|
||||
}
|
||||
data, err := json.Marshal(TakeInAsk{V: 1, Held: branch, HeldSource: int64(source), Skip: true, Why: why, By: stdio.CallerOf(args), At: now().UTC()})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := record(fmt.Sprintf("skipped the held source revision %d of %s unwritten by hand", int64(source), branch), why); err != nil {
|
||||
return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was skipped: %w", err)
|
||||
}
|
||||
a, err := ask(fmt.Sprintf("held.%s.skip.%d", keys.EscapeName(branch), int64(source)), data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.Result, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
@@ -64,8 +64,13 @@ type TakeInAsk struct {
|
||||
NamedFingerprint string `json:"named-fingerprint,omitempty"`
|
||||
// UnpinRevision and UnpinRecord ask one record's pin on a revision of the state pins to be removed (indexer_unpin).
|
||||
UnpinRevision string `json:"unpin-revision,omitempty"`
|
||||
UnpinRecord string `json:"unpin-record,omitempty"`
|
||||
NamedMember string `json:"named-member,omitempty"`
|
||||
// Held names the watched branch whose held source-moved is taken in again now (indexer_take_in_pending) or, with Skip and
|
||||
// HeldSource, ended unwritten (indexer_take_in_skip; design 47 H §744).
|
||||
Held string `json:"held,omitempty"`
|
||||
HeldSource int64 `json:"held-source,omitempty"`
|
||||
Skip bool `json:"skip,omitempty"`
|
||||
UnpinRecord string `json:"unpin-record,omitempty"`
|
||||
NamedMember string `json:"named-member,omitempty"`
|
||||
// RunDrop asks one run on request, by its group run-<id>, to be dropped with its candidate (indexer_run_drop, runs_tools.go, A-194).
|
||||
RunDrop string `json:"run-drop,omitempty"`
|
||||
// Registry asks one of the registrations' acts (registrations.go): a trusted change of one, the verb registrations,
|
||||
|
||||
@@ -30,9 +30,21 @@ func TakenID(branch string, source int64) string {
|
||||
// mover (§768). A revision already kept under that name (a
|
||||
// restart before the acknowledgement) is said again from its kept movers, recomputing nothing (H-A-92). The first watched
|
||||
// branch is taken in on the indexer's own view; another on its newest revision's, computed and kept only as that revision.
|
||||
// An error leaves the source-moved held: ErrOldForm never takes it in, ErrObjectsFull waits for room (§744), any other is the
|
||||
// failed read or write it says.
|
||||
// An error leaves the source-moved held, said once per source as take-in.pending with branch, source and why (§744):
|
||||
// ErrObjectsFull waits for the indexer's own room, any other is the failed read or write it says; ErrOldForm is never taken in
|
||||
// and said as no take-in.
|
||||
func (ix *Indexer) TakeInSource(ctx context.Context, e meshdelivery.SourceMovedEvent) (meshcatalog.TakeInTaken, error) {
|
||||
taken, err := ix.takeInSource(ctx, e)
|
||||
switch {
|
||||
case err == nil:
|
||||
ix.unhold(ctx, e.Branch)
|
||||
case !errors.Is(err, ErrOldForm):
|
||||
ix.hold(ctx, e, err.Error())
|
||||
}
|
||||
return taken, err
|
||||
}
|
||||
|
||||
func (ix *Indexer) takeInSource(ctx context.Context, e meshdelivery.SourceMovedEvent) (meshcatalog.TakeInTaken, error) {
|
||||
taken := meshcatalog.TakeInTaken{Branch: e.Branch, Source: e.Source, Call: e.Call}
|
||||
if e.Branch == "" || e.Source <= 0 || e.Call == "" {
|
||||
return taken, fmt.Errorf("%w: branch %q, source %d, call %q", ErrOldForm, e.Branch, e.Source, e.Call)
|
||||
@@ -92,6 +104,9 @@ func (ix *Indexer) TakeInSource(ctx context.Context, e meshdelivery.SourceMovedE
|
||||
movers = nil // a branch's first revision moves nothing: its entries are the base later take-ins are held against (§768, H-A-19)
|
||||
}
|
||||
if err := ix.KeepBranchEntries(ctx, e.Branch, e.Source, entries, movers); err != nil {
|
||||
if errors.Is(err, ErrObjectsFull) {
|
||||
ix.heldFull.Store(true) // room is said once the store has room again (RoomAgain), and wakes the take-in (§744)
|
||||
}
|
||||
return taken, err
|
||||
}
|
||||
taken.Revision, taken.Entries = e.Source, movers
|
||||
@@ -119,3 +134,8 @@ func (ix *Indexer) sayTaken(ctx context.Context, t meshcatalog.TakeInTaken) erro
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PendingID is the duplicate id take-in.pending of a branch's source revision is said under: said once per source.
|
||||
func PendingID(branch string, source int64) string {
|
||||
return takein.TakeInPendingEvent + "." + keys.EscapeName(branch) + "." + strconv.FormatInt(source, 10)
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
)
|
||||
|
||||
// takeInIndexer is an indexer that takes source revisions in from the catalogue repository's trees at f's commit, keeping its
|
||||
// revisions and pins, and the take-in events it says.
|
||||
// revisions and pins, and the take-in events it says (a take-in.pending kept as a taken with its why and skipped set).
|
||||
func takeInIndexer(t *testing.T) (*Indexer, *memObjects, *[]meshcatalog.TakeInTaken, whole) {
|
||||
t.Helper()
|
||||
ix, _, f := newFormIndexerUnmerged(t, mod{"avahi", catalogueManifest(t, "avahi")})
|
||||
@@ -28,6 +28,10 @@ func takeInIndexer(t *testing.T) (*Indexer, *memObjects, *[]meshcatalog.TakeInTa
|
||||
ix.ReadFile = func(context.Context, string, string, string) ([]byte, error) { return nil, nil }
|
||||
said := &[]meshcatalog.TakeInTaken{}
|
||||
ix.SayTakeIn = func(_ context.Context, event, id string, body any) error {
|
||||
if p, ok := body.(meshcatalog.TakeInPending); ok && event == "take-in.pending" && id == PendingID(p.Branch, p.Source) {
|
||||
*said = append(*said, meshcatalog.TakeInTaken{Branch: p.Branch, Source: p.Source, Why: p.Why, Skipped: true})
|
||||
return nil
|
||||
}
|
||||
tk, ok := body.(meshcatalog.TakeInTaken)
|
||||
if event != "take-in.taken" || !ok || id != TakenID(tk.Branch, tk.Source) {
|
||||
t.Errorf("said %s %s %+v, want take-in.taken under its branch and source", event, id, body)
|
||||
@@ -150,3 +154,53 @@ func TestATakeInOfTheCatalogsCommitRewritesPredefined(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H §744 (acceptance test 116): a take-in the full catalog store cannot write is held, said once per source as
|
||||
// take-in.pending with branch, source and why, and room is said once the store has room again; taken in then.
|
||||
func TestATakeInOnAFullStoreIsHeldAndSaidOnce(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix, objects, said, f := takeInIndexer(t)
|
||||
objects.capacity = 10
|
||||
ix.Held = newMem()
|
||||
moved := meshdelivery.SourceMovedEvent{V: 1, Branch: "main", Source: 3, Call: "0123456789abcdef",
|
||||
Heads: []meshdelivery.SourceMovedEventHeadsItem{{Repository: f.Repository, Commit: f.Merge}}}
|
||||
var rooms int
|
||||
ix.SayFill = func(_ context.Context, event, _ string, _ meshcatalog.Fill) error {
|
||||
if event == meshcatalog.EventRoom {
|
||||
rooms++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for range 2 {
|
||||
if _, err := ix.TakeInSource(ctx, moved); !errors.Is(err, ErrObjectsFull) {
|
||||
t.Fatalf("a full store answered %v, want ErrObjectsFull", err)
|
||||
}
|
||||
}
|
||||
if len(*said) != 1 || (*said)[0].Branch != "main" || (*said)[0].Source != 3 || (*said)[0].Why == "" || !(*said)[0].Skipped {
|
||||
t.Fatalf("said %+v, want one take-in.pending of main 3 with its why", *said)
|
||||
}
|
||||
if held, err := ReadHeld(ctx, ix.Held); err != nil || len(held) != 1 || held[0].Branch != "main" || held[0].Source != 3 || held[0].Why == "" {
|
||||
t.Errorf("held = %+v (%v), want main's source revision 3 and why", held, err)
|
||||
}
|
||||
objects.capacity = 1 << 22
|
||||
if ok, err := ix.RoomAgain(ctx); err != nil || !ok || rooms != 1 {
|
||||
t.Fatalf("room said %v %d times (%v), want once the store has room", ok, rooms, err)
|
||||
}
|
||||
if taken, err := ix.TakeInSource(ctx, moved); err != nil || taken.Revision != 3 || len(*said) != 2 {
|
||||
t.Errorf("taken again on room: %+v (%v), said %d events", taken, err, len(*said))
|
||||
}
|
||||
if held, _ := ReadHeld(ctx, ix.Held); len(held) != 0 {
|
||||
t.Errorf("held = %+v after the take-in, want none", held)
|
||||
}
|
||||
// A skip ends a held source revision unwritten: take-in.taken with no entries, skipped, by and why; held is deleted.
|
||||
ix.hold(ctx, meshdelivery.SourceMovedEvent{Branch: "nox-mesh", Source: 9, Call: "fedcba9876543210"}, "the catalog store is full")
|
||||
if err := ix.SkipTakeIn(ctx, meshdelivery.SourceMovedEvent{Branch: "nox-mesh", Source: 9, Call: "fedcba9876543210"}, "operator", "a stale source"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if last := (*said)[len(*said)-1]; !last.Skipped || last.By != "operator" || last.Why != "a stale source" || len(last.Entries) != 0 || last.Source != 9 {
|
||||
t.Errorf("the skip said %+v, want take-in.taken skipped by the operator with its why", last)
|
||||
}
|
||||
if held, _ := ReadHeld(ctx, ix.Held); len(held) != 0 {
|
||||
t.Errorf("held = %+v after the skip, want none", held)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -137,7 +137,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
{Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5},
|
||||
{Name: index.EntryIndexState, History: 1, MaxMiB: 8}, {Name: index.NamedState, History: 1, MaxMiB: 4}, // design 47 G §695
|
||||
{Name: index.RegistrationsState, History: 1}, {Name: index.ProposalsState, History: 1, MaxMiB: 4},
|
||||
{Name: index.VersionsState, History: 1, MaxMiB: 64}, {Name: index.BuildEdgesState, History: 1}, {Name: index.InputsState, History: 1}, {Name: index.PinsState, History: 1, MaxMiB: index.PinsCapMiB}, {Name: process.ConsumersState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17
|
||||
{Name: index.VersionsState, History: 1, MaxMiB: 64}, {Name: index.BuildEdgesState, History: 1}, {Name: index.InputsState, History: 1}, {Name: index.PinsState, History: 1, MaxMiB: index.PinsCapMiB}, {Name: process.ConsumersState, History: 1}, {Name: index.HeldState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
// Milestone 47/G W4.2 (design 47, Two graphs; ADR 0619 §2): the catalog graph revisions are objects in the old section,
|
||||
@@ -169,7 +169,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control
|
||||
// internal/broker/state.go:62-93).
|
||||
// The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*).
|
||||
if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-conditions.findings-asked",
|
||||
if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-catalog.room", "mesh-conditions.findings-asked",
|
||||
"mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.run-started", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
||||
}
|
||||
@@ -204,8 +204,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered.
|
||||
index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool,
|
||||
index.EntryIndexTool, index.NamedDropTool, // design 47 G §675
|
||||
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.ConsumerTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194)
|
||||
if !reflect.DeepEqual(m.Tools, want) || len(want) != 41 || !slices.Contains(m.Tools, "modules") {
|
||||
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.PendingTool, index.TakeInPendingTool, index.TakeInSkipTool, index.ConsumerTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194)
|
||||
if !reflect.DeepEqual(m.Tools, want) || len(want) != 44 || !slices.Contains(m.Tools, "modules") {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
// Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section.
|
||||
|
||||
@@ -65,6 +65,9 @@
|
||||
"indexer_inputs",
|
||||
"indexer_pins",
|
||||
"indexer_unpin",
|
||||
"indexer_pending",
|
||||
"indexer_take_in_pending",
|
||||
"indexer_take_in_skip",
|
||||
"indexer_consumer",
|
||||
"indexer_runs",
|
||||
"indexer_run_drop"
|
||||
@@ -82,6 +85,7 @@
|
||||
"mesh-wants.wait-ended",
|
||||
"mesh-catalog.catalog-revision",
|
||||
"mesh-catalog.registration-changed",
|
||||
"mesh-catalog.room",
|
||||
"mesh-delivery.source-moved",
|
||||
"mesh-wants.wanted",
|
||||
"mesh-pipeline-runner.run-started",
|
||||
@@ -115,7 +119,8 @@
|
||||
{ "name": "build-edges", "history": 1 },
|
||||
{ "name": "inputs", "history": 1 },
|
||||
{ "name": "pins", "history": 1, "max-mib": 4 },
|
||||
{ "name": "consumers", "history": 1 }
|
||||
{ "name": "consumers", "history": 1 },
|
||||
{ "name": "held", "history": 1 }
|
||||
],
|
||||
"streams": [
|
||||
{ "name": "registration-history", "events": ["registration.*"], "retention": "history", "max-mib": 64 },
|
||||
|
||||
Reference in New Issue
Block a user