I5: a take-in that cannot be written is held, said once and skippable

A source-moved the indexer cannot take in (the catalog store full, a read unanswered) stays
unacknowledged on its branch's consumer, so that branch's next waits while another branch's
go on; it is written in the new state held, one key per branch {branch, source, why, since}
(H-A-97), and said once per source as take-in.pending {branch, source, why}. A full store
sets heldFull, so RoomAgain says room once it has room, and the indexer's own room (H-A-50),
heard on its events consumer, takes every held one again, never a timer. indexer_pending
reads the state held; indexer_take_in_pending takes a branch's held one again now and
indexer_take_in_skip, with branch, source and why, acknowledges it unwritten, says
take-in.taken with no entries, skipped, by and why, and is recorded in the hand-act log;
both are asks in the state takeins. The held key goes at the take-in or the skip.

Builds: design 47 H §744; H-A-12, H-A-50, H-A-97

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
This commit is contained in:
2026-11-20 04:56:27 +01:00
parent 9ee012bce5
commit 1b8f377cad
12 changed files with 420 additions and 18 deletions
@@ -256,6 +256,10 @@ func main() {
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.UnpinTool, Verb: "unpin"})
return err
})...)
tools = append(tools, index.HeldTools(keptRead{stdio.State(index.HeldState)}, asks, time.Now, func(what, why string) error {
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.TakeInSkipTool, Verb: "take-in-skip"})
return err
})...)
tools = append(tools, index.RunsTools(keptRead{stdio.State(index.RunsState)}, keptRead{stdio.State(index.CandidatesState)}, asks, time.Now, func(what, why string) error {
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.RunDropTool, Verb: "run-drop"})
return err
@@ -344,6 +344,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
Versions: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.VersionsState), "no released version is kept, so no package-published makes a version", logger.Printf)},
BuildEdges: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.BuildEdgesState), "no build of another watched branch keeps its edges", logger.Printf)},
Pins: pinsStore, PinsPending: pendingOf(js, cred),
Held: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.HeldState), "no held take-in is kept, so indexer_pending reads none", logger.Printf)},
Inputs: inputsStore,
InputObjects: inObjects,
SayVersionReleased: sayVersionReleased(conn, cred), SayVersionsFilled: sayVersionsFilled(conn, cred),
@@ -356,6 +357,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
routerRecordOf(cred.Module, calls.Call, calls.Ask), ix, logger.Printf)
registry.Pruned = ix.PruneBranchBuilds
ix.Registry = registry
// The gate's source revisions, on one durable per watched branch (design 47 H §743, H-A-90), bound below.
sources := &sourceMoved{ctx: ctx, ix: ix, branches: watchedBranches(registry), bind: bindBranch(js, cred, logger.Printf), logf: logger.Printf}
defer sources.stop()
named.Changed = ix.RemoveUnlisted
// The first fill of the versions (design 47 G §372): at the start, and again at each catalog revision kept until it is made.
fill := &versionsFill{ix: ix, ctx: ctx, ask: versionsVia(calls.CallKeptWhileOpen, ix.ModuleInNewestView, logger.Printf), floor: index.FloorSources{Tags: tagsVia(calls.CallKept), Holder: holderVia(calls.CallKept)}, logf: logger.Printf}
@@ -505,7 +509,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// merged message (index.RetryUnread).
// A state not on the bus yet at this start is watched once it appears (issue 1790).
process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.TakeInsState), "indexer_take_in is not answered; a module the forge could not read is still read again by itself", logger.Printf).Ready(func(kv jetstream.KeyValue) {
eachTakeIn := func(e jetstream.KeyValueEntry) { takenIn(ctx, ix, kv, e, logger.Printf) }
eachTakeIn := func(e jetstream.KeyValueEntry) { takenIn(ctx, actor{ix, sources}, kv, e, logger.Printf) }
if tw, err := settle(ctx, kv, eachTakeIn); err != nil {
process.NotWatched(logger.Printf, index.Bucket(cred.Module, index.TakeInsState), err, "indexer_take_in is not answered until the indexer starts again")
} else {
@@ -575,6 +579,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err := hub.Handle(decidedSubject(cred.Module), func(m jetstream.Msg) { heardDecided(ctx, registry, m, logger.Printf) }); err != nil {
return err
}
// The indexer's own room wakes every take-in held on a full catalog store (design 47 H §744, H-A-50).
if err := hub.Handle(meshcatalog.SubjectRoom, func(m jetstream.Msg) { _ = m.Ack(); go sources.takeAgain() }); err != nil {
return err
}
for subject := range publishedSeats {
if err := hub.Handle(subject, versions); err != nil {
return err
@@ -594,10 +602,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
})
hearNamed(ctx, js, cred, notHeard, named, logger.Printf)
hearRuns(ctx, js, cred, notHeard, ix, logger.Printf)
// The gate's source revisions, on one durable per watched branch (design 47 H §743, H-A-90): bound now for every branch the
// registrations watch, and for a newly watched one on the indexer's own registration-changed.
sources := &sourceMoved{ctx: ctx, ix: ix, branches: watchedBranches(registry), bind: bindBranch(js, cred, logger.Printf), logf: logger.Printf}
defer sources.stop()
// The consumer of every branch the registrations watch, bound now, and of a newly watched one on the indexer's own
// registration-changed.
hear(meshcatalog.StreamRegistrationChanged, "a newly watched branch's source revisions are not taken in until the indexer starts again", func(m jetstream.Msg) {
_ = m.Ack()
sources.refresh()
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"sync"
@@ -28,7 +29,16 @@ type sourceMoved struct {
logf func(string, ...any)
mu sync.Mutex
bound map[string]func() // by branch: the consumer's stop
bound map[string]func() // by branch: the consumer's stop
held map[string]heldTake // by branch: the source-moved held unacknowledged (design 47 H §744)
again sync.Mutex // one take-in again at a time
}
// heldTake is a source-moved its branch's consumer gave and the indexer could not take in: unacknowledged, so its consumer
// gives no later one of that branch, and taken in again on the indexer's own room or by hand, never on a timer.
type heldTake struct {
m jetstream.Msg
e meshdelivery.SourceMovedEvent
}
// watchedBranches are the branches every registration watches, each once, sorted.
@@ -110,6 +120,12 @@ func (s *sourceMoved) heard(branch string, m jetstream.Msg) {
return
}
s.logf("source-moved: source revision %d of %s is held, not taken in: %v", e.Source, branch, err)
s.mu.Lock()
if s.held == nil {
s.held = map[string]heldTake{}
}
s.held[branch] = heldTake{m: m, e: e}
s.mu.Unlock()
return
}
if err := m.Ack(); err != nil {
@@ -117,6 +133,83 @@ func (s *sourceMoved) heard(branch string, m jetstream.Msg) {
}
}
// takeAgain takes every held source-moved in again, on the indexer's own room (H-A-50): each taken in is acknowledged, so its
// branch's consumer gives the next; one that still cannot be taken in stays held.
func (s *sourceMoved) takeAgain() {
s.again.Lock()
defer s.again.Unlock()
s.mu.Lock()
held := make(map[string]heldTake, len(s.held))
for b, h := range s.held {
held[b] = h
}
s.mu.Unlock()
for branch, h := range held {
if _, err := s.ix.TakeInSource(s.ctx, h.e); err != nil {
s.logf("source-moved: source revision %d of %s is still held: %v", h.e.Source, branch, err)
continue
}
s.mu.Lock()
delete(s.held, branch)
s.mu.Unlock()
if err := h.m.Ack(); err != nil {
s.logf("source-moved: source revision %d of %s is taken in again and its acknowledgement failed: %v", h.e.Source, branch, err)
}
}
}
// actor is the indexer and its hearing of source-moved, as the asks of the state takeins act on them.
type actor struct {
*index.Indexer
s *sourceMoved
}
// HeldAgain takes a branch's held source-moved in again now (indexer_take_in_pending).
func (a actor) HeldAgain(_ context.Context, branch string) index.TakenIn {
r := index.TakenIn{Module: branch}
a.s.mu.Lock()
_, held := a.s.held[branch]
a.s.mu.Unlock()
if !held {
r.Answer = "the branch " + branch + " holds no source-moved"
return r
}
a.s.takeAgain()
a.s.mu.Lock()
_, still := a.s.held[branch]
a.s.mu.Unlock()
r.Stored, r.Held = !still, still
r.Answer = "taken in"
if still {
r.Answer = "still held: indexer_pending says why"
}
return r
}
// HeldSkip acknowledges a branch's held source-moved unwritten (indexer_take_in_skip): take-in.taken says it skipped.
func (a actor) HeldSkip(ctx context.Context, branch string, source int64, by, why string) index.TakenIn {
r := index.TakenIn{Module: branch}
a.s.mu.Lock()
h, held := a.s.held[branch]
a.s.mu.Unlock()
if !held || h.e.Source != source {
r.Answer = fmt.Sprintf("the branch %s holds no source revision %d", branch, source)
return r
}
if err := a.Indexer.SkipTakeIn(ctx, h.e, by, why); err != nil {
r.Answer = "not skipped: " + err.Error()
return r
}
a.s.mu.Lock()
delete(a.s.held, branch)
a.s.mu.Unlock()
if err := h.m.Ack(); err != nil {
a.s.logf("source-moved: source revision %d of %s is skipped and its acknowledgement failed: %v", source, branch, err)
}
r.Stored, r.Answer = true, "skipped"
return r
}
// stop stops every consumer bound.
func (s *sourceMoved) stop() {
s.mu.Lock()
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"reflect"
"slices"
"strings"
"testing"
@@ -49,3 +50,34 @@ func TestEachWatchedBranchHasItsConsumerAndTheOldFormIsRefused(t *testing.T) {
t.Errorf("the old form was terminated %d and acknowledged %d times, journal %q; want refused with a line naming it", old.termd, old.acked, lines)
}
}
// Design 47 H §744: a source-moved that cannot be taken in stays unacknowledged, held; indexer_take_in_skip acknowledges it and
// take-in.taken says it skipped.
func TestAHeldSourceMovedIsSkippedByHand(t *testing.T) {
var said []string
ix := &index.Indexer{SayTakeIn: func(_ context.Context, event, id string, _ any) error { said = append(said, event+" "+id); return nil }}
handlers := map[string]func(jetstream.Msg){}
s := &sourceMoved{ctx: context.Background(), ix: ix,
branches: func(context.Context) ([]string, error) { return []string{"main"}, nil },
bind: func(_ context.Context, branch string, each func(jetstream.Msg)) (func(), error) {
handlers[branch] = each
return func() {}, nil
},
logf: func(string, ...any) {}}
s.refresh()
m := &fakeMsg{subject: "mesh.seat.mesh-delivery.event.source-moved.main.4", data: []byte(`{"v":1,"branch":"main","source":4,"call":"0123456789abcdef","heads":[]}`)}
handlers["main"](msgOf{f: m})
if m.acked != 0 || m.termd != 0 {
t.Fatalf("a source-moved the indexer cannot take in was acknowledged %d and terminated %d times, want held", m.acked, m.termd)
}
a := actor{ix, s}
if r := a.HeldSkip(context.Background(), "main", 3, "operator", "stale"); r.Stored || m.acked != 0 {
t.Errorf("a skip of another source answered %+v", r)
}
if r := a.HeldSkip(context.Background(), "main", 4, "operator", "stale"); !r.Stored || m.acked != 1 {
t.Errorf("the skip answered %+v and acknowledged %d times, want it skipped and acknowledged once", r, m.acked)
}
if !slices.Contains(said, "take-in.taken "+index.TakenID("main", 4)) {
t.Errorf("said %v, want take-in.taken of main 4", said)
}
}
@@ -23,6 +23,9 @@ type takerIn interface {
Unpin(ctx context.Context, revision, record, by, why string) index.TakenIn
DropRunByHand(ctx context.Context, run, by, why string) index.TakenIn
RegistryAct(ctx context.Context, a index.TakeInAsk) index.TakenIn
// HeldAgain and HeldSkip act on a watched branch's held source-moved (design 47 H §744).
HeldAgain(ctx context.Context, branch string) index.TakenIn
HeldSkip(ctx context.Context, branch string, source int64, by, why string) index.TakenIn
}
// takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the
@@ -46,7 +49,13 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
defer func() {
logf("%s.%s: ended after %s", index.TakeInsState, e.Key(), time.Since(started).Round(time.Millisecond))
}()
if a.Registry != nil {
if a.Held != "" && a.Skip {
asked, r = "skip of a held take-in", ix.HeldSkip(ctx, a.Held, a.HeldSource, a.By, a.Why)
a.Module = a.Held
} else if a.Held != "" {
asked, r = "take-in again of a held take-in", ix.HeldAgain(ctx, a.Held)
a.Module = a.Held
} else if a.Registry != nil {
asked, r = "registrations' act "+a.Registry.Verb, ix.RegistryAct(ctx, a)
} else if a.Recompute {
asked, r = "recompute of the catalog graph", ix.RecomputeByHand(ctx, a.By, a.Why)
@@ -143,7 +143,6 @@ type Indexer struct {
mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach
source sourceNow
saidSource map[string]uint64 // the newest source revision a take-in.taken named, by watched branch (nextSource)
toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go)
buildModules map[string]BuildModule // by language: the build modules whose toolchains are taken in (buildmodules.go)
floors sync.Map // the modules whose floor FillFloor reads now (tag_floor.go), one fill each
@@ -194,6 +193,8 @@ type Indexer struct {
// SayTakeIn publishes a take-in event (take-in.taken, take-in.pending; design 47 H §743, §744) with its body, under the
// duplicate id given; nil says none.
SayTakeIn func(ctx context.Context, event, id string, body any) error
// Held is the state held: each watched branch's source-moved held unacknowledged (held.go, design 47 H §744); nil keeps none.
Held Store
// Runs is the state runs: the runs on request the indexer holds a candidate for (runcandidates.go, A-174); nil keeps none.
Runs Store
runMu sync.Mutex // one run event at a time (runcandidates.go)
@@ -0,0 +1,173 @@
package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// HeldState is the indexer's state held (design 47 H §744, H-A-97): one key per watched branch, the branch escaped as a key's
// name is, holding the source-moved its consumer gave and the indexer could not take in, written when it is held and deleted at
// its take-in or skip. indexer_pending reads it.
const HeldState = "held"
// The held take-in's tools (§744).
const (
PendingTool = "indexer_pending"
TakeInPendingTool = "indexer_take_in_pending"
TakeInSkipTool = "indexer_take_in_skip"
)
// HeldTakeIn is a value of the state held: the branch's held source revision, why it is held and since when.
type HeldTakeIn struct {
Branch string `json:"branch"`
Source int64 `json:"source"`
Why string `json:"why"`
Since time.Time `json:"since"`
}
// hold writes the branch's held take-in and says take-in.pending, once per source: a source already held is neither written nor
// said again, so its since stays.
func (ix *Indexer) hold(ctx context.Context, e meshdelivery.SourceMovedEvent, why string) {
if ix.Held != nil {
key := keys.EscapeName(e.Branch)
if v, _, found, err := ix.Held.Get(ctx, key); err == nil && found {
var h HeldTakeIn
if json.Unmarshal(v, &h) == nil && h.Source == e.Source {
return
}
}
data, err := json.Marshal(HeldTakeIn{Branch: e.Branch, Source: e.Source, Why: why, Since: ix.now().UTC()})
if err == nil {
err = write(ctx, ix.Held, key, func([]byte, bool) ([]byte, bool, error) { return data, false, nil })
}
if err != nil {
ix.logf("take-in: %s.%s is not written: %v", HeldState, key, err)
}
}
ix.logf("take-in: source revision %d of %s is held: %s", e.Source, e.Branch, why)
if ix.SayTakeIn == nil {
return
}
p := meshcatalog.TakeInPending{Branch: e.Branch, Source: e.Source, Why: why}
if err := ix.SayTakeIn(ctx, takein.TakeInPendingEvent, PendingID(e.Branch, e.Source), p); err != nil {
ix.logf("take-in: take-in.pending of %s is not said: %v", BranchObject(e.Branch, e.Source), err)
}
}
// unhold deletes the branch's held take-in, once its source revision is taken in or skipped.
func (ix *Indexer) unhold(ctx context.Context, branch string) {
if ix.Held == nil {
return
}
key := keys.EscapeName(branch)
if err := write(ctx, ix.Held, key, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }); err != nil {
ix.logf("take-in: %s.%s is not deleted: %v", HeldState, key, err)
}
}
// SkipTakeIn ends a held source revision unwritten by hand (indexer_take_in_skip, §744): it says take-in.taken with no entries,
// skipped, by and why, and deletes the branch's held take-in; the caller acknowledges the source-moved.
func (ix *Indexer) SkipTakeIn(ctx context.Context, e meshdelivery.SourceMovedEvent, by, why string) error {
if err := ix.sayTaken(ctx, meshcatalog.TakeInTaken{Branch: e.Branch, Source: e.Source, Call: e.Call, Skipped: true, By: by, Why: why}); err != nil {
return err
}
ix.unhold(ctx, e.Branch)
ix.logf("take-in: source revision %d of %s skipped by %s: %s", e.Source, e.Branch, by, why)
return nil
}
// ReadHeld is every branch's held take-in, by branch.
func ReadHeld(ctx context.Context, st Store) ([]HeldTakeIn, error) {
ks, err := st.Keys(ctx)
if err != nil {
return nil, err
}
slices.Sort(ks)
out := []HeldTakeIn{}
for _, k := range ks {
v, _, found, err := st.Get(ctx, k)
if err != nil {
return nil, err
}
var h HeldTakeIn
if !found || json.Unmarshal(v, &h) != nil {
continue
}
out = append(out, h)
}
return out, nil
}
// HeldTools are the held take-in's tools (§744): indexer_pending reads the state held; indexer_take_in_pending takes a branch's
// held source-moved again now and indexer_take_in_skip acknowledges it unwritten, each an ask in the state takeins the indexer
// acts on (H-A-97), the skip a hand act recorded through mesh-hand-acts before it is asked.
func HeldTools(held Store, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time, record func(what, why string) error) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
return []stdio.Tool{
{Name: PendingTool,
Description: "Each watched branch's held take-in (design 47 H §744): the source revision of the gate's source-moved the indexer " +
"could not take in (the catalog store full, a read of the forge unanswered), why and since when; it stays unacknowledged " +
"until the indexer's room wakes it. indexer_take_in_pending takes it again now, indexer_take_in_skip ends it unwritten.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) { return ReadHeld(context.Background(), held) }},
{Name: TakeInPendingTool,
Description: "Take a watched branch's held source-moved in again now (design 47 H §744), as the indexer's room would: " +
"answers whether it was taken in. indexer_pending names the held branches.",
Input: map[string]any{"type": "object", "required": []string{"branch"}, "properties": map[string]any{
"branch": map[string]any{"type": "string", "description": "the watched branch, as indexer_pending names it"},
"why": map[string]any{"type": "string", "description": "why it is taken again now"}}},
Run: func(args map[string]any) (any, error) {
branch := str(args, "branch")
if branch == "" {
return nil, errors.New("name the held branch (branch), as indexer_pending names it")
}
data, err := json.Marshal(TakeInAsk{V: 1, Held: branch, Why: str(args, "why"), By: stdio.CallerOf(args), At: now().UTC()})
if err != nil {
return nil, err
}
a, err := ask("held."+keys.EscapeName(branch)+".again", data)
if err != nil {
return nil, err
}
return a.Result, nil
}},
{Name: TakeInSkipTool,
Description: "End a watched branch's held source-moved unwritten (design 47 H §744): it is acknowledged, take-in.taken is said " +
"with no entries and skipped, and the hand act is recorded through mesh-hand-acts before it is asked. indexer_pending " +
"names the held branch and source.",
Input: map[string]any{"type": "object", "required": []string{"branch", "source", "why"}, "properties": map[string]any{
"branch": map[string]any{"type": "string", "description": "the watched branch, as indexer_pending names it"},
"source": map[string]any{"type": "integer", "description": "the held source revision"},
"why": map[string]any{"type": "string", "description": "why it is skipped, in plain words"}}},
Run: func(args map[string]any) (any, error) {
branch, why := str(args, "branch"), str(args, "why")
source, _ := args["source"].(float64)
if branch == "" || source <= 0 || why == "" {
return nil, errors.New("name the held branch (branch) and source revision (source) and say why (why): a skip is a hand act, recorded through mesh-hand-acts")
}
data, err := json.Marshal(TakeInAsk{V: 1, Held: branch, HeldSource: int64(source), Skip: true, Why: why, By: stdio.CallerOf(args), At: now().UTC()})
if err != nil {
return nil, err
}
if err := record(fmt.Sprintf("skipped the held source revision %d of %s unwritten by hand", int64(source), branch), why); err != nil {
return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was skipped: %w", err)
}
a, err := ask(fmt.Sprintf("held.%s.skip.%d", keys.EscapeName(branch), int64(source)), data)
if err != nil {
return nil, err
}
return a.Result, nil
}},
}
}
@@ -64,8 +64,13 @@ type TakeInAsk struct {
NamedFingerprint string `json:"named-fingerprint,omitempty"`
// UnpinRevision and UnpinRecord ask one record's pin on a revision of the state pins to be removed (indexer_unpin).
UnpinRevision string `json:"unpin-revision,omitempty"`
UnpinRecord string `json:"unpin-record,omitempty"`
NamedMember string `json:"named-member,omitempty"`
// Held names the watched branch whose held source-moved is taken in again now (indexer_take_in_pending) or, with Skip and
// HeldSource, ended unwritten (indexer_take_in_skip; design 47 H §744).
Held string `json:"held,omitempty"`
HeldSource int64 `json:"held-source,omitempty"`
Skip bool `json:"skip,omitempty"`
UnpinRecord string `json:"unpin-record,omitempty"`
NamedMember string `json:"named-member,omitempty"`
// RunDrop asks one run on request, by its group run-<id>, to be dropped with its candidate (indexer_run_drop, runs_tools.go, A-194).
RunDrop string `json:"run-drop,omitempty"`
// Registry asks one of the registrations' acts (registrations.go): a trusted change of one, the verb registrations,
@@ -30,9 +30,21 @@ func TakenID(branch string, source int64) string {
// mover (§768). A revision already kept under that name (a
// restart before the acknowledgement) is said again from its kept movers, recomputing nothing (H-A-92). The first watched
// branch is taken in on the indexer's own view; another on its newest revision's, computed and kept only as that revision.
// An error leaves the source-moved held: ErrOldForm never takes it in, ErrObjectsFull waits for room (§744), any other is the
// failed read or write it says.
// An error leaves the source-moved held, said once per source as take-in.pending with branch, source and why (§744):
// ErrObjectsFull waits for the indexer's own room, any other is the failed read or write it says; ErrOldForm is never taken in
// and said as no take-in.
func (ix *Indexer) TakeInSource(ctx context.Context, e meshdelivery.SourceMovedEvent) (meshcatalog.TakeInTaken, error) {
taken, err := ix.takeInSource(ctx, e)
switch {
case err == nil:
ix.unhold(ctx, e.Branch)
case !errors.Is(err, ErrOldForm):
ix.hold(ctx, e, err.Error())
}
return taken, err
}
func (ix *Indexer) takeInSource(ctx context.Context, e meshdelivery.SourceMovedEvent) (meshcatalog.TakeInTaken, error) {
taken := meshcatalog.TakeInTaken{Branch: e.Branch, Source: e.Source, Call: e.Call}
if e.Branch == "" || e.Source <= 0 || e.Call == "" {
return taken, fmt.Errorf("%w: branch %q, source %d, call %q", ErrOldForm, e.Branch, e.Source, e.Call)
@@ -92,6 +104,9 @@ func (ix *Indexer) TakeInSource(ctx context.Context, e meshdelivery.SourceMovedE
movers = nil // a branch's first revision moves nothing: its entries are the base later take-ins are held against (§768, H-A-19)
}
if err := ix.KeepBranchEntries(ctx, e.Branch, e.Source, entries, movers); err != nil {
if errors.Is(err, ErrObjectsFull) {
ix.heldFull.Store(true) // room is said once the store has room again (RoomAgain), and wakes the take-in (§744)
}
return taken, err
}
taken.Revision, taken.Entries = e.Source, movers
@@ -119,3 +134,8 @@ func (ix *Indexer) sayTaken(ctx context.Context, t meshcatalog.TakeInTaken) erro
}
return nil
}
// PendingID is the duplicate id take-in.pending of a branch's source revision is said under: said once per source.
func PendingID(branch string, source int64) string {
return takein.TakeInPendingEvent + "." + keys.EscapeName(branch) + "." + strconv.FormatInt(source, 10)
}
@@ -12,7 +12,7 @@ import (
)
// takeInIndexer is an indexer that takes source revisions in from the catalogue repository's trees at f's commit, keeping its
// revisions and pins, and the take-in events it says.
// revisions and pins, and the take-in events it says (a take-in.pending kept as a taken with its why and skipped set).
func takeInIndexer(t *testing.T) (*Indexer, *memObjects, *[]meshcatalog.TakeInTaken, whole) {
t.Helper()
ix, _, f := newFormIndexerUnmerged(t, mod{"avahi", catalogueManifest(t, "avahi")})
@@ -28,6 +28,10 @@ func takeInIndexer(t *testing.T) (*Indexer, *memObjects, *[]meshcatalog.TakeInTa
ix.ReadFile = func(context.Context, string, string, string) ([]byte, error) { return nil, nil }
said := &[]meshcatalog.TakeInTaken{}
ix.SayTakeIn = func(_ context.Context, event, id string, body any) error {
if p, ok := body.(meshcatalog.TakeInPending); ok && event == "take-in.pending" && id == PendingID(p.Branch, p.Source) {
*said = append(*said, meshcatalog.TakeInTaken{Branch: p.Branch, Source: p.Source, Why: p.Why, Skipped: true})
return nil
}
tk, ok := body.(meshcatalog.TakeInTaken)
if event != "take-in.taken" || !ok || id != TakenID(tk.Branch, tk.Source) {
t.Errorf("said %s %s %+v, want take-in.taken under its branch and source", event, id, body)
@@ -150,3 +154,53 @@ func TestATakeInOfTheCatalogsCommitRewritesPredefined(t *testing.T) {
}
}
}
// Design 47 H §744 (acceptance test 116): a take-in the full catalog store cannot write is held, said once per source as
// take-in.pending with branch, source and why, and room is said once the store has room again; taken in then.
func TestATakeInOnAFullStoreIsHeldAndSaidOnce(t *testing.T) {
ctx := context.Background()
ix, objects, said, f := takeInIndexer(t)
objects.capacity = 10
ix.Held = newMem()
moved := meshdelivery.SourceMovedEvent{V: 1, Branch: "main", Source: 3, Call: "0123456789abcdef",
Heads: []meshdelivery.SourceMovedEventHeadsItem{{Repository: f.Repository, Commit: f.Merge}}}
var rooms int
ix.SayFill = func(_ context.Context, event, _ string, _ meshcatalog.Fill) error {
if event == meshcatalog.EventRoom {
rooms++
}
return nil
}
for range 2 {
if _, err := ix.TakeInSource(ctx, moved); !errors.Is(err, ErrObjectsFull) {
t.Fatalf("a full store answered %v, want ErrObjectsFull", err)
}
}
if len(*said) != 1 || (*said)[0].Branch != "main" || (*said)[0].Source != 3 || (*said)[0].Why == "" || !(*said)[0].Skipped {
t.Fatalf("said %+v, want one take-in.pending of main 3 with its why", *said)
}
if held, err := ReadHeld(ctx, ix.Held); err != nil || len(held) != 1 || held[0].Branch != "main" || held[0].Source != 3 || held[0].Why == "" {
t.Errorf("held = %+v (%v), want main's source revision 3 and why", held, err)
}
objects.capacity = 1 << 22
if ok, err := ix.RoomAgain(ctx); err != nil || !ok || rooms != 1 {
t.Fatalf("room said %v %d times (%v), want once the store has room", ok, rooms, err)
}
if taken, err := ix.TakeInSource(ctx, moved); err != nil || taken.Revision != 3 || len(*said) != 2 {
t.Errorf("taken again on room: %+v (%v), said %d events", taken, err, len(*said))
}
if held, _ := ReadHeld(ctx, ix.Held); len(held) != 0 {
t.Errorf("held = %+v after the take-in, want none", held)
}
// A skip ends a held source revision unwritten: take-in.taken with no entries, skipped, by and why; held is deleted.
ix.hold(ctx, meshdelivery.SourceMovedEvent{Branch: "nox-mesh", Source: 9, Call: "fedcba9876543210"}, "the catalog store is full")
if err := ix.SkipTakeIn(ctx, meshdelivery.SourceMovedEvent{Branch: "nox-mesh", Source: 9, Call: "fedcba9876543210"}, "operator", "a stale source"); err != nil {
t.Fatal(err)
}
if last := (*said)[len(*said)-1]; !last.Skipped || last.By != "operator" || last.Why != "a stale source" || len(last.Entries) != 0 || last.Source != 9 {
t.Errorf("the skip said %+v, want take-in.taken skipped by the operator with its why", last)
}
if held, _ := ReadHeld(ctx, ix.Held); len(held) != 0 {
t.Errorf("held = %+v after the skip, want none", held)
}
}
@@ -137,7 +137,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
{Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5},
{Name: index.EntryIndexState, History: 1, MaxMiB: 8}, {Name: index.NamedState, History: 1, MaxMiB: 4}, // design 47 G §695
{Name: index.RegistrationsState, History: 1}, {Name: index.ProposalsState, History: 1, MaxMiB: 4},
{Name: index.VersionsState, History: 1, MaxMiB: 64}, {Name: index.BuildEdgesState, History: 1}, {Name: index.InputsState, History: 1}, {Name: index.PinsState, History: 1, MaxMiB: index.PinsCapMiB}, {Name: process.ConsumersState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17
{Name: index.VersionsState, History: 1, MaxMiB: 64}, {Name: index.BuildEdgesState, History: 1}, {Name: index.InputsState, History: 1}, {Name: index.PinsState, History: 1, MaxMiB: index.PinsCapMiB}, {Name: process.ConsumersState, History: 1}, {Name: index.HeldState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17
t.Errorf("state = %+v, want %+v", m.State, want)
}
// Milestone 47/G W4.2 (design 47, Two graphs; ADR 0619 §2): the catalog graph revisions are objects in the old section,
@@ -169,7 +169,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
// "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control
// internal/broker/state.go:62-93).
// The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*).
if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-conditions.findings-asked",
if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-catalog.room", "mesh-conditions.findings-asked",
"mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.run-started", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
t.Errorf("consumes = %v, want %v", m.Consumes, want)
}
@@ -204,8 +204,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
// milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered.
index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool,
index.EntryIndexTool, index.NamedDropTool, // design 47 G §675
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.ConsumerTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194)
if !reflect.DeepEqual(m.Tools, want) || len(want) != 41 || !slices.Contains(m.Tools, "modules") {
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.PendingTool, index.TakeInPendingTool, index.TakeInSkipTool, index.ConsumerTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194)
if !reflect.DeepEqual(m.Tools, want) || len(want) != 44 || !slices.Contains(m.Tools, "modules") {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section.
+6 -1
View File
@@ -65,6 +65,9 @@
"indexer_inputs",
"indexer_pins",
"indexer_unpin",
"indexer_pending",
"indexer_take_in_pending",
"indexer_take_in_skip",
"indexer_consumer",
"indexer_runs",
"indexer_run_drop"
@@ -82,6 +85,7 @@
"mesh-wants.wait-ended",
"mesh-catalog.catalog-revision",
"mesh-catalog.registration-changed",
"mesh-catalog.room",
"mesh-delivery.source-moved",
"mesh-wants.wanted",
"mesh-pipeline-runner.run-started",
@@ -115,7 +119,8 @@
{ "name": "build-edges", "history": 1 },
{ "name": "inputs", "history": 1 },
{ "name": "pins", "history": 1, "max-mib": 4 },
{ "name": "consumers", "history": 1 }
{ "name": "consumers", "history": 1 },
{ "name": "held", "history": 1 }
],
"streams": [
{ "name": "registration-history", "events": ["registration.*"], "retention": "history", "max-mib": 64 },