diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index a2eba94a8..09fd60658 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -208,7 +208,7 @@ func main() { _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.PinDropTool, Verb: "pin-drop"}) return err })...) - tools = append(tools, index.HeldTools(keptRead{stdio.State(index.HeldState)}, asks, time.Now, func(what, why string) error { + tools = append(tools, index.HeldTools(keptRead{stdio.State(index.HeldState)}, keptRead{stdio.State(index.TakeInsState)}, asks, time.Now, func(what, why string) error { _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.TakeInSkipTool, Verb: "take-in-skip"}) return err })...) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index c87463d79..aca38fc31 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -331,7 +331,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return fmt.Errorf("answering the verb input: %w", err) } defer stopInput() - ix := &index.Indexer{Module: cred.Module, NamedSets: named, Findings: writer, OldSections: awaitedStore{b: oldSections}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known, + // The state takeins: hand acts ask there (watched below), and a running take-in keeps what it waits on (H2 §803, H-A-409). + takeIns := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.TakeInsState), "no hand act of the indexer is answered", logger.Printf) + ix := &index.Indexer{Module: cred.Module, TakeIns: awaitedStore{b: takeIns}, NamedSets: named, Findings: writer, OldSections: awaitedStore{b: oldSections}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known, Predefined: awaitedStore{b: predefined}, Logf: logger.Printf, Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), BranchHead: branchHeadVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.Bucket(cred.Module, index.PackagesState)}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), @@ -502,7 +504,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro done() // A person's hand act writes the takeins state (issue 1945): watched, as every other input. // A state not on the bus yet at this start is watched once it appears (issue 1790). - process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.TakeInsState), "no hand act of the indexer is answered", logger.Printf).Ready(func(kv jetstream.KeyValue) { + takeIns.Ready(func(kv jetstream.KeyValue) { eachTakeIn := func(e jetstream.KeyValueEntry) { takenIn(ctx, actor{ix, sources}, kv, e, logger.Printf) } if tw, err := settle(ctx, kv, eachTakeIn); err != nil { process.NotWatched(logger.Printf, index.Bucket(cred.Module, index.TakeInsState), err, "no hand act of the indexer is answered until the indexer starts again") diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go index 31c7f4a84..11a52cd33 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "strings" "time" "github.com/nats-io/nats.go/jetstream" @@ -32,8 +33,8 @@ type takerIn interface { // written back, by compare-and-set at its revision, with what the indexer stored or why not. A value // answered already (this indexer's own write) or deleted is left. func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) { - if e.Operation() != jetstream.KeyValuePut { - return + if e.Operation() != jetstream.KeyValuePut || strings.HasPrefix(e.Key(), index.RunningPrefix) { + return // a running take-in's waiting-on is the indexer's own record, no ask (H-A-409) } a, err := index.DecodeTakeIn(e.Value()) if err != nil { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go index 3055a5ac7..474cfc4f8 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go @@ -592,7 +592,7 @@ func TestIndexerWhole(t *testing.T) { t.Errorf("main 14 at main 13's head said %+v, want no mover", next) } var log []string - tools := index.HeldTools(kvStore{held}, askVia(t, ctx, takeinsKV), time.Now, func(what, why string) error { + tools := index.HeldTools(kvStore{held}, kvStore{takeinsKV}, askVia(t, ctx, takeinsKV), time.Now, func(what, why string) error { log = append(log, what+": "+why) return nil }) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_test.go index 713eb631d..495844ebf 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_test.go @@ -92,7 +92,7 @@ func TestEveryHandToolsAskIsReadAndActedOn(t *testing.T) { } record := func(string, string) error { return nil } tools := map[string]func(map[string]any) (any, error){} - for _, tl := range slices.Concat(index.HeldTools(nil, ask, time.Now, record), index.PinsTools(nil, nil, ask, time.Now, record), + for _, tl := range slices.Concat(index.HeldTools(nil, nil, ask, time.Now, record), index.PinsTools(nil, nil, ask, time.Now, record), index.RunsTools(nil, nil, ask, time.Now, record), index.NamedTools(ask, time.Now, record), index.FingerprintsTools(ask, time.Now, record, time.Minute), index.CatalogRevisionTools(nil, ask, time.Now, record), index.RegistrationTools(nil, nil, ask, time.Now, func(string, string, string, string) error { return nil })) { diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index b880cdb9c..90dbbde3c 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -126,8 +126,13 @@ type Indexer struct { // content; nil reads none. GoMod func(ctx context.Context, repository, sha, dir string) ([]byte, error) // Now is the clock of a taken value; nil is time.Now. - Now func() time.Time - Builds *Builds + Now func() time.Time + // Bound bounds a running take-in's wait (takein_waiting.go), a context's bound and no timer (ADR 0437 §8); nil is + // context.WithTimeout. + Bound func(context.Context, time.Duration) (context.Context, context.CancelFunc) + // TakeIns is the state takeins, where a running take-in keeps what it waits on (design 47 H2 §803, H-A-409); nil keeps none. + TakeIns Store + Builds *Builds // Named asks the forge for the git object of each named path of the module directories given, at a commit // (the seat git's trees with paths; novox/hq ADR 0499 clauses 2 and 3): the indexer is the one reader of a // manifest's paths (ADR 0495 clause 1, the lead's decision L8 of R4e). Nil asks nothing, and a module naming paths diff --git a/modules/mesh-delivery-indexer/internal/index/dryrun.go b/modules/mesh-delivery-indexer/internal/index/dryrun.go index bdf7a4bdb..2fe12d418 100644 --- a/modules/mesh-delivery-indexer/internal/index/dryrun.go +++ b/modules/mesh-delivery-indexer/internal/index/dryrun.go @@ -29,7 +29,9 @@ func (ix *Indexer) dry() *dryRun { } d := &dryRun{} x := &Indexer{Module: ix.Module, Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum, - GoMod: ix.GoMod, Now: ix.Now, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, BranchHead: ix.BranchHead, Read: ix.Read, Contains: ix.Contains, + GoMod: ix.GoMod, Now: ix.Now, Bound: ix.Bound, + // A dry run keeps no waiting-on: the take-in it computes for says its waits through the live indexer (H-A-409). + TakeIns: nil, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, BranchHead: ix.BranchHead, Read: ix.Read, Contains: ix.Contains, SayTakeIn: nil, // The registry milestone's fields: what a take-in would write on the bus stays out of a dry run, so the states of the // requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is diff --git a/modules/mesh-delivery-indexer/internal/index/findings.go b/modules/mesh-delivery-indexer/internal/index/findings.go index 542c707b2..f21c16227 100644 --- a/modules/mesh-delivery-indexer/internal/index/findings.go +++ b/modules/mesh-delivery-indexer/internal/index/findings.go @@ -22,7 +22,7 @@ const CallUnansweredKind = "indexer-call-unanswered" // H-A-242, H-A-247): its own, the process reader's event-unhandled and the family-not-heard of the heard writer it is handed // (H-A-247 (1)); a views reader's and the emitter's (views.ViewKinds) are always the writer's besides. The tool runner's // writer over the same state `findings` owns its own. TestFindingKindsHoldEveryKindTheCodeRaises holds the code to this list. -var FindingKinds = []string{PinMissingKind, OutOfStepKind, PackagesNotPublishedKind, HistoryFillingKind, SeedWaitsKind, +var FindingKinds = []string{TakeInWaitingKind, PinMissingKind, OutOfStepKind, PackagesNotPublishedKind, HistoryFillingKind, SeedWaitsKind, SeedLeftOutKind, SeatDefinitionUnparsedKind, SeatDefinedTwiceKind, SeatOutsideItsDomainKind, CallUnansweredKind, views.EventUnhandledKind, heard.Kind} diff --git a/modules/mesh-delivery-indexer/internal/index/held.go b/modules/mesh-delivery-indexer/internal/index/held.go index 5e8774ad8..ca1e685da 100644 --- a/modules/mesh-delivery-indexer/internal/index/held.go +++ b/modules/mesh-delivery-indexer/internal/index/held.go @@ -49,6 +49,8 @@ func (h HeldTakeIn) Event() meshdelivery.SourceMovedEvent { // PendingAnswer is indexer_pending's answer (H-A-102): every branch's held take-ins, in the state held's form (since in RFC 3339). type PendingAnswer struct { Held []HeldTakeIn `json:"held"` + // Running are the take-ins that run, each with what it waits on (design 47 H2 §803, H-A-409). + Running []RunningTakeIn `json:"running,omitempty"` } // ErrHeldUnwritten says a source-moved that could not be taken in is not written into the state held either: its message @@ -224,20 +226,25 @@ func ReadHeld(ctx context.Context, st Store) ([]HeldTakeIn, error) { return out, nil } -// HeldTools are the held take-in's tools (§744): indexer_pending reads the state held; indexer_take_in_pending takes a branch's +// HeldTools are the held take-in's tools (§744): indexer_pending reads the state held and the running take-ins' waiting-on in +// the state takeins (H2 §803, H-A-409); indexer_take_in_pending takes a branch's // held source-moved again now and indexer_take_in_skip acknowledges it unwritten, each an ask in the state takeins the indexer // acts on (H-A-97), the skip a hand act recorded through mesh-hand-acts before it is asked. -func HeldTools(held Store, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time, record func(what, why string) error) []stdio.Tool { +func HeldTools(held, takeIns Store, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time, record func(what, why string) error) []stdio.Tool { str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } return []stdio.Tool{ {Name: PendingTool, Description: "Each watched branch's held take-in (design 47 H §744): the source revision of the gate's source-moved the indexer " + "could not take in (the catalog store full, a read of the forge unanswered), why and since when, in order, each kept here once " + - "acknowledged and taken in again on the indexer's room and at its start. indexer_take_in_pending takes it again now, indexer_take_in_skip ends it unwritten.", + "acknowledged and taken in again on the indexer's room and at its start; and each running take-in that waits over a minute on another part, its waiting-on (part, verb, since). indexer_take_in_pending takes it again now, indexer_take_in_skip ends it unwritten.", Input: map[string]any{"type": "object", "properties": map[string]any{}}, Run: func(map[string]any) (any, error) { h, err := ReadHeld(context.Background(), held) - return PendingAnswer{Held: h}, err + if err != nil { + return PendingAnswer{Held: h}, err + } + r, err := ReadRunning(context.Background(), takeIns) + return PendingAnswer{Held: h, Running: r}, err }}, {Name: TakeInPendingTool, Description: "Take a watched branch's held source-moved in again now (design 47 H §744), as the indexer's room would: " + diff --git a/modules/mesh-delivery-indexer/internal/index/held_test.go b/modules/mesh-delivery-indexer/internal/index/held_test.go index 976e29eb0..fd9c77ca0 100644 --- a/modules/mesh-delivery-indexer/internal/index/held_test.go +++ b/modules/mesh-delivery-indexer/internal/index/held_test.go @@ -26,7 +26,7 @@ func TestTheHeldAndPinToolsAskWhatTheIndexerReads(t *testing.T) { } record := func(what, _ string) error { recorded = append(recorded, what); return nil } tools := map[string]stdio.Tool{} - for _, tl := range append(HeldTools(newMem(), ask, time.Now, record), PinsTools(newMem(), newMemObjects(0), ask, time.Now, record)...) { + for _, tl := range append(HeldTools(newMem(), nil, ask, time.Now, record), PinsTools(newMem(), newMemObjects(0), ask, time.Now, record)...) { tools[tl.Name] = tl } for _, c := range []struct { @@ -56,7 +56,7 @@ func TestTheHeldAndPinToolsAskWhatTheIndexerReads(t *testing.T) { held := newMem() ix := &Indexer{Held: held} ix.hold(t.Context(), meshdelivery.SourceMovedEvent{Branch: "main", Source: 4, Call: "0123456789abcdef"}, "the catalog store is full") - got, err := HeldTools(held, ask, time.Now, record)[0].Run(map[string]any{}) + got, err := HeldTools(held, nil, ask, time.Now, record)[0].Run(map[string]any{}) data, _ := json.Marshal(got) var answer struct { Held []struct { diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 9a59b83d6..60321160f 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -227,7 +227,7 @@ func (ix *Indexer) namedObjects(ctx context.Context, f whole) (map[string]map[st if len(need) == 0 || ix.Named == nil { return objects, nil } - got, err := ix.Named(ctx, f.Repository, f.Merge, need) + got, err := waited(ctx, "git", "trees", func() (map[string]map[string]string, error) { return ix.Named(ctx, f.Repository, f.Merge, need) }) if err != nil { words := map[string]string{} for dir := range need { @@ -320,7 +320,7 @@ func (ix *Indexer) reconcile(ctx context.Context) error { if len(mine) == 0 { continue } - dirs, err := ix.Trunk(ctx, repository, heads[repository]) + dirs, err := waited(ctx, "git", "trees", func() ([]string, error) { return ix.Trunk(ctx, repository, heads[repository]) }) if err != nil { ix.logf("%s at %s: the modules of the trunk cannot be read, so none is dropped: %v", repository, heads[repository], err) continue @@ -346,7 +346,7 @@ func (ix *Indexer) reconcile(ctx context.Context) error { if m.Commit == "" || ix.Contains == nil { return true } - in, err := ix.Contains(ctx, repository, m.Commit, head) + in, err := waited(ctx, "git", "contains", func() (bool, error) { return ix.Contains(ctx, repository, m.Commit, head) }) if err != nil { ix.logf("%s: whether %s is in %s cannot be read, so the module at %s is kept: %v", repository, m.Commit, head, m.Path, err) return true @@ -421,7 +421,7 @@ func (ix *Indexer) goModOf(ctx context.Context, f whole, dir string) ([]byte, bo if ix.GoMod == nil { return nil, false } - raw, err := ix.GoMod(ctx, f.Repository, f.Merge, dir) + raw, err := waited(ctx, "git", "file", func() ([]byte, error) { return ix.GoMod(ctx, f.Repository, f.Merge, dir) }) if err != nil { ix.logf("%s at %s: the go.mod of %q cannot be read, so its module keeps the packages it held: %v", f.Repository, f.Merge, dir, err) return nil, false @@ -444,7 +444,7 @@ func (ix *Indexer) requirementsOf(ctx context.Context, f whole, dir string, cur } } } - raw, err := ix.GoSum(ctx, f.Repository, f.Merge, dir) + raw, err := waited(ctx, "git", "file", func() ([]byte, error) { return ix.GoSum(ctx, f.Repository, f.Merge, dir) }) if err != nil { ix.logf("%s at %s: the go.sum of %q cannot be read, so what it requires is not kept: %v", f.Repository, f.Merge, dir, err) return diff --git a/modules/mesh-delivery-indexer/internal/index/moves.go b/modules/mesh-delivery-indexer/internal/index/moves.go index 03c7de2b6..17088ef94 100644 --- a/modules/mesh-delivery-indexer/internal/index/moves.go +++ b/modules/mesh-delivery-indexer/internal/index/moves.go @@ -26,7 +26,7 @@ func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository st if head == "" || trunk.SameRepository(other, repository) { continue } - dirs, err := ix.Trunk(ctx, other, head) + dirs, err := waited(ctx, "git", "trees", func() ([]string, error) { return ix.Trunk(ctx, other, head) }) if err != nil { ix.logf("%s at %s: the modules of the trunk cannot be read, so %s is not looked for there: %v", other, head, module, err) continue @@ -36,7 +36,7 @@ func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository st if nameOfDirectory(dir) != module { continue } - read, err := ix.Read(ctx, other, head, dir) + read, err := waited(ctx, "git", "trees", func() (RepoRead, error) { return ix.Read(ctx, other, head, dir) }) if err != nil { ix.logf("%s at %s: %q cannot be read, so %s is not handed over from it: %v", other, head, dir, module, err) continue diff --git a/modules/mesh-delivery-indexer/internal/index/order.go b/modules/mesh-delivery-indexer/internal/index/order.go index 395cebd96..90a99491f 100644 --- a/modules/mesh-delivery-indexer/internal/index/order.go +++ b/modules/mesh-delivery-indexer/internal/index/order.go @@ -67,7 +67,7 @@ func (ix *Indexer) older(ctx context.Context, repository, commit, later string) if ix.Contains == nil || commit == "" || later == "" || commit == later { return false, nil } - in, err := ix.Contains(ctx, repository, commit, later) + in, err := waited(ctx, "git", "contains", func() (bool, error) { return ix.Contains(ctx, repository, commit, later) }) if notFound(err) { ix.logf("%s: the forge does not have %.12s or %.12s (a rewritten history?), so %.12s is not known to be older and is written: %v", repository, commit, later, commit, err) return false, nil diff --git a/modules/mesh-delivery-indexer/internal/index/package_trees.go b/modules/mesh-delivery-indexer/internal/index/package_trees.go index 128150fff..9cb36c2e6 100644 --- a/modules/mesh-delivery-indexer/internal/index/package_trees.go +++ b/modules/mesh-delivery-indexer/internal/index/package_trees.go @@ -36,7 +36,7 @@ func (t *goModTree) goMod(dir string) []byte { var raw []byte if t.ix.GoMod != nil { var err error - if raw, err = t.ix.GoMod(t.ctx, t.f.Repository, t.f.Merge, dir); err != nil { + if raw, err = waited(t.ctx, "git", "file", func() ([]byte, error) { return t.ix.GoMod(t.ctx, t.f.Repository, t.f.Merge, dir) }); err != nil { t.ix.logf("%s at %s: the go.mod of %q cannot be read, so no module of the tree is found there: %v", t.f.Repository, t.f.Merge, dir, err) raw = nil } @@ -167,7 +167,9 @@ func (ix *Indexer) derivedTree(ctx context.Context, f whole, t *goModTree, m cat if ix.Named == nil { return "", "the forge cannot be asked the git objects of the derived directories " + strings.Join(dirs, ", ") } - got, err := ix.Named(ctx, f.Repository, f.Merge, map[string][]string{"": dirs}) + got, err := waited(ctx, "git", "trees", func() (map[string]map[string]string, error) { + return ix.Named(ctx, f.Repository, f.Merge, map[string][]string{"": dirs}) + }) if err != nil { return "", fmt.Sprintf("the git objects of the derived directories %s: %v", strings.Join(dirs, ", "), err) } diff --git a/modules/mesh-delivery-indexer/internal/index/packages.go b/modules/mesh-delivery-indexer/internal/index/packages.go index d898e2c9a..a730f4d14 100644 --- a/modules/mesh-delivery-indexer/internal/index/packages.go +++ b/modules/mesh-delivery-indexer/internal/index/packages.go @@ -174,7 +174,7 @@ func (ix *Indexer) packagesIn(ctx context.Context, repository, commit string, mo if len(ask) == 0 { return nil } - got, err := ix.Files(ctx, repository, commit, ask) + got, err := waited(ctx, "git", "trees", func() (map[string]map[string]string, error) { return ix.Files(ctx, repository, commit, ask) }) if err != nil { return fmt.Errorf("the forge's listing of %s at %s (%s): %w", repository, commit, strings.Join(ask, ", "), err) } @@ -231,7 +231,7 @@ func (ix *Indexer) packagesIn(ctx context.Context, repository, commit string, mo if ix.Trunk == nil { return nil, nil, whole } - if _, err := ix.Trunk(ctx, repository, commit); err != nil { + if _, err := waited(ctx, "git", "trees", func() ([]string, error) { return ix.Trunk(ctx, repository, commit) }); err != nil { return nil, nil, fmt.Errorf("%w (and the commit's trees: %v)", whole, err) } ix.logf("%s at %s: the commit holds a tree, so every directory asked (%s) is gone", repository, commit, strings.Join(dirs, ", ")) diff --git a/modules/mesh-delivery-indexer/internal/index/source.go b/modules/mesh-delivery-indexer/internal/index/source.go index 55fbf1937..ea7531504 100644 --- a/modules/mesh-delivery-indexer/internal/index/source.go +++ b/modules/mesh-delivery-indexer/internal/index/source.go @@ -32,9 +32,9 @@ func (ix *Indexer) wholeAt(ctx context.Context, repository, head string) (whole, if ix.ReadFile == nil { return nil, errors.New("the indexer reads no file of the forge") } - return ix.ReadFile(ctx, repository, head, p) + return waited(ctx, "git", "file", func() ([]byte, error) { return ix.ReadFile(ctx, repository, head, p) }) } - first, err := ix.Read(ctx, repository, head, "") + first, err := waited(ctx, "git", "trees", func() (RepoRead, error) { return ix.Read(ctx, repository, head, "") }) if err != nil { return f, fmt.Errorf("the seat git could not read %s at %.12s: %w", repository, head, err) } @@ -42,7 +42,7 @@ func (ix *Indexer) wholeAt(ctx context.Context, repository, head string) (whole, for _, dir := range slices.Sorted(maps.Keys(first.Trees)) { read := first if dir != "" { - if read, err = ix.Read(ctx, repository, head, dir); err != nil { + if read, err = waited(ctx, "git", "trees", func() (RepoRead, error) { return ix.Read(ctx, repository, head, dir) }); err != nil { return f, fmt.Errorf("the seat git could not read %q of %s at %.12s: %w", dir, repository, head, err) } } diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source.go b/modules/mesh-delivery-indexer/internal/index/takein_source.go index 667eb11a2..ecb392210 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein_source.go +++ b/modules/mesh-delivery-indexer/internal/index/takein_source.go @@ -38,7 +38,8 @@ func TakenID(branch string, source int64) string { // ErrObjectsFull waits for the indexer's own room, any other is the failed read or write it says; ErrOldForm is never taken in // and said as no take-in. func (ix *Indexer) TakeInSource(ctx context.Context, e meshdelivery.SourceMovedEvent) (meshcatalog.TakeInTaken, error) { - taken, err := ix.takeInSource(ctx, e) + taken, err := ix.takeInSource(ix.withTakeIn(ctx, e.Branch, e.Source), e) + ix.endTakeIn(ctx, e.Source) switch { case err == nil: ix.unhold(ctx, e.Branch, e.Source) diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go index 2e439e85d..f0da42a54 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go +++ b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go @@ -6,7 +6,14 @@ import ( "errors" "fmt" "reflect" + "slices" + "strings" + "sync" "testing" + "time" + + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" @@ -98,6 +105,110 @@ func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) { if _, err := ix.TakeInSource(ctx, old); !errors.Is(err, ErrOldForm) { t.Errorf("a source-moved with no call answered %v, want ErrOldForm", err) } + // Design 47 H2 §803, H-A-409: a take-in says what it waits on, in the journal, in takeins and past five minutes as a finding. + t.Run("a take-in says what it waits on", func(t *testing.T) { + running := newMem() + ix.TakeIns = running + ix.Findings = newFindings() + // The take-in's end removes its waiting-on. + if err := running.Create(ctx, RunningKey(10), []byte(`{"source":10}`)); err != nil { + t.Fatal(err) + } + next := moved + next.Source, next.Call = 10, "00112233445566ff" + if _, err := ix.TakeInSource(ctx, next); err != nil { + t.Fatal(err) + } + if _, _, found, _ := running.Get(ctx, RunningKey(10)); found { + t.Error("the take-in's end left its waiting-on in takeins") + } + start := time.Date(2026, 11, 22, 1, 0, 0, 0, time.UTC) + now := start + timers := make(chan context.CancelFunc) + var mu sync.Mutex + var lines []string + wrote := make(chan struct{}, 1) + findings := newFindings() + w := &Indexer{Module: testSelf, TakeIns: running, Findings: findings, Now: func() time.Time { mu.Lock(); defer mu.Unlock(); return now }, + Bound: func(parent context.Context, _ time.Duration) (context.Context, context.CancelFunc) { + b, cancel := context.WithCancel(parent) + timers <- cancel + return b, cancel + }, + Logf: func(format string, args ...any) { + mu.Lock() + defer mu.Unlock() + lines = append(lines, fmt.Sprintf(format, args...)) + }} + w.Findings = findingsSignal{findings, wrote} + listing := func(answerAfter time.Duration) { + t.Helper() + mu.Lock() + now, lines = start, nil + mu.Unlock() + release, res := make(chan struct{}), make(chan error) + go func() { + _, err := waited(w.withTakeIn(ctx, "main", 9), "git", "trees", func() (int, error) { <-release; return 1, nil }) + res <- err + }() + first := <-timers + if answerAfter < WaitingLineAfter { + close(release) + <-res + return + } + mu.Lock() + now = start.Add(70 * time.Second) + mu.Unlock() + first() + second := <-timers // asked once the line is said and the waiting-on written + if answerAfter >= WaitingFindingAfter { + second() + <-wrote + if h := findings.Held(); len(h) != 1 || h[0].Kind != TakeInWaitingKind || h[0].Scope != mcc.ScopeMerge || h[0].ID != "9" { + t.Errorf("after %s: findings %+v, want take-in-waiting of merge 9", answerAfter, h) + } + } + close(release) + if err := <-res; err != nil { + t.Fatal(err) + } + } + for _, c := range []struct { + after time.Duration + lines int + finding bool + }{{30 * time.Second, 0, false}, {70 * time.Second, 1, false}, {6 * time.Minute, 1, false}} { + listing(c.after) + mu.Lock() + got := slices.Clone(lines) + mu.Unlock() + if len(got) != c.lines || (c.lines == 1 && !strings.Contains(got[0], "waits on git trees since 2026-11-22T01:00:00Z")) { + t.Errorf("answered after %s: lines %q, want %d naming git trees and since when", c.after, got, c.lines) + } + r, err := ReadRunning(ctx, running) + want := c.lines == 1 + if err != nil || want != (len(r) == 1) || (want && (r[0].Source != 9 || r[0].Branch != "main" || r[0].WaitingOn != WaitingOn{Part: "git", Verb: "trees", Since: start})) { + t.Errorf("answered after %s: running %+v (%v), want waiting-on git trees since %s: %v", c.after, r, err, start, want) + } + if len(findings.Held()) != 0 { + t.Errorf("answered after %s: take-in-waiting still raised: %+v", c.after, findings.Held()) + } + w.endTakeIn(ctx, 9) + } + }) +} + +// findingsSignal is a finding writer that says each write it passes on. +type findingsSignal struct { + *memFindings + wrote chan struct{} +} + +func (f findingsSignal) Write(ctx context.Context, x mcseat.Finding) error { + err := f.memFindings.Write(ctx, x) + f.wrote <- struct{}{} + return err } // Design 47 H §768, §743, H-A-19: a branch's first source revision (start true) is taken in with no mover, says take-in.taken diff --git a/modules/mesh-delivery-indexer/internal/index/takein_waiting.go b/modules/mesh-delivery-indexer/internal/index/takein_waiting.go new file mode 100644 index 000000000..e732b658f --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/takein_waiting.go @@ -0,0 +1,194 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "time" + + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" +) + +// A take-in says what it waits on (design 47 H2 §803, H-A-409): a source-moved take-in that waits longer than +// WaitingLineAfter on another part (the seat git's listing or a file, a package's versions, a registry) writes one journal +// line naming the part, the verb and since when, and keeps its waiting-on in the state takeins under the key +// RunningPrefix and the source revision, rewritten at each new wait and removed at the take-in's end; past +// WaitingFindingAfter it raises the finding take-in-waiting (scope merge, id the source revision), cleared when the wait +// ends. indexer_pending answers it beside the held take-ins. +const ( + TakeInWaitingKind = "take-in-waiting" + RunningPrefix = "running." + WaitingLineAfter = time.Minute + WaitingFindingAfter = 5 * time.Minute +) + +// WaitingOn is what a running take-in waits on: the part, its verb and since when. +type WaitingOn struct { + Part string `json:"part"` + Verb string `json:"verb"` + Since time.Time `json:"since"` +} + +// RunningTakeIn is the value of a running take-in in the state takeins (H-A-409). +type RunningTakeIn struct { + Branch string `json:"branch"` + Source int64 `json:"source"` + WaitingOn WaitingOn `json:"waiting-on"` +} + +// RunningKey is the takeins key of the running take-in of a source revision. +func RunningKey(source int64) string { return RunningPrefix + strconv.FormatInt(source, 10) } + +type takeInRunKey struct{} + +// takeInRun is the running take-in a context carries, and the live indexer that says its waits (a branch's take-in +// computes on a dry copy, which writes nothing). +type takeInRun struct { + ix *Indexer + branch string + source int64 +} + +// withTakeIn is ctx carrying the running take-in of the source revision of branch. +func (ix *Indexer) withTakeIn(ctx context.Context, branch string, source int64) context.Context { + return context.WithValue(ctx, takeInRunKey{}, &takeInRun{ix: ix, branch: branch, source: source}) +} + +// endTakeIn removes the running take-in's value from the state takeins, when one was written. +func (ix *Indexer) endTakeIn(ctx context.Context, source int64) { + if ix.TakeIns == nil { + return + } + key := RunningKey(source) + _, rev, found, err := ix.TakeIns.Get(ctx, key) + if err == nil && found { + err = ix.TakeIns.Delete(ctx, key, rev) + } + if err != nil && !errors.Is(err, ErrConflict) { + ix.logf("take-in: source revision %d: its waiting-on is not removed from %s: %v", source, TakeInsState, err) + } +} + +// bound is ctx bounded by d, a context's bound and no timer (ADR 0437 §8, as fingerprints.go bounds its wait); Bound +// when set. +func (ix *Indexer) bound(ctx context.Context, d time.Duration) (context.Context, context.CancelFunc) { + if ix.Bound != nil { + return ix.Bound(ctx, d) + } + return context.WithTimeout(ctx, d) +} + +// passed is whether the wait outlasted the bound b: false once it ended (done) or the take-in's own context ended. +func passed(ctx, b context.Context, done <-chan struct{}) bool { + select { + case <-done: + return false + case <-b.Done(): + } + select { + case <-done: + return false + default: + } + return ctx.Err() == nil +} + +// waited is f, a call on part's verb: within a running take-in (ctx), a wait past WaitingLineAfter is said and kept, one +// past WaitingFindingAfter raises take-in-waiting until it ends. Outside a take-in it is f alone. +func waited[T any](ctx context.Context, part, verb string, f func() (T, error)) (T, error) { + run, ok := ctx.Value(takeInRunKey{}).(*takeInRun) + if !ok { + return f() + } + done, finished := make(chan struct{}), make(chan struct{}) + go func() { + defer close(finished) + run.ix.watchWait(ctx, run, part, verb, done) + }() + v, err := f() + close(done) + <-finished + return v, err +} + +func (ix *Indexer) watchWait(ctx context.Context, run *takeInRun, part, verb string, done <-chan struct{}) { + since := ix.now().UTC() + line, cancel := ix.bound(ctx, WaitingLineAfter) + defer cancel() + if !passed(ctx, line, done) { + return + } + ix.logf("take-in: source revision %d of %s waits on %s %s since %s", run.source, run.branch, part, verb, since.Format(time.RFC3339)) + ix.writeWaiting(ctx, RunningTakeIn{Branch: run.branch, Source: run.source, WaitingOn: WaitingOn{Part: part, Verb: verb, Since: since}}) + id := strconv.FormatInt(run.source, 10) + finding, cancelFinding := ix.bound(ctx, WaitingFindingAfter-WaitingLineAfter) + defer cancelFinding() + if !passed(ctx, finding, done) { + return + } + if ix.Findings != nil { + f := ix.observed(mcc.ScopeMerge, id, TakeInWaitingKind, + fmt.Sprintf("the take-in of source revision %d of %s waits on %s %s", run.source, run.branch, part, verb), + "A take-in waits on another part", + fmt.Sprintf("The take-in of source revision %d of %s has waited on %s's %s since %s, so the revision is not taken in.", run.source, run.branch, part, verb, since.Format(time.RFC3339)), + "indexer."+TakeInWaitingKind) + if err := ix.Findings.Write(ctx, f); err != nil { + ix.logf("take-in: the finding %s is not written: %v", TakeInWaitingKind, err) + } + } + <-done + if ix.Findings != nil { + if err := ix.clearFinding(context.WithoutCancel(ctx), mcc.ScopeMerge, id, TakeInWaitingKind, part+" "+verb+" answered"); err != nil { + ix.logf("take-in: the finding %s is not cleared: %v", TakeInWaitingKind, err) + } + } +} + +// writeWaiting writes the running take-in's value in the state takeins, over the one it holds. +func (ix *Indexer) writeWaiting(ctx context.Context, v RunningTakeIn) { + if ix.TakeIns == nil { + return + } + raw, _ := json.Marshal(v) + key := RunningKey(v.Source) + _, rev, found, err := ix.TakeIns.Get(ctx, key) + switch { + case err != nil: + case found: + err = ix.TakeIns.Update(ctx, key, raw, rev) + default: + err = ix.TakeIns.Create(ctx, key, raw) + } + if err != nil { + ix.logf("take-in: source revision %d: its waiting-on is not written in %s: %v", v.Source, TakeInsState, err) + } +} + +// ReadRunning is every running take-in's value of the state takeins (indexer_pending's running part); none for no state. +func ReadRunning(ctx context.Context, st Store) ([]RunningTakeIn, error) { + if st == nil { + return nil, nil + } + keys, err := st.Keys(ctx) + if err != nil { + return nil, err + } + var out []RunningTakeIn + for _, k := range keys { + if len(k) <= len(RunningPrefix) || k[:len(RunningPrefix)] != RunningPrefix { + continue + } + raw, _, found, err := st.Get(ctx, k) + if err != nil { + return nil, err + } + var r RunningTakeIn + if !found || json.Unmarshal(raw, &r) != nil { + continue + } + out = append(out, r) + } + return out, nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/taken.go b/modules/mesh-delivery-indexer/internal/index/taken.go index 3d5df687b..4727a25c3 100644 --- a/modules/mesh-delivery-indexer/internal/index/taken.go +++ b/modules/mesh-delivery-indexer/internal/index/taken.go @@ -43,7 +43,7 @@ func (ix *Indexer) writeTakenKnown(ctx context.Context, f whole, notOlder string return nil, false, nil } if ix.Contains != nil && cur.Merge != notOlder { - later, err := ix.Contains(ctx, f.Repository, f.Merge, cur.Merge) + later, err := waited(ctx, "git", "contains", func() (bool, error) { return ix.Contains(ctx, f.Repository, f.Merge, cur.Merge) }) if err != nil { ix.logf("%s: whether %s is in %s cannot be read, so taken.%s stays at %s: %v", f.Repository, f.Merge, cur.Merge, key, cur.Merge, err) return nil, false, nil