indexer: each start step says when it begins and how long it took, and the build-end replay and package publishing say their progress, so a start that does not end is read in the journal
This commit is contained in:
@@ -509,6 +509,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// node-engine names the successor while it replaces this process, and an old process that learnt it later would take
|
||||
// revisions in with its old judge.
|
||||
ix.FixOwnBuild(ctx)
|
||||
// Each step of the start says when it begins and how long it took, so a start that does not end is read in the journal
|
||||
// and not guessed (lead-47 2026-11-16: two starts on novox ran 25 minutes at one CPU with no line).
|
||||
step := func(name string) func() {
|
||||
began := time.Now()
|
||||
logger.Printf("start: %s", name)
|
||||
return func() { logger.Printf("start: %s done in %s", name, time.Since(began).Round(time.Millisecond)) }
|
||||
}
|
||||
done := step("reading the build log")
|
||||
// The log's entries at the start are read before any fingerprint is computed or registered module indexed (below); each
|
||||
// new one indexes the module it names when that module is registered and the bucket holds none (registered.go).
|
||||
var started atomic.Bool
|
||||
@@ -537,6 +545,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
"indexed from it, until the indexer starts again", builds.BuildLogRead, err)
|
||||
}
|
||||
started.Store(true)
|
||||
done()
|
||||
logger.Printf("start: the build log held %d entries at the start", len(startEnds))
|
||||
// The source revision of a watched branch (design 47 G §15, §115, §362): asked of the seat mesh-delivery's verb source as a
|
||||
// kept call, once at the start, before the handler of its event source-moved reads anything; a source-moved heard
|
||||
// meanwhile is read after.
|
||||
@@ -553,6 +563,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
done = step("the build modules and the seat holders")
|
||||
buildModules.refresh()
|
||||
// The assignments is the seat mesh-module-assigner's: read through its holder's first view and views family, never its bucket (DLV-F1r).
|
||||
assignmentsErrs := make(chan error)
|
||||
@@ -566,11 +577,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
seatHolders := &seatHoldersFollow{ctx: ctx, ix: ix, reader: reader, logf: logger.Printf}
|
||||
seatHolders.refresh()
|
||||
buildModules.seatHolders.Store(seatHolders)
|
||||
done()
|
||||
done = step("following the assignments")
|
||||
followAssignments(ctx, reader, seatHolders.holders(), process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, views.StatusState), "indexer_first_views reads nothing and indexer_first_view_again is not heard", logger.Printf), cred, js, gate, func(ctx context.Context) {
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
logger.Printf("writing the modules again after the assignments of a build module changed: %v", err)
|
||||
}
|
||||
}, logger.Printf)
|
||||
done()
|
||||
done = step("the toolchain watch and the previews")
|
||||
tools := newToolchainWatch(ctx, js, toolchainSink{
|
||||
put: gate.Toolchain,
|
||||
apply: func(ctx context.Context, module, key string, value []byte, deleted bool) error {
|
||||
@@ -597,30 +612,42 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// having no toolchain reported, and back).
|
||||
// The packages the held modules reach and the state does not hold yet are published at their trunk's taken merge, and
|
||||
// the imports the log's done builds reported are then kept, before the fingerprints are computed (ADR 0593 §2, §4).
|
||||
done()
|
||||
packages.Ready(func(jetstream.KeyValue) {
|
||||
done := step("publishing the packages at start")
|
||||
if err := ix.FillPackages(ctx); err != nil {
|
||||
logger.Printf("publishing the packages at start: %v", err)
|
||||
}
|
||||
done()
|
||||
})
|
||||
done = step(fmt.Sprintf("the out-of-step findings of %d build ends", len(startEnds)))
|
||||
values := make([][]byte, len(startEnds))
|
||||
for i, le := range startEnds {
|
||||
ix.ApplyOutOfStepBuild(ctx, le.Value)
|
||||
values[i] = le.Value
|
||||
if (i+1)%2000 == 0 {
|
||||
logger.Printf("start: out-of-step findings: %d of %d entries", i+1, len(startEnds))
|
||||
}
|
||||
}
|
||||
done()
|
||||
if err := ix.ApplyBuildEndsAtStart(ctx, values, func(i int, err error) {
|
||||
logger.Printf("%s entry %d: its imports are not kept: %v", builds.BuildLogRead, startEnds[i].Sequence, err)
|
||||
}); err != nil {
|
||||
logger.Printf("keeping the view after the build ends at start: %v", err)
|
||||
}
|
||||
startEnds = nil
|
||||
done = step("computing the fingerprints")
|
||||
if err := ix.RecomputeAtStart(ctx); err != nil {
|
||||
logger.Printf("computing the new-form fingerprints at start: %v", err)
|
||||
}
|
||||
done()
|
||||
done = step("indexing the registered modules")
|
||||
// A module registered in the catalogue from its own repository gets its manifest indexed when the builder's log names
|
||||
// it and the bucket holds none (registered.go); the entries that follow are heard above.
|
||||
if err := ix.IndexRegisteredAll(ctx); err != nil {
|
||||
logger.Printf("indexing the registered modules at start: %v", err)
|
||||
}
|
||||
done()
|
||||
// A person takes a registered module in now with indexer_take_in, which writes the takeins state (issue 1945): watched,
|
||||
// as every other input. A module the forge could not read is also read again by itself on the next source revision and
|
||||
// merged message (index.RetryUnread).
|
||||
|
||||
@@ -42,8 +42,17 @@ func (ix *Indexer) ApplyBuildEndsAtStart(ctx context.Context, values [][]byte, f
|
||||
began := time.Now()
|
||||
ix.logf("replaying %d build ends of the build log at start", len(values))
|
||||
wrote := 0
|
||||
slow := 0 // entries that took over a second
|
||||
for i, v := range values {
|
||||
if (i+1)%500 == 0 {
|
||||
ix.logf("replaying the build ends at start: %d of %d, %d wrote a module, %d took over a second, %s so far", i+1, len(values), wrote, slow, time.Since(began).Round(time.Millisecond))
|
||||
}
|
||||
t := time.Now()
|
||||
w, err := ix.applyBuildEnd(ctx, v)
|
||||
if d := time.Since(t); d > time.Second {
|
||||
slow++
|
||||
ix.logf("replaying the build ends at start: entry %d took %s", i+1, d.Round(time.Millisecond))
|
||||
}
|
||||
if err != nil {
|
||||
if failed != nil {
|
||||
failed(i, err)
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
|
||||
@@ -347,9 +348,11 @@ func (ix *Indexer) FillPackages(ctx context.Context) error {
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
began := time.Now()
|
||||
if err := ix.publish(ctx, repo, t.Merge, held, true); err != nil {
|
||||
ix.logf("%s: its packages are not published at %s: %v", repo, t.Merge, err)
|
||||
}
|
||||
ix.logf("publishing the packages at start: %s at %.12s took %s", repo, t.Merge, time.Since(began).Round(time.Millisecond))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user