indexer: each start step says when it begins and how long it took, and the build-end replay and package publishing say their progress, so a start that does not end is read in the journal

This commit is contained in:
2026-11-16 02:13:58 +01:00
parent 2979bc3b84
commit e4302579ce
3 changed files with 39 additions and 0 deletions
@@ -509,6 +509,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// node-engine names the successor while it replaces this process, and an old process that learnt it later would take
// revisions in with its old judge.
ix.FixOwnBuild(ctx)
// Each step of the start says when it begins and how long it took, so a start that does not end is read in the journal
// and not guessed (lead-47 2026-11-16: two starts on novox ran 25 minutes at one CPU with no line).
step := func(name string) func() {
began := time.Now()
logger.Printf("start: %s", name)
return func() { logger.Printf("start: %s done in %s", name, time.Since(began).Round(time.Millisecond)) }
}
done := step("reading the build log")
// The log's entries at the start are read before any fingerprint is computed or registered module indexed (below); each
// new one indexes the module it names when that module is registered and the bucket holds none (registered.go).
var started atomic.Bool
@@ -537,6 +545,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
"indexed from it, until the indexer starts again", builds.BuildLogRead, err)
}
started.Store(true)
done()
logger.Printf("start: the build log held %d entries at the start", len(startEnds))
// The source revision of a watched branch (design 47 G §15, §115, §362): asked of the seat mesh-delivery's verb source as a
// kept call, once at the start, before the handler of its event source-moved reads anything; a source-moved heard
// meanwhile is read after.
@@ -553,6 +563,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
return nil
}}
done = step("the build modules and the seat holders")
buildModules.refresh()
// The assignments is the seat mesh-module-assigner's: read through its holder's first view and views family, never its bucket (DLV-F1r).
assignmentsErrs := make(chan error)
@@ -566,11 +577,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
seatHolders := &seatHoldersFollow{ctx: ctx, ix: ix, reader: reader, logf: logger.Printf}
seatHolders.refresh()
buildModules.seatHolders.Store(seatHolders)
done()
done = step("following the assignments")
followAssignments(ctx, reader, seatHolders.holders(), process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, views.StatusState), "indexer_first_views reads nothing and indexer_first_view_again is not heard", logger.Printf), cred, js, gate, func(ctx context.Context) {
if err := ix.Recompute(ctx); err != nil {
logger.Printf("writing the modules again after the assignments of a build module changed: %v", err)
}
}, logger.Printf)
done()
done = step("the toolchain watch and the previews")
tools := newToolchainWatch(ctx, js, toolchainSink{
put: gate.Toolchain,
apply: func(ctx context.Context, module, key string, value []byte, deleted bool) error {
@@ -597,30 +612,42 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// having no toolchain reported, and back).
// The packages the held modules reach and the state does not hold yet are published at their trunk's taken merge, and
// the imports the log's done builds reported are then kept, before the fingerprints are computed (ADR 0593 §2, §4).
done()
packages.Ready(func(jetstream.KeyValue) {
done := step("publishing the packages at start")
if err := ix.FillPackages(ctx); err != nil {
logger.Printf("publishing the packages at start: %v", err)
}
done()
})
done = step(fmt.Sprintf("the out-of-step findings of %d build ends", len(startEnds)))
values := make([][]byte, len(startEnds))
for i, le := range startEnds {
ix.ApplyOutOfStepBuild(ctx, le.Value)
values[i] = le.Value
if (i+1)%2000 == 0 {
logger.Printf("start: out-of-step findings: %d of %d entries", i+1, len(startEnds))
}
}
done()
if err := ix.ApplyBuildEndsAtStart(ctx, values, func(i int, err error) {
logger.Printf("%s entry %d: its imports are not kept: %v", builds.BuildLogRead, startEnds[i].Sequence, err)
}); err != nil {
logger.Printf("keeping the view after the build ends at start: %v", err)
}
startEnds = nil
done = step("computing the fingerprints")
if err := ix.RecomputeAtStart(ctx); err != nil {
logger.Printf("computing the new-form fingerprints at start: %v", err)
}
done()
done = step("indexing the registered modules")
// A module registered in the catalogue from its own repository gets its manifest indexed when the builder's log names
// it and the bucket holds none (registered.go); the entries that follow are heard above.
if err := ix.IndexRegisteredAll(ctx); err != nil {
logger.Printf("indexing the registered modules at start: %v", err)
}
done()
// A person takes a registered module in now with indexer_take_in, which writes the takeins state (issue 1945): watched,
// as every other input. A module the forge could not read is also read again by itself on the next source revision and
// merged message (index.RetryUnread).
@@ -42,8 +42,17 @@ func (ix *Indexer) ApplyBuildEndsAtStart(ctx context.Context, values [][]byte, f
began := time.Now()
ix.logf("replaying %d build ends of the build log at start", len(values))
wrote := 0
slow := 0 // entries that took over a second
for i, v := range values {
if (i+1)%500 == 0 {
ix.logf("replaying the build ends at start: %d of %d, %d wrote a module, %d took over a second, %s so far", i+1, len(values), wrote, slow, time.Since(began).Round(time.Millisecond))
}
t := time.Now()
w, err := ix.applyBuildEnd(ctx, v)
if d := time.Since(t); d > time.Second {
slow++
ix.logf("replaying the build ends at start: entry %d took %s", i+1, d.Round(time.Millisecond))
}
if err != nil {
if failed != nil {
failed(i, err)
@@ -10,6 +10,7 @@ import (
"reflect"
"slices"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
@@ -347,9 +348,11 @@ func (ix *Indexer) FillPackages(ctx context.Context) error {
if err != nil {
continue
}
began := time.Now()
if err := ix.publish(ctx, repo, t.Merge, held, true); err != nil {
ix.logf("%s: its packages are not published at %s: %v", repo, t.Merge, err)
}
ix.logf("publishing the packages at start: %s at %.12s took %s", repo, t.Merge, time.Since(began).Round(time.Millisecond))
}
return nil
}