diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index f1aa923e..b0e113b1 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -602,11 +602,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro logger.Printf("publishing the packages at start: %v", err) } }) - for _, le := range startEnds { + values := make([][]byte, len(startEnds)) + for i, le := range startEnds { ix.ApplyOutOfStepBuild(ctx, le.Value) - if err := ix.ApplyBuildEnd(ctx, le.Value); err != nil { - logger.Printf("%s entry %d: its imports are not kept: %v", builds.BuildLogRead, le.Sequence, err) - } + values[i] = le.Value + } + if err := ix.ApplyBuildEndsAtStart(ctx, values, func(i int, err error) { + logger.Printf("%s entry %d: its imports are not kept: %v", builds.BuildLogRead, startEnds[i].Sequence, err) + }); err != nil { + logger.Printf("keeping the view after the build ends at start: %v", err) } startEnds = nil if err := ix.RecomputeAtStart(ctx); err != nil { diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index defb93bc..784258f4 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -55,6 +55,8 @@ const writeTries = 8 // Indexer writes candidates. from the forge bridge's messages. type Indexer struct { + // keptView replaces KeepView for a test that counts the writes of the view; nil in the program. + keptView func(context.Context) error // Module is the indexer's own module, the credential's: the observer of its findings and the key of its own entry (design 47 G A-296). Module string Store Store // candidates diff --git a/modules/mesh-delivery-indexer/internal/index/imports.go b/modules/mesh-delivery-indexer/internal/index/imports.go index f59e9324..05cdf7d1 100644 --- a/modules/mesh-delivery-indexer/internal/index/imports.go +++ b/modules/mesh-delivery-indexer/internal/index/imports.go @@ -6,6 +6,7 @@ import ( "maps" "slices" "strings" + "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" @@ -27,50 +28,87 @@ import ( // Imports that cannot be resolved (a package the forge cannot list) are kept with no ImportForm: the next recompute that // resolves them writes the import form, one rebuild too many, never a stale module. func (ix *Indexer) ApplyBuildEnd(ctx context.Context, value []byte) error { - if err := ix.applyBuildEnd(ctx, value); err != nil { - return err + wrote, err := ix.applyBuildEnd(ctx, value) + if err != nil || !wrote { + return err // an entry that wrote nothing leaves the view as it is } - return ix.KeepView(ctx) // the imports kept with the view (KeepView), so a restart reads them back + return ix.keepView(ctx) // the imports kept with the view (KeepView), so a restart reads them back } -func (ix *Indexer) applyBuildEnd(ctx context.Context, value []byte) error { +// ApplyBuildEndsAtStart applies every entry of the build log read at start as ApplyBuildEnd does, and keeps the view +// once, after them, when any entry wrote a module: KeepView writes the whole catalog view, so once per entry is a start +// that never ends (measured: 50 min CPU per start). An entry that fails is passed to failed with its position. +func (ix *Indexer) ApplyBuildEndsAtStart(ctx context.Context, values [][]byte, failed func(i int, err error)) error { + began := time.Now() + ix.logf("replaying %d build ends of the build log at start", len(values)) + wrote := 0 + for i, v := range values { + w, err := ix.applyBuildEnd(ctx, v) + if err != nil { + if failed != nil { + failed(i, err) + } + continue + } + if w { + wrote++ + } + } + var err error + if wrote > 0 { + err = ix.keepView(ctx) + } + ix.logf("replayed %d build ends at start: %d wrote a module, in %s", len(values), wrote, time.Since(began).Round(time.Millisecond)) + return err +} + +// keepView is KeepView; a test replaces it to count the writes of the view. +func (ix *Indexer) keepView(ctx context.Context) error { + if ix.keptView != nil { + return ix.keptView(ctx) + } + return ix.KeepView(ctx) +} + +// applyBuildEnd reports whether it wrote the module (its final write changed the held value). +func (ix *Indexer) applyBuildEnd(ctx context.Context, value []byte) (bool, error) { if !builds.IsBuildEntry(value) { - return nil + return false, nil } e, err := builds.DecodeBuildLogEntry(value) if err != nil { - return err + return false, err } if e.Event != builds.BuildEndedEvent || e.Outcome != builds.BuildDone || len(e.Imports) == 0 || ix.Modules == nil || e.FP == "" { - return nil + return false, nil } key, err := meshcatalogclient.ModulesKey(e.Module) if err != nil { - return nil + return false, nil } ix.recompute.Lock() defer ix.recompute.Unlock() raw, _, found, err := ix.Modules.Get(ctx, key) if err != nil || !found { - return err + return false, err } cur, err := meshcatalogclient.DecodeModule(raw) if err != nil || cur.Fp2 != e.FP || cur.Inputs2 == nil || cur.Manifest == "" { - return nil // a build of a fingerprint the module no longer holds says nothing of its inputs now + return false, nil // a build of a fingerprint the module no longer holds says nothing of its inputs now } if slices.Equal(cur.Inputs2.Imports, e.Imports) && cur.Inputs2.ImportForm != "" { - return nil + return false, nil } _, m, err := ix.manifestRaw(ctx, cur.Manifest) if err != nil { - return fmt.Errorf("modules.%s: its manifest: %w", key, err) + return false, fmt.Errorf("modules.%s: its manifest: %w", key, err) } toolchains := map[string]meshcatalogclient.Toolchain2{} for artifact, language := range languagesOf(m) { used, ok := cur.Toolchains[language] if !ok { ix.logf("%s: its fingerprint names no %s toolchain, so the imports of its build are not kept", e.Module, language) - return nil + return false, nil } toolchains[artifact] = meshcatalogclient.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe} } @@ -87,7 +125,9 @@ func (ix *Indexer) applyBuildEnd(ctx context.Context, value []byte) error { kept.ImportForm = fp } } - return write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) { + wrote := false + err = write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) { + wrote = false // a conflict computes again if !found { return nil, false, nil } @@ -99,8 +139,10 @@ func (ix *Indexer) applyBuildEnd(ctx context.Context, value []byte) error { in.Imports, in.ImportForm = kept.Imports, kept.ImportForm now.Inputs2 = &in data, err := meshcatalogclient.EncodeModule(now) + wrote = err == nil && !unchanged(data, false, value, found) return data, false, err }) + return wrote && err == nil, err } // publishedAsBuilt are the published Go modules holding the imports of build e of module cur, each as the build took diff --git a/modules/mesh-delivery-indexer/internal/index/imports_test.go b/modules/mesh-delivery-indexer/internal/index/imports_test.go index b4fdb202..3f0c3c82 100644 --- a/modules/mesh-delivery-indexer/internal/index/imports_test.go +++ b/modules/mesh-delivery-indexer/internal/index/imports_test.go @@ -407,3 +407,43 @@ func anUnreadPackagesStateWritesNoFingerprintAndFingerprintsAreComputedAgain(t * } }) } + +// A start replays every build end of the build log: an entry that writes no module keeps no view, and the entries that +// write keep it once, after all of them (measured: the view kept per entry cost 50 min CPU per start). +func TestBuildEndsKeepTheViewOnlyWhenTheyWrite(t *testing.T) { + ctx := context.Background() + ix, _ := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")}) + reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") + if err := ix.Recompute(ctx); err != nil { + t.Fatal(err) + } + raw, _, _, _ := ix.Modules.Get(ctx, "avahi") + cur, err := meshcatalogclient.DecodeModule(raw) + if err != nil || cur.Fp2 == "" { + t.Fatalf("no fingerprint to start from: %v %q", err, cur.None2) + } + older := buildEnd(t, "avahi", "src2:"+strings.Repeat("e", 64), meshSDK) + writes := buildEnd(t, "avahi", cur.Fp2, meshSDK) + kept := 0 + ix.keptView = func(context.Context) error { kept++; return nil } + for _, c := range []struct { + name string + entries [][]byte + want int + }{ + {"no entry", nil, 0}, + {"entries that write nothing", [][]byte{older, []byte("not a build entry"), older}, 0}, + {"a write among entries that write nothing", [][]byte{older, writes, older}, 1}, + {"the same write again keeps nothing", [][]byte{writes, older}, 0}, + } { + kept = 0 + if err := ix.ApplyBuildEndsAtStart(ctx, c.entries, nil); err != nil || kept != c.want { + t.Errorf("%s: the view was kept %d times (%v), want %d", c.name, kept, err, c.want) + } + } + // The live path, one entry: the view is kept when it wrote. + kept = 0 + if err := ix.ApplyBuildEnd(ctx, older); err != nil || kept != 0 { + t.Errorf("a live entry that writes nothing kept the view %d times (%v)", kept, err) + } +}