Indexer: the start replay keeps the view once, and only when a build end wrote
lead answer A-391: the start replay keeps the view once (measured: 50 min CPU per start). applyBuildEnd reports whether it wrote; ApplyBuildEnd keeps the view only then; ApplyBuildEndsAtStart applies every entry and keeps the view once if any wrote. Builds: design 47 G lead answer A-391 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
This commit is contained in:
@@ -602,11 +602,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
logger.Printf("publishing the packages at start: %v", err)
|
||||
}
|
||||
})
|
||||
for _, le := range startEnds {
|
||||
values := make([][]byte, len(startEnds))
|
||||
for i, le := range startEnds {
|
||||
ix.ApplyOutOfStepBuild(ctx, le.Value)
|
||||
if err := ix.ApplyBuildEnd(ctx, le.Value); err != nil {
|
||||
logger.Printf("%s entry %d: its imports are not kept: %v", builds.BuildLogRead, le.Sequence, err)
|
||||
}
|
||||
values[i] = le.Value
|
||||
}
|
||||
if err := ix.ApplyBuildEndsAtStart(ctx, values, func(i int, err error) {
|
||||
logger.Printf("%s entry %d: its imports are not kept: %v", builds.BuildLogRead, startEnds[i].Sequence, err)
|
||||
}); err != nil {
|
||||
logger.Printf("keeping the view after the build ends at start: %v", err)
|
||||
}
|
||||
startEnds = nil
|
||||
if err := ix.RecomputeAtStart(ctx); err != nil {
|
||||
|
||||
@@ -55,6 +55,8 @@ const writeTries = 8
|
||||
|
||||
// Indexer writes candidates.<group> from the forge bridge's messages.
|
||||
type Indexer struct {
|
||||
// keptView replaces KeepView for a test that counts the writes of the view; nil in the program.
|
||||
keptView func(context.Context) error
|
||||
// Module is the indexer's own module, the credential's: the observer of its findings and the key of its own entry (design 47 G A-296).
|
||||
Module string
|
||||
Store Store // candidates
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
@@ -27,50 +28,87 @@ import (
|
||||
// Imports that cannot be resolved (a package the forge cannot list) are kept with no ImportForm: the next recompute that
|
||||
// resolves them writes the import form, one rebuild too many, never a stale module.
|
||||
func (ix *Indexer) ApplyBuildEnd(ctx context.Context, value []byte) error {
|
||||
if err := ix.applyBuildEnd(ctx, value); err != nil {
|
||||
return err
|
||||
wrote, err := ix.applyBuildEnd(ctx, value)
|
||||
if err != nil || !wrote {
|
||||
return err // an entry that wrote nothing leaves the view as it is
|
||||
}
|
||||
return ix.KeepView(ctx) // the imports kept with the view (KeepView), so a restart reads them back
|
||||
return ix.keepView(ctx) // the imports kept with the view (KeepView), so a restart reads them back
|
||||
}
|
||||
|
||||
func (ix *Indexer) applyBuildEnd(ctx context.Context, value []byte) error {
|
||||
// ApplyBuildEndsAtStart applies every entry of the build log read at start as ApplyBuildEnd does, and keeps the view
|
||||
// once, after them, when any entry wrote a module: KeepView writes the whole catalog view, so once per entry is a start
|
||||
// that never ends (measured: 50 min CPU per start). An entry that fails is passed to failed with its position.
|
||||
func (ix *Indexer) ApplyBuildEndsAtStart(ctx context.Context, values [][]byte, failed func(i int, err error)) error {
|
||||
began := time.Now()
|
||||
ix.logf("replaying %d build ends of the build log at start", len(values))
|
||||
wrote := 0
|
||||
for i, v := range values {
|
||||
w, err := ix.applyBuildEnd(ctx, v)
|
||||
if err != nil {
|
||||
if failed != nil {
|
||||
failed(i, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if w {
|
||||
wrote++
|
||||
}
|
||||
}
|
||||
var err error
|
||||
if wrote > 0 {
|
||||
err = ix.keepView(ctx)
|
||||
}
|
||||
ix.logf("replayed %d build ends at start: %d wrote a module, in %s", len(values), wrote, time.Since(began).Round(time.Millisecond))
|
||||
return err
|
||||
}
|
||||
|
||||
// keepView is KeepView; a test replaces it to count the writes of the view.
|
||||
func (ix *Indexer) keepView(ctx context.Context) error {
|
||||
if ix.keptView != nil {
|
||||
return ix.keptView(ctx)
|
||||
}
|
||||
return ix.KeepView(ctx)
|
||||
}
|
||||
|
||||
// applyBuildEnd reports whether it wrote the module (its final write changed the held value).
|
||||
func (ix *Indexer) applyBuildEnd(ctx context.Context, value []byte) (bool, error) {
|
||||
if !builds.IsBuildEntry(value) {
|
||||
return nil
|
||||
return false, nil
|
||||
}
|
||||
e, err := builds.DecodeBuildLogEntry(value)
|
||||
if err != nil {
|
||||
return err
|
||||
return false, err
|
||||
}
|
||||
if e.Event != builds.BuildEndedEvent || e.Outcome != builds.BuildDone || len(e.Imports) == 0 || ix.Modules == nil || e.FP == "" {
|
||||
return nil
|
||||
return false, nil
|
||||
}
|
||||
key, err := meshcatalogclient.ModulesKey(e.Module)
|
||||
if err != nil {
|
||||
return nil
|
||||
return false, nil
|
||||
}
|
||||
ix.recompute.Lock()
|
||||
defer ix.recompute.Unlock()
|
||||
raw, _, found, err := ix.Modules.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
return err
|
||||
return false, err
|
||||
}
|
||||
cur, err := meshcatalogclient.DecodeModule(raw)
|
||||
if err != nil || cur.Fp2 != e.FP || cur.Inputs2 == nil || cur.Manifest == "" {
|
||||
return nil // a build of a fingerprint the module no longer holds says nothing of its inputs now
|
||||
return false, nil // a build of a fingerprint the module no longer holds says nothing of its inputs now
|
||||
}
|
||||
if slices.Equal(cur.Inputs2.Imports, e.Imports) && cur.Inputs2.ImportForm != "" {
|
||||
return nil
|
||||
return false, nil
|
||||
}
|
||||
_, m, err := ix.manifestRaw(ctx, cur.Manifest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("modules.%s: its manifest: %w", key, err)
|
||||
return false, fmt.Errorf("modules.%s: its manifest: %w", key, err)
|
||||
}
|
||||
toolchains := map[string]meshcatalogclient.Toolchain2{}
|
||||
for artifact, language := range languagesOf(m) {
|
||||
used, ok := cur.Toolchains[language]
|
||||
if !ok {
|
||||
ix.logf("%s: its fingerprint names no %s toolchain, so the imports of its build are not kept", e.Module, language)
|
||||
return nil
|
||||
return false, nil
|
||||
}
|
||||
toolchains[artifact] = meshcatalogclient.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe}
|
||||
}
|
||||
@@ -87,7 +125,9 @@ func (ix *Indexer) applyBuildEnd(ctx context.Context, value []byte) error {
|
||||
kept.ImportForm = fp
|
||||
}
|
||||
}
|
||||
return write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
wrote := false
|
||||
err = write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
wrote = false // a conflict computes again
|
||||
if !found {
|
||||
return nil, false, nil
|
||||
}
|
||||
@@ -99,8 +139,10 @@ func (ix *Indexer) applyBuildEnd(ctx context.Context, value []byte) error {
|
||||
in.Imports, in.ImportForm = kept.Imports, kept.ImportForm
|
||||
now.Inputs2 = &in
|
||||
data, err := meshcatalogclient.EncodeModule(now)
|
||||
wrote = err == nil && !unchanged(data, false, value, found)
|
||||
return data, false, err
|
||||
})
|
||||
return wrote && err == nil, err
|
||||
}
|
||||
|
||||
// publishedAsBuilt are the published Go modules holding the imports of build e of module cur, each as the build took
|
||||
|
||||
@@ -407,3 +407,43 @@ func anUnreadPackagesStateWritesNoFingerprintAndFingerprintsAreComputedAgain(t *
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// A start replays every build end of the build log: an entry that writes no module keeps no view, and the entries that
|
||||
// write keep it once, after all of them (measured: the view kept per entry cost 50 min CPU per start).
|
||||
func TestBuildEndsKeepTheViewOnlyWhenTheyWrite(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix, _ := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")})
|
||||
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _, _, _ := ix.Modules.Get(ctx, "avahi")
|
||||
cur, err := meshcatalogclient.DecodeModule(raw)
|
||||
if err != nil || cur.Fp2 == "" {
|
||||
t.Fatalf("no fingerprint to start from: %v %q", err, cur.None2)
|
||||
}
|
||||
older := buildEnd(t, "avahi", "src2:"+strings.Repeat("e", 64), meshSDK)
|
||||
writes := buildEnd(t, "avahi", cur.Fp2, meshSDK)
|
||||
kept := 0
|
||||
ix.keptView = func(context.Context) error { kept++; return nil }
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
entries [][]byte
|
||||
want int
|
||||
}{
|
||||
{"no entry", nil, 0},
|
||||
{"entries that write nothing", [][]byte{older, []byte("not a build entry"), older}, 0},
|
||||
{"a write among entries that write nothing", [][]byte{older, writes, older}, 1},
|
||||
{"the same write again keeps nothing", [][]byte{writes, older}, 0},
|
||||
} {
|
||||
kept = 0
|
||||
if err := ix.ApplyBuildEndsAtStart(ctx, c.entries, nil); err != nil || kept != c.want {
|
||||
t.Errorf("%s: the view was kept %d times (%v), want %d", c.name, kept, err, c.want)
|
||||
}
|
||||
}
|
||||
// The live path, one entry: the view is kept when it wrote.
|
||||
kept = 0
|
||||
if err := ix.ApplyBuildEnd(ctx, older); err != nil || kept != 0 {
|
||||
t.Errorf("a live entry that writes nothing kept the view %d times (%v)", kept, err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user