G starts: grants composes the first rights from the trunk, the indexer answers at once and knows its build, unpublished packages build at the source commit #217

Merged
mesh-tools merged 152 commits from fix/47-G-start-2 into main 2026-11-16 14:15:13 +00:00
4 changed files with 106 additions and 18 deletions
Showing only changes of commit dc4f1dff39 - Show all commits
@@ -602,11 +602,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
logger.Printf("publishing the packages at start: %v", err)
}
})
for _, le := range startEnds {
values := make([][]byte, len(startEnds))
for i, le := range startEnds {
ix.ApplyOutOfStepBuild(ctx, le.Value)
if err := ix.ApplyBuildEnd(ctx, le.Value); err != nil {
logger.Printf("%s entry %d: its imports are not kept: %v", builds.BuildLogRead, le.Sequence, err)
}
values[i] = le.Value
}
if err := ix.ApplyBuildEndsAtStart(ctx, values, func(i int, err error) {
logger.Printf("%s entry %d: its imports are not kept: %v", builds.BuildLogRead, startEnds[i].Sequence, err)
}); err != nil {
logger.Printf("keeping the view after the build ends at start: %v", err)
}
startEnds = nil
if err := ix.RecomputeAtStart(ctx); err != nil {
@@ -55,6 +55,8 @@ const writeTries = 8
// Indexer writes candidates.<group> from the forge bridge's messages.
type Indexer struct {
// keptView replaces KeepView for a test that counts the writes of the view; nil in the program.
keptView func(context.Context) error
// Module is the indexer's own module, the credential's: the observer of its findings and the key of its own entry (design 47 G A-296).
Module string
Store Store // candidates
@@ -6,6 +6,7 @@ import (
"maps"
"slices"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
@@ -27,50 +28,87 @@ import (
// Imports that cannot be resolved (a package the forge cannot list) are kept with no ImportForm: the next recompute that
// resolves them writes the import form, one rebuild too many, never a stale module.
func (ix *Indexer) ApplyBuildEnd(ctx context.Context, value []byte) error {
if err := ix.applyBuildEnd(ctx, value); err != nil {
return err
wrote, err := ix.applyBuildEnd(ctx, value)
if err != nil || !wrote {
return err // an entry that wrote nothing leaves the view as it is
}
return ix.KeepView(ctx) // the imports kept with the view (KeepView), so a restart reads them back
return ix.keepView(ctx) // the imports kept with the view (KeepView), so a restart reads them back
}
func (ix *Indexer) applyBuildEnd(ctx context.Context, value []byte) error {
// ApplyBuildEndsAtStart applies every entry of the build log read at start as ApplyBuildEnd does, and keeps the view
// once, after them, when any entry wrote a module: KeepView writes the whole catalog view, so once per entry is a start
// that never ends (measured: 50 min CPU per start). An entry that fails is passed to failed with its position.
func (ix *Indexer) ApplyBuildEndsAtStart(ctx context.Context, values [][]byte, failed func(i int, err error)) error {
began := time.Now()
ix.logf("replaying %d build ends of the build log at start", len(values))
wrote := 0
for i, v := range values {
w, err := ix.applyBuildEnd(ctx, v)
if err != nil {
if failed != nil {
failed(i, err)
}
continue
}
if w {
wrote++
}
}
var err error
if wrote > 0 {
err = ix.keepView(ctx)
}
ix.logf("replayed %d build ends at start: %d wrote a module, in %s", len(values), wrote, time.Since(began).Round(time.Millisecond))
return err
}
// keepView is KeepView; a test replaces it to count the writes of the view.
func (ix *Indexer) keepView(ctx context.Context) error {
if ix.keptView != nil {
return ix.keptView(ctx)
}
return ix.KeepView(ctx)
}
// applyBuildEnd reports whether it wrote the module (its final write changed the held value).
func (ix *Indexer) applyBuildEnd(ctx context.Context, value []byte) (bool, error) {
if !builds.IsBuildEntry(value) {
return nil
return false, nil
}
e, err := builds.DecodeBuildLogEntry(value)
if err != nil {
return err
return false, err
}
if e.Event != builds.BuildEndedEvent || e.Outcome != builds.BuildDone || len(e.Imports) == 0 || ix.Modules == nil || e.FP == "" {
return nil
return false, nil
}
key, err := meshcatalogclient.ModulesKey(e.Module)
if err != nil {
return nil
return false, nil
}
ix.recompute.Lock()
defer ix.recompute.Unlock()
raw, _, found, err := ix.Modules.Get(ctx, key)
if err != nil || !found {
return err
return false, err
}
cur, err := meshcatalogclient.DecodeModule(raw)
if err != nil || cur.Fp2 != e.FP || cur.Inputs2 == nil || cur.Manifest == "" {
return nil // a build of a fingerprint the module no longer holds says nothing of its inputs now
return false, nil // a build of a fingerprint the module no longer holds says nothing of its inputs now
}
if slices.Equal(cur.Inputs2.Imports, e.Imports) && cur.Inputs2.ImportForm != "" {
return nil
return false, nil
}
_, m, err := ix.manifestRaw(ctx, cur.Manifest)
if err != nil {
return fmt.Errorf("modules.%s: its manifest: %w", key, err)
return false, fmt.Errorf("modules.%s: its manifest: %w", key, err)
}
toolchains := map[string]meshcatalogclient.Toolchain2{}
for artifact, language := range languagesOf(m) {
used, ok := cur.Toolchains[language]
if !ok {
ix.logf("%s: its fingerprint names no %s toolchain, so the imports of its build are not kept", e.Module, language)
return nil
return false, nil
}
toolchains[artifact] = meshcatalogclient.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe}
}
@@ -87,7 +125,9 @@ func (ix *Indexer) applyBuildEnd(ctx context.Context, value []byte) error {
kept.ImportForm = fp
}
}
return write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
wrote := false
err = write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
wrote = false // a conflict computes again
if !found {
return nil, false, nil
}
@@ -99,8 +139,10 @@ func (ix *Indexer) applyBuildEnd(ctx context.Context, value []byte) error {
in.Imports, in.ImportForm = kept.Imports, kept.ImportForm
now.Inputs2 = &in
data, err := meshcatalogclient.EncodeModule(now)
wrote = err == nil && !unchanged(data, false, value, found)
return data, false, err
})
return wrote && err == nil, err
}
// publishedAsBuilt are the published Go modules holding the imports of build e of module cur, each as the build took
@@ -407,3 +407,43 @@ func anUnreadPackagesStateWritesNoFingerprintAndFingerprintsAreComputedAgain(t *
}
})
}
// A start replays every build end of the build log: an entry that writes no module keeps no view, and the entries that
// write keep it once, after all of them (measured: the view kept per entry cost 50 min CPU per start).
func TestBuildEndsKeepTheViewOnlyWhenTheyWrite(t *testing.T) {
ctx := context.Background()
ix, _ := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")})
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
if err := ix.Recompute(ctx); err != nil {
t.Fatal(err)
}
raw, _, _, _ := ix.Modules.Get(ctx, "avahi")
cur, err := meshcatalogclient.DecodeModule(raw)
if err != nil || cur.Fp2 == "" {
t.Fatalf("no fingerprint to start from: %v %q", err, cur.None2)
}
older := buildEnd(t, "avahi", "src2:"+strings.Repeat("e", 64), meshSDK)
writes := buildEnd(t, "avahi", cur.Fp2, meshSDK)
kept := 0
ix.keptView = func(context.Context) error { kept++; return nil }
for _, c := range []struct {
name string
entries [][]byte
want int
}{
{"no entry", nil, 0},
{"entries that write nothing", [][]byte{older, []byte("not a build entry"), older}, 0},
{"a write among entries that write nothing", [][]byte{older, writes, older}, 1},
{"the same write again keeps nothing", [][]byte{writes, older}, 0},
} {
kept = 0
if err := ix.ApplyBuildEndsAtStart(ctx, c.entries, nil); err != nil || kept != c.want {
t.Errorf("%s: the view was kept %d times (%v), want %d", c.name, kept, err, c.want)
}
}
// The live path, one entry: the view is kept when it wrote.
kept = 0
if err := ix.ApplyBuildEnd(ctx, older); err != nil || kept != 0 {
t.Errorf("a live entry that writes nothing kept the view %d times (%v)", kept, err)
}
}