From a5da6079de01f4ff5cf94a054128d4585221fd48 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 21:02:44 +0000 Subject: [PATCH] Write each module once per merge and watch a build module's toolchains bucket on its toolchain.reported event, with no timer (issue 1444, review round 1) --- .../mesh-delivery-indexer/integration_test.go | 27 +++- .../cmd/mesh-delivery-indexer/main.go | 66 +++----- .../cmd/mesh-delivery-indexer/toolchains.go | 115 ++++++++++++++ .../mesh-delivery-indexer/toolchains_test.go | 141 ++++++++++++++++++ .../internal/index/modules.go | 18 +-- .../internal/index/toolchains.go | 87 +++++++++-- .../internal/index/toolchains_test.go | 62 ++++++-- .../mesh-delivery-indexer/manifest_test.go | 2 +- modules/mesh-delivery-indexer/module.json | 2 + 9 files changed, 428 insertions(+), 92 deletions(-) create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains.go create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains_test.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index 3962c90..c6aadb7 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -137,6 +137,15 @@ func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue), b if _, err := js.CreateOrUpdateConsumer(ctx, "FORGE", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-indexer", AckPolicy: jetstream.AckExplicitPolicy}); err != nil { t.Fatal(err) } + // The build modules' streams for toolchain.reported (declared by them, heard by the indexer) and its durable consumers. + for module, stream := range map[string]string{"mesh-build-go": "TOOLCHAINS_GO", "mesh-build-node": "TOOLCHAINS_NODE"} { + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{"mesh.stream." + module + ".event.toolchain.reported"}}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-indexer", AckPolicy: jetstream.AckExplicitPolicy}); err != nil { + t.Fatal(err) + } + } candidates, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.CandidatesBucket, History: delivery.CandidatesHistory}) if err != nil { t.Fatal(err) @@ -271,7 +280,7 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4 // TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain: the bucket mesh-build-go_toolchains exists (the state // toolchains of the build module, delivery.ToolchainsBucket) and holds no key; a merged message of mesh-catalog carrying -// avahi's real manifest makes modules.avahi with none2 'toolchain not yet reported'; a put of the key n1.go (the build +// avahi's real manifest makes modules.avahi with none2 'toolchain not yet reported'; the bucket made after the indexer started, a put of the key n1.go and the build module's event toolchain.reported (the build // module's own layout, delivery.ToolchainKey and EncodeToolchain) makes, within 5 s and with no merge, none2 "builder's // fingerprint not yet known". func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *testing.T) { // 1444/AC10 @@ -289,10 +298,6 @@ func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *tes if _, err := builds.Put(ctx, delivery.NewestKey("avahi"), []byte("build-1")); err != nil { t.Fatal(err) } - }, func(ctx context.Context, js jetstream.JetStream) { - if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket("mesh-build-go"), History: 1}); err != nil { - t.Fatal(err) - } }) data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0", @@ -321,6 +326,12 @@ func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *tes if got := none2(delivery.ToolchainNoReport); got != delivery.ToolchainNoReport { t.Fatalf("none2 = %q within 5 s, want %q", got, delivery.ToolchainNoReport) } + // The bucket is made only now, as when the build module is assigned after the indexer started (review finding 9): the + // indexer finds it on its own. + kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket("mesh-build-go"), History: 1}) + if err != nil { + t.Fatal(err) + } key, err := delivery.ToolchainKey("n1", "go") if err != nil { t.Fatal(err) @@ -329,11 +340,11 @@ func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *tes if err != nil { t.Fatal(err) } - kv, err := js.KeyValue(ctx, delivery.ToolchainsBucket("mesh-build-go")) - if err != nil { + if _, err := kv.Put(ctx, key, value); err != nil { t.Fatal(err) } - if _, err := kv.Put(ctx, key, value); err != nil { + // The build module says it wrote the key (its event on its stream): the indexer then watches the bucket it did not know. + if _, err := js.Publish(ctx, "mesh.stream.mesh-build-go.event.toolchain.reported", []byte(`{"node":"n1","language":"go"}`)); err != nil { t.Fatal(err) } if got := none2("builder's fingerprint not yet known"); got != "builder's fingerprint not yet known" { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 4abb8e3..8a5502c 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -15,10 +15,8 @@ import ( "fmt" "io" "log" - "maps" "os" "os/signal" - "slices" "strings" "syscall" @@ -298,10 +296,27 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } go func() { sourceErrs <- follow(ctx, sw, each) }() } - toolErrs, err := watchToolchains(ctx, js, ix, logger) - if err != nil { + tools := newToolchainWatch(ctx, js, toolchainSink{put: ix.PutToolchain, apply: ix.ApplyToolchain, recompute: ix.Recompute}, logger) + if err := tools.start(); err != nil { return err } + // A build module says each time it wrote its toolchains key: its stream, and the indexer's durable consumer on it, are + // made by the controller as the forge's are. Where the stream is not there yet the indexer says so and is started again + // with the build module, as with the gate's source. + for _, module := range sortedModules() { + stream := index.ToolchainStreams[module] + tcons, err := js.Consumer(ctx, stream, cred.Node+"_"+cred.Module) + if err != nil { + logger.Printf("the consumer %s_%s on %s does not exist yet, so %s's reports are not heard until the indexer starts again: %v", cred.Node, cred.Module, stream, module, err) + continue + } + tc, err := tcons.Consume(func(m jetstream.Msg) { tools.Reported(m) }, + jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) })) + if err != nil { + return err + } + defer tc.Stop() + } cons, err := js.Consumer(ctx, forgeStream, cred.Node+"_"+cred.Module) if err != nil { return fmt.Errorf("the consumer %s_%s on %s: %w", cred.Node, cred.Module, forgeStream, err) @@ -319,48 +334,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err case err := <-sourceErrs: return err - case err := <-toolErrs: + case err := <-tools.errs: return err } } - -// watchToolchains reads the toolchains each build module reports (mesh-build-go_toolchains, mesh-build-node_toolchains: the -// module's state toolchains, delivery.ToolchainsBucket; novox/hq ADR 0465 clause 2, issue 1444) and follows them: every -// change of a language's agreed toolchain writes the modules of that language again (index.Indexer.Recompute). A bucket -// that does not exist yet (its build module is not delivered) is said, and not waited for (that needs a timer, ADR 0437 -// §8): the indexer is started again with the module, as with the gate's source. The returned channel gets the error of a -// watch that ended. -func watchToolchains(ctx context.Context, js jetstream.JetStream, ix *index.Indexer, logger *log.Logger) (<-chan error, error) { - errs := make(chan error, len(index.ToolchainModules)) - watched := 0 - for _, module := range slices.Sorted(maps.Values(index.ToolchainModules)) { - name := delivery.ToolchainsBucket(module) - kv, err := js.KeyValue(ctx, name) - if errors.Is(err, jetstream.ErrBucketNotFound) { - logger.Printf("the bucket %s does not exist yet: no toolchain of %s is read until the indexer starts again", name, module) - continue - } - if err != nil { - return nil, fmt.Errorf("the bucket %s: %w", name, err) - } - w, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { ix.PutToolchain(module, e.Key(), e.Value(), !opOf(e)) }) - if err != nil { - return nil, err - } - watched++ - go func() { - errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) { - if err := ix.ApplyToolchain(ctx, module, e.Key(), e.Value(), !opOf(e)); err != nil { - logger.Printf("%s.%s could not be taken into the modules: %v", name, e.Key(), err) - } - }) - }() - } - // What the buckets held when the indexer started moves the modules once, not once per key. - if watched > 0 { - if err := ix.Recompute(ctx); err != nil { - logger.Printf("the modules could not be written again from the toolchains: %v", err) - } - } - return errs, nil -} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains.go new file mode 100644 index 0000000..30892ae --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains.go @@ -0,0 +1,115 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log" + "maps" + "slices" + "strings" + "sync" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-indexer/internal/index" +) + +// toolchainSink is where the toolchains the build modules report go: the indexer's own methods (index.Indexer.PutToolchain, +// ApplyToolchain, Recompute). +type toolchainSink struct { + put func(module, key string, value []byte, deleted bool) bool + apply func(ctx context.Context, module, key string, value []byte, deleted bool) error + recompute func(ctx context.Context) error +} + +// bucketOpener is the part of the bus the toolchain watch uses. +type bucketOpener interface { + KeyValue(ctx context.Context, bucket string) (jetstream.KeyValue, error) +} + +// toolchainWatch reads the toolchains each build module reports (mesh-build-go_toolchains, mesh-build-node_toolchains: the +// module's state toolchains, delivery.ToolchainsBucket; novox/hq ADR 0465 clause 2, issue 1444) and follows them: every +// change of a language's agreed toolchain writes the modules of that language again (index.Indexer.Recompute). A bucket +// is made when its build module is assigned, after the indexer started, and the indexer holds no timer (ADR 0437 §8): +// it watches the buckets that exist at start, and a build module says each time it wrote its toolchains key +// (toolchain.reported, handled by Reported); a bucket not watched yet is then watched, settled as the others. +type toolchainWatch struct { + ctx context.Context + js bucketOpener + sink toolchainSink + logger *log.Logger + errs chan error // the error of a watch that ended + mu sync.Mutex + watched map[string]bool +} + +func newToolchainWatch(ctx context.Context, js bucketOpener, sink toolchainSink, logger *log.Logger) *toolchainWatch { + return &toolchainWatch{ctx: ctx, js: js, sink: sink, logger: logger, errs: make(chan error, len(index.ToolchainModules)), watched: map[string]bool{}} +} + +// start watches the buckets that exist now; a missing one is said and waits for its build module's event. +func (w *toolchainWatch) start() error { + for _, module := range sortedModules() { + if err := w.ensure(module); err != nil { + return err + } + } + return nil +} + +// ensure watches the module's toolchains bucket if it is not watched and exists (settling what it holds), then writes the +// modules again from what the watches hold. +func (w *toolchainWatch) ensure(module string) error { + w.mu.Lock() + defer w.mu.Unlock() + name := delivery.ToolchainsBucket(module) + if !w.watched[module] { + kv, err := w.js.KeyValue(w.ctx, name) + switch { + case errors.Is(err, jetstream.ErrBucketNotFound): + w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module) + return nil + case err != nil: + return fmt.Errorf("the bucket %s: %w", name, err) + } + watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.sink.put(module, e.Key(), e.Value(), !opOf(e)) }) + if err != nil { + return err + } + w.watched[module] = true + go func() { + w.errs <- follow(w.ctx, watcher, func(e jetstream.KeyValueEntry) { + if err := w.sink.apply(w.ctx, module, e.Key(), e.Value(), !opOf(e)); err != nil { + w.logger.Printf("%s.%s could not be taken into the modules: %v", name, e.Key(), err) + } + }) + }() + } + return w.sink.recompute(w.ctx) +} + +// Reported takes one toolchain.reported event of a build module (the message on its stream) in: the module is read off the +// subject, mesh.stream..event.toolchain.reported, its bucket is watched if it is not, and the modules are written +// again. A subject of no build module is terminated; a failure stays unacknowledged, and the server delivers it again. +func (w *toolchainWatch) Reported(m delivered) { + module, _, _ := strings.Cut(strings.TrimPrefix(m.Subject(), "mesh.stream."), ".event.") + known := false + for _, k := range index.ToolchainModules { + known = known || k == module + } + if !known || !strings.HasPrefix(m.Subject(), "mesh.stream.") { + w.logger.Printf("%s: not a build module's event, dropped", m.Subject()) + _ = m.Term() + return + } + if err := w.ensure(module); err != nil { + w.logger.Printf("%s: left for redelivery: %v", m.Subject(), err) + return + } + _ = m.Ack() +} + +func sortedModules() []string { return slices.Sorted(maps.Values(index.ToolchainModules)) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains_test.go new file mode 100644 index 0000000..e26bf76 --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains_test.go @@ -0,0 +1,141 @@ +package main + +import ( + "bytes" + "context" + "log" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// A build module's bucket is made when the module is assigned, after the indexer started (issue 1444, review finding 9). +// The fakes implement only what the watch calls: the bus's KeyValue by bucket name, and a bucket's WatchAll. + +type laterBuckets struct { + mu sync.Mutex + appears map[string]jetstream.KeyValue // buckets that exist now + asked map[string]int +} + +func (l *laterBuckets) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) { + l.mu.Lock() + defer l.mu.Unlock() + l.asked[name]++ + if kv, ok := l.appears[name]; ok { + return kv, nil + } + return nil, jetstream.ErrBucketNotFound +} + +func (l *laterBuckets) make(name string, kv jetstream.KeyValue) { + l.mu.Lock() + defer l.mu.Unlock() + l.appears[name] = kv +} + +type fakeBucket struct { + jetstream.KeyValue + held []jetstream.KeyValueEntry + w *fakeWatcher +} + +func (b *fakeBucket) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { + ch := make(chan jetstream.KeyValueEntry, 8) + for _, e := range b.held { + ch <- e + } + ch <- nil // the mark of the last value held + b.w = &fakeWatcher{ch: ch} + return b.w, nil +} + +type fakeWatcher struct { + jetstream.KeyWatcher + ch chan jetstream.KeyValueEntry +} + +func (w *fakeWatcher) Updates() <-chan jetstream.KeyValueEntry { return w.ch } +func (w *fakeWatcher) Stop() error { return nil } + +type fakeEntry struct { + jetstream.KeyValueEntry + key string + value []byte +} + +func (e fakeEntry) Key() string { return e.key } +func (e fakeEntry) Value() []byte { return e.value } +func (e fakeEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut } + +func TestABuildModulesBucketThatAppearsLaterIsWatchedWhenItsEventArrives(t *testing.T) { // 1444 (review finding 9) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "1", At: time.Now()}) + if err != nil { + t.Fatal(err) + } + buckets := &laterBuckets{appears: map[string]jetstream.KeyValue{}, asked: map[string]int{}} + var mu sync.Mutex + var puts, applies []string + recomputes := 0 + sink := toolchainSink{ + put: func(module, key string, _ []byte, _ bool) bool { + mu.Lock() + puts = append(puts, module+" "+key) + mu.Unlock() + return true + }, + apply: func(_ context.Context, module, key string, _ []byte, _ bool) error { + mu.Lock() + applies = append(applies, module+" "+key) + mu.Unlock() + return nil + }, + recompute: func(context.Context) error { mu.Lock(); recomputes++; mu.Unlock(); return nil }, + } + var logs bytes.Buffer + w := newToolchainWatch(ctx, buckets, sink, log.New(&logs, "", 0)) + if err := w.start(); err != nil { // neither bucket exists: nothing is watched, nothing is written, nobody waits + t.Fatal(err) + } + if len(puts) != 0 || recomputes != 0 || !strings.Contains(logs.String(), "does not exist yet") { + t.Fatalf("puts %v recomputes %d log %q after a start with no bucket", puts, recomputes, logs.String()) + } + // The bucket of mesh-build-go is made and already holds a key, and the module says it reported. + bucket := &fakeBucket{held: []jetstream.KeyValueEntry{fakeEntry{key: "novox.go", value: value}}} + buckets.make(delivery.ToolchainsBucket("mesh-build-go"), bucket) + msg := &fakeMsg{subject: "mesh.stream.mesh-build-go.event.toolchain.reported"} + w.Reported(msg) + if msg.acked != 1 || len(puts) != 1 || puts[0] != "mesh-build-go novox.go" || recomputes != 1 { + t.Fatalf("acked %d puts %v recomputes %d, want the settled key, one recompute and an ack", msg.acked, puts, recomputes) + } + // It is followed from now on: a key put later is applied, and a second report only recomputes. + bucket.w.ch <- fakeEntry{key: "ace.go", value: value} + for deadline := time.Now().Add(3 * time.Second); time.Now().Before(deadline); time.Sleep(5 * time.Millisecond) { + mu.Lock() + n := len(applies) + mu.Unlock() + if n == 1 { + break + } + } + if len(applies) != 1 || applies[0] != "mesh-build-go ace.go" { + t.Errorf("applies = %v, want the key put later", applies) + } + w.Reported(&fakeMsg{subject: "mesh.stream.mesh-build-go.event.toolchain.reported"}) + if len(puts) != 1 || recomputes != 2 || buckets.asked[delivery.ToolchainsBucket("mesh-build-go")] != 2 { + t.Errorf("puts %v recomputes %d asked %v, want the bucket opened again only at the first report", puts, recomputes, buckets.asked) + } + // An event of no build module is terminated. + other := &fakeMsg{subject: "mesh.stream.avahi.event.toolchain.reported"} + w.Reported(other) + if other.termd != 1 || other.acked != 0 { + t.Errorf("terminated %d acked %d for a module that is no build module", other.termd, other.acked) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 1538192..6869bd4 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -2,8 +2,6 @@ package index import ( "context" - "fmt" - "reflect" "sort" "strings" @@ -57,6 +55,7 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error { if err != nil { return err } + overlay := map[string]delivery.Module{} for _, k := range ix.Builds.Modules(f.Repository) { key, err := delivery.ModulesKey(k.Module) if err != nil { @@ -77,21 +76,10 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error { in := inputsOf(k.Module, raw, f.Trees[k.Path], f.Locks[k.Path], ix.baseFP(ctx)) next.Inputs2, next.None2 = in.Kept, in.None2 } - err = write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) { - if found { - if cur, err := delivery.DecodeModule(value); err == nil && reflect.DeepEqual(cur, withV(next)) { - return nil, false, nil - } - } - data, err := delivery.EncodeModule(next) - return data, false, err - }) - if err != nil { - return fmt.Errorf("modules.%s: %w", key, err) - } + overlay[key] = next } // What this merge moved reaches the modules standing on it (a base's tree, build-agent's fingerprint), in the graph's order. - return ix.Recompute(ctx) + return ix.recomputeWith(ctx, overlay) } // baseFP answers the new-form fingerprint of a base module as modules. holds it, or why it has none. diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index fac1331..fd5ce2c 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "maps" + "reflect" "slices" "strings" @@ -30,6 +31,11 @@ const ( // indexer reads: the language's agreed toolchain is the one every node's entry in it agrees on. var ToolchainModules = map[string]string{"go": "mesh-build-go", "typescript": "mesh-build-node"} +// ToolchainStreams are the stream each build module carries its toolchain.reported event on, by module, as the bus names it +// (a stream's local name in capitals, catalogue.StreamBusName): `toolchains-go` and `toolchains-node`, declared by the +// build modules (streams, with the event toolchain.reported) and heard by this module (consumes). +var ToolchainStreams = map[string]string{"mesh-build-go": "TOOLCHAINS_GO", "mesh-build-node": "TOOLCHAINS_NODE"} + // PutToolchain takes one change of a build module's toolchains bucket in, without writing any module: a put of a node's // toolchain of a language (key ., delivery.ToolchainKey), or its delete. It answers whether what the // language's toolchain is of (version and recipe) changed. A key, or a value, that is not a toolchain of the module's @@ -120,31 +126,46 @@ type known struct { // the order the bucket lists them in; modules that stand on each other have a cycle as their reason. A module is written // only when its fingerprint or its reason changes, and only if its manifest is still the one computed from. A module with // no kept inputs (a message that carried no manifest) is left as it is. -func (ix *Indexer) Recompute(ctx context.Context) error { - if ix.Modules == nil || ix.Manifests == nil { +func (ix *Indexer) Recompute(ctx context.Context) error { return ix.recomputeWith(ctx, nil) } + +// recomputeWith is Recompute for a merge as well: the records the merge gives (overlay, by module key) stand in place +// of what the bucket holds, are computed with the others, and are written once, with their final fingerprint or reason +// (issue 1444: a merge writes each module once). A record of the overlay that has no kept inputs is written as it is. +func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]delivery.Module) error { + if ix.Modules == nil { return nil } + if ix.Manifests == nil { + return ix.writeOverlay(ctx, overlay, nil) + } ix.recompute.Lock() defer ix.recompute.Unlock() keys, err := ix.Modules.Keys(ctx) if err != nil { return fmt.Errorf("listing the modules: %w", err) } + for key := range overlay { + if !slices.Contains(keys, key) { + keys = append(keys, key) + } + } slices.Sort(keys) held := map[string]delivery.Module{} all := map[string]*known{} for _, key := range keys { - value, _, found, err := ix.Modules.Get(ctx, key) - if err != nil { - return fmt.Errorf("modules.%s: %w", key, err) - } - if !found { - continue - } - cur, err := delivery.DecodeModule(value) - if err != nil { - ix.logf("modules.%s holds what is not a module, left: %v", key, err) - continue + cur, ok := overlay[key] + if !ok { + value, _, found, err := ix.Modules.Get(ctx, key) + if err != nil { + return fmt.Errorf("modules.%s: %w", key, err) + } + if !found { + continue + } + if cur, err = delivery.DecodeModule(value); err != nil { + ix.logf("modules.%s holds what is not a module, left: %v", key, err) + continue + } } held[key] = cur if cur.Manifest == "" || cur.Inputs2 == nil { @@ -242,9 +263,25 @@ func (ix *Indexer) Recompute(ctx context.Context) error { k.cur.Inputs2 = &kept return r } + finals := map[string]delivery.Module{} for _, key := range slices.Sorted(maps.Keys(all)) { k := all[key] r := of(key, nil) + if _, merged := overlay[key]; merged { + final := overlay[key] + final.FP2, final.None2, final.Inputs2 = r.fp, r.none, k.cur.Inputs2 + // What the bases and the builder gave is kept as it was while the value it gave does not change, so a merge + // that moves nothing for the module writes nothing (as Recompute does). + if value, _, found, err := ix.Modules.Get(ctx, key); err == nil && found { + if old, err := delivery.DecodeModule(value); err == nil && old.Inputs2 != nil && old.FP2 == r.fp && old.None2 == r.none { + in := *final.Inputs2 + in.Bases, in.Builder = old.Inputs2.Bases, old.Inputs2.Builder + final.Inputs2 = &in + } + } + finals[key] = final + continue + } if k.cur.FP2 == r.fp && k.cur.None2 == r.none { continue } @@ -269,6 +306,30 @@ func (ix *Indexer) Recompute(ctx context.Context) error { return fmt.Errorf("modules.%s: %w", key, err) } } + return ix.writeOverlay(ctx, overlay, finals) +} + +// writeOverlay writes the records of a merge, each once and only if it differs from what the bucket holds; finals has +// the ones that were computed, the others are written as the merge gave them. +func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]delivery.Module) error { + for _, key := range slices.Sorted(maps.Keys(overlay)) { + next := overlay[key] + if f, ok := finals[key]; ok { + next = f + } + err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) { + if found { + if cur, err := delivery.DecodeModule(value); err == nil && reflect.DeepEqual(cur, withV(next)) { + return nil, false, nil + } + } + data, err := delivery.EncodeModule(next) + return data, false, err + }) + if err != nil { + return fmt.Errorf("modules.%s: %w", key, err) + } + } return nil } diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 32de130..d9fd7f4 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -42,6 +42,20 @@ type mod struct { // merged applies one merge of the catalogue carrying these modules (every one a Go bundle's go.sum, a TypeScript bundle's // package-lock.json) and returns the indexer. func newFormIndexer(t *testing.T, mods ...mod) (*Indexer, *memStore) { + t.Helper() + ix, store, f := newFormIndexerUnmerged(t, mods...) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + // build-agent's value of today's form, from a merge of the controller (modules.build-agent holds its fingerprint). + if err := ix.Apply(context.Background(), mergedMsg(buildAgentRepo, map[string]string{"modules/build-agent": "agent-tree-1"})); err != nil { + t.Fatal(err) + } + return ix, store +} + +// newFormIndexerUnmerged is the indexer and the catalogue merge of these modules, not yet applied. +func newFormIndexerUnmerged(t *testing.T, mods ...mod) (*Indexer, *memStore, delivery.Forge) { t.Helper() var recs [][2]any trees, manifests := map[string]string{}, map[string]string{} @@ -54,15 +68,7 @@ func newFormIndexer(t *testing.T, mods ...mod) (*Indexer, *memStore) { } recs = append(recs, rec(t, "build-agent-1", "build-agent", buildAgentRepo, "modules/build-agent", "main", "", "2026-10-19T09:00:00Z", aInputs)) ix, store, _ := manifestsIndexer(t, recs...) - f := withManifests(mergedMsg(catalogueRepo, trees), manifests, locks) - if err := ix.Apply(context.Background(), f); err != nil { - t.Fatal(err) - } - // build-agent's value of today's form, from a merge of the controller (modules.build-agent holds its fingerprint). - if err := ix.Apply(context.Background(), mergedMsg(buildAgentRepo, map[string]string{"modules/build-agent": "agent-tree-1"})); err != nil { - t.Fatal(err) - } - return ix, store + return ix, store, withManifests(mergedMsg(catalogueRepo, trees), manifests, locks) } // toolchainValue is a toolchains value as the build module's writer encodes it (mesh-sdk go/delivery EncodeToolchain). @@ -194,3 +200,41 @@ func TestModulesThatStandOnEachOtherAreNamedAsACycle(t *testing.T) { // 1444 (th } } } + +func TestAMergeWritesEachModuleOnceWithItsFinalValueAndAReplayWritesNothing(t *testing.T) { // 1444 (review finding 8) + ctx := context.Background() + ix, mods, f := newFormIndexerUnmerged(t, mod{"avahi", catalogueManifest(t, "avahi")}) + reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") // agreed before the merge is heard + if err := ix.Apply(ctx, f); err != nil { + t.Fatal(err) + } + m, _ := moduleOf(t, mods, "avahi") + if mods.writes != 1 || m.None2 != None2Builder { + t.Errorf("writes = %d none2 = %q after one merge, want one write with %q", mods.writes, m.None2, None2Builder) + } + if err := ix.Apply(ctx, f); err != nil { + t.Fatal(err) + } + if mods.writes != 1 { + t.Errorf("writes = %d after the same merge again, want still 1", mods.writes) + } +} + +func TestAMergeKeepsAFingerprintTheModuleStillHas(t *testing.T) { // 1444 (review finding 8: a merge never blanks FP2) + ctx := context.Background() + ix, mods := newFormIndexer(t, builderModules(t)...) + reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") + before, _ := moduleOf(t, mods, "mesh-tools") + if before.FP2 == "" { + t.Fatalf("mesh-tools has no fp2: %+v", before) + } + w := mods.writes + // the same catalogue merged again at a later commit: nothing moved + _, _, f := newFormIndexerUnmerged(t, builderModules(t)...) + if err := ix.Apply(ctx, f); err != nil { + t.Fatal(err) + } + if after, _ := moduleOf(t, mods, "mesh-tools"); after.FP2 != before.FP2 || mods.writes != w { + t.Errorf("fp2 = %q (was %q), writes %d more", after.FP2, before.FP2, mods.writes-w) + } +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 24a52cc..90d96c3 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -84,7 +84,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build } // "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control // internal/broker/state.go:62-93). - if want := []string{"mesh-delivery-forge-bridge.forge.**"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-delivery-forge-bridge.forge.**"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 3492dc2..fcfe983 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -3,6 +3,8 @@ "version": "1", "slug": "indexer", "consumes": [ + "mesh-build-go.toolchain.reported", + "mesh-build-node.toolchain.reported", "mesh-delivery-forge-bridge.forge.**" ], "state": [