diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go index 52b5aea2..d658ea70 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go @@ -38,6 +38,18 @@ func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.So } return out, nil }, + Module: func(name string) (meshcatalog.ModulesByModuleValue, bool, error) { + key, err := meshcatalogclient.ModulesKey(name) + if err != nil { + return meshcatalog.ModulesByModuleValue{}, false, nil // a name no key holds is a module the trunk holds none of + } + e, ok, err := modules.Get(key) + if err != nil || !ok { + return meshcatalog.ModulesByModuleValue{}, false, err + } + m, err := meshcatalogclient.DecodeModule(e.Value) + return m, err == nil, nil + }, Manifest: func(digest string) (string, error) { key, err := meshcatalogclient.ManifestsKey(digest) if err != nil { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 664447b7..16cce0a8 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -17,9 +17,11 @@ import ( "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/busstate" "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-indexer/internal/index" ) @@ -286,9 +288,19 @@ func main() { tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts) // The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go. revisions := bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)} - trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: &index.KeptTrunk{Say: func(format string, a ...any) { + // The view of the trunk is the half the bundle's follower of the seat holds (design 47 G A-487): it reads the newest + // revision once when the runtime starts and again on the seat's event catalog-revision, like every part's follower. + say := func(format string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...) - }}} + } + follow := &views.Reader{Module: os.Getenv(stdio.ServedAs), Logf: say, Acts: acts} + half, err := meshcatalogclient.WireCatalog(follow, stdio.Ask, index.TrunkBranch, nil) + if err != nil { + fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err) + os.Exit(1) + } + follow.Start(context.Background()) + trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: &index.KeptTrunk{View: func() (meshcatalogclient.Half, bool) { return half.View() }, Say: say}} tools = append(tools, catalogTools(trunk.State)...) // The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly // the parts whose entry declares it (novox/hq ADR 0663). diff --git a/modules/mesh-delivery-indexer/internal/index/catalog.go b/modules/mesh-delivery-indexer/internal/index/catalog.go index bca1a826..a6c4a5d0 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog.go @@ -25,6 +25,9 @@ import ( type CatalogRead struct { // Modules is every value of the modules state, by module. Modules func() (map[string]meshcatalog.ModulesByModuleValue, error) + // Module is the one value of the modules state of that module, and whether the state holds it: the verb module reads + // the one module it is asked, never every module (design 47 G A-487). + Module func(name string) (meshcatalog.ModulesByModuleValue, bool, error) // Manifest is the manifest kept under a digest in the manifests state, as its JSON. Manifest func(digest string) (string, error) // Graph is the module graph, by module. @@ -51,17 +54,26 @@ func (c CatalogRead) CatalogModulesAnswer() (catalogue.CatalogModules, error) { return catalogue.CatalogModules{Modules: out}, nil } +// module is one module's value: through Module when set, else out of every module's. +func (c CatalogRead) module(name string) (meshcatalog.ModulesByModuleValue, bool, error) { + if c.Module != nil { + return c.Module(name) + } + mods, err := c.Modules() + m, ok := mods[name] + return m, ok, err +} + // CatalogModuleAnswer is the verb module: one module whole, its manifest included. func (c CatalogRead) CatalogModuleAnswer(name string) (catalogue.CatalogOne, error) { name = strings.TrimSpace(name) if name == "" { return catalogue.CatalogOne{}, fmt.Errorf("name the module") } - mods, err := c.Modules() + m, ok, err := c.module(name) if err != nil { return catalogue.CatalogOne{}, fmt.Errorf("the indexer's modules cannot be read: %w", err) } - m, ok := mods[name] if !ok { return catalogue.CatalogOne{}, fmt.Errorf("the trunk holds no module %s", name) } diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index ca51d308..891486a8 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -438,15 +438,21 @@ func TrunkView(ctx context.Context, o Objects) (modules, nodes Store, at int64, // BranchView is TrunkView of a watched branch: the view read from the newest revision the bucket keeps of that branch // (design 47 G §291, A-93), empty and at 0 when it keeps none. func BranchView(ctx context.Context, o Objects, branch string) (modules, nodes Store, at int64, err error) { - view, held := newOverlay(nil), newOverlay(nil) newest, any, err := newestOf(ctx, o, branch) if err != nil || !any { - return view, held, 0, err + return newOverlay(nil), newOverlay(nil), 0, err } _, entries, err := ReadRevision(ctx, o, RevisionObject(newest)) if err != nil { return nil, nil, 0, err } + return viewOfEntries(newest.Revision, entries) +} + +// viewOfEntries is the view of the entries of one trunk revision, as BranchView reads them. +func viewOfEntries(revision int64, entries []meshcatalog.CatalogEntry) (modules, nodes Store, at int64, err error) { + view, held := newOverlay(nil), newOverlay(nil) + ctx, newest := context.Background(), meshcatalog.CatalogRevision{Revision: revision} for _, e := range entries { key, err := meshcatalogclient.ModulesKey(e.Module) if err != nil { @@ -496,10 +502,16 @@ type TrunkStates struct { Kept *KeptTrunk } -// KeptTrunk is the view of the trunk as last read, at its revision, shared by every read of a TrunkStates; Say, when set, -// says each read of a revision and how long it took. +// KeptTrunk is the view of the newest trunk revision, held in memory (design 47 G A-487): it is made from the half of the +// catalog the bundle's follower of the seat holds (meshcatalogclient's Catalog, which reads the newest revision once when +// its runtime starts and again only when it hears the seat's event catalog-revision), and a read answers from it without +// a call to the bus or a listing of the bucket. Say, when set, says each read of a revision with its number, its module +// count and how long it took. type KeptTrunk struct { + // View is the half the follower holds; false while it has none. + View func() (meshcatalogclient.Half, bool) Say func(format string, a ...any) + mu sync.Mutex at int64 mods Store @@ -518,33 +530,44 @@ func (t TrunkStates) read(nodes bool) (Store, int64, error) { return mods, at, err } -// view is the view of the newest trunk revision: the one kept when it is still the newest, else read whole and kept. +// view is the view of the trunk: the one held while the follower's half is still at its revision, else made from the +// half and held. func (t TrunkStates) view() (modules, nodes Store, at int64, err error) { - ctx := context.Background() if t.Kept == nil { - return TrunkView(ctx, t.Objects) - } - k := t.Kept - newest, any, err := newestOf(ctx, t.Objects, TrunkBranch) - if err != nil || !any { - return newOverlay(nil), newOverlay(nil), 0, err + return TrunkView(context.Background(), t.Objects) } + return t.Kept.current() +} + +// current is the view of the half the follower holds now; a newer half than the view held is made into a view once. +func (k *KeptTrunk) current() (Store, Store, int64, error) { + half, ok := k.View() k.mu.Lock() defer k.mu.Unlock() - if k.mods != nil && k.at == newest.Revision { + if !ok { + if k.mods == nil { + return newOverlay(nil), newOverlay(nil), 0, nil + } return k.mods, k.held, k.at, nil } - started := time.Now() - mods, held, at, err := TrunkView(ctx, t.Objects) + if k.mods != nil && k.at == half.Revision.Revision { + return k.mods, k.held, k.at, nil + } + began := time.Now() + entries := make([]meshcatalog.CatalogEntry, 0, len(half.Entries)) + for _, e := range half.Entries { + entries = append(entries, e) + } + mods, held, at, err := viewOfEntries(half.Revision.Revision, entries) if err != nil { if k.Say != nil { - k.Say("the view of the trunk: revision %d cannot be read after %s: %v", newest.Revision, time.Since(started).Round(time.Millisecond), err) + k.Say("the view of the trunk: revision %d cannot be made: %v", half.Revision.Revision, err) } return nil, nil, 0, err } if k.Say != nil { - n, _ := mods.Keys(ctx) - k.Say("the view of the trunk: read revision %d (was %d), %d modules, in %s", at, k.at, len(n), time.Since(started).Round(time.Millisecond)) + n, _ := mods.Keys(context.Background()) + k.Say("the view of the trunk: read revision %d (was %d), %d modules, in %s", at, k.at, len(n), time.Since(began).Round(time.Millisecond)) } k.mods, k.held, k.at = mods, held, at return mods, held, at, nil diff --git a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go new file mode 100644 index 00000000..508cddeb --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go @@ -0,0 +1,91 @@ +package index + +import ( + "context" + "errors" + "fmt" + "strings" + "sync/atomic" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// listedObjects counts the listings of its bucket. +type listedObjects struct { + *memObjects + n atomic.Int64 +} + +func (l *listedObjects) Names(ctx context.Context) ([]string, error) { + l.n.Add(1) + return l.memObjects.Names(ctx) +} + +// A-487: the view of the trunk is made from the half the follower holds, once per revision it holds; a read never lists +// the bucket or asks the bus. +func TestTheKeptTrunkIsMadeOncePerRevisionOfTheHalfHeld(t *testing.T) { + objects := &listedObjects{memObjects: newMemObjects(0)} + var half meshcatalogclient.Half + held := false + var asks int + said := []string{} + kept := &KeptTrunk{ + View: func() (meshcatalogclient.Half, bool) { asks++; return half, held }, + Say: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }, + } + modules := TrunkStates{Objects: objects, Under: func(string) views.Source { return nil }, Kept: kept}.State("modules") + if _, err := modules.Keys(); !errors.Is(err, ErrNoRevision) { + t.Fatalf("with no half held a read answered %v, want it to wait", err) + } + holds := func(revision int64, names ...string) { + half = meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk"}, Entries: map[string]meshcatalog.CatalogEntry{}} + for _, name := range names { + half.Entries[name] = meshcatalog.CatalogEntry{Module: name, Repository: catalogueRepo, None: "not built"} + } + held = true + } + holds(1, "avahi") + for i := 0; i < 50; i++ { + if _, _, err := modules.Get("avahi"); err != nil { + t.Fatal(err) + } + if _, err := modules.Keys(); err != nil { + t.Fatal(err) + } + } + if len(said) != 1 || !strings.Contains(said[0], "read revision 1 (was 0), 1 modules, in ") { + t.Errorf("100 reads said %q, want one read of revision 1 with its module count and time", said) + } + holds(2, "avahi", "ca-trust") + if e, ok, err := modules.Get("ca-trust"); err != nil || !ok || e.Revision != 2 { + t.Errorf("after the half moved ca-trust = %+v %v (%v), want it at revision 2", e, ok, err) + } + if len(said) != 2 || !strings.Contains(said[1], "read revision 2 (was 1), 2 modules, in ") { + t.Errorf("said %q, want each read with its revision, module count and time", said) + } + if lists := objects.n.Load(); lists != 0 { + t.Errorf("the reads listed the bucket %d times, want never", lists) + } +} + +// A-487: the verb module reads the one module it is asked. +func TestTheVerbModuleReadsOneModule(t *testing.T) { + c := CatalogRead{ + Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) { + return nil, errors.New("every module was read") + }, + Module: func(name string) (meshcatalog.ModulesByModuleValue, bool, error) { + return meshcatalog.ModulesByModuleValue{Manifest: "d1"}, name == "avahi", nil + }, + Manifest: func(string) (string, error) { return `{"module":"avahi"}`, nil }, + } + if one, err := c.CatalogModuleAnswer("avahi"); err != nil || one.Module.Module != "avahi" { + t.Errorf("module avahi = %+v (%v)", one, err) + } + if _, err := c.CatalogModuleAnswer("nope"); err == nil || !strings.Contains(err.Error(), "holds no module nope") { + t.Errorf("module nope = %v, want the trunk holds none", err) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 491b153b..375db333 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -611,7 +611,18 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // // A kept view reads the revision once while it stays the newest, and its first view answers in pages of values, never // one revision read per key (about 11 s a page live, 2026-11-14). reads := 0 - keptView := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{Say: func(string, ...any) { reads++ }}} + keptView := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{Say: func(string, ...any) { reads++ }, View: func() (meshcatalogclient.Half, bool) { + newest, _, _ := newestTrunk(context.Background(), first.Revisions) + _, entries, err := ReadRevision(context.Background(), first.Revisions, RevisionObject(newest)) + if err != nil { + t.Fatal(err) + } + half := meshcatalogclient.Half{Revision: newest, Entries: map[string]meshcatalog.CatalogEntry{}} + for _, e := range entries { + half.Entries[e.Module] = e + } + return half, true + }}} if a, err := views.FirstView(testSelf, ModulesState, keptView.State(ModulesState), keptView.State(views.MarksState), "", time.Now()); err != nil || a.Revision != 2 || len(a.Values) != len(builderModules(t)) || a.Next != "" { t.Errorf("the kept first view modules answered %d values at %d, next %q (%v), want every module at revision 2 in one page", len(a.Values), a.Revision, a.Next, err) diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index cc051196..d1003273 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -165,7 +165,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control // internal/broker/state.go:62-93). // The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*). - if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-conditions.findings-asked", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**", + if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**", "mesh-delivery.freed", "mesh-delivery.heads-replaced", "mesh-delivery.pinned", "mesh-delivery.pins-stated", "mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.run-started", "mesh-wants.version-delivered", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } @@ -178,7 +178,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths: // ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a // commit is in another (issue 1900). - if want := []string{"seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", + if want := []string{"seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", "seat:node-build-go.accept.check", // design 47 G A-231: the validation's run ask (§712) on the check queue "seat:node-build-go.output", // design 47 G A-254: a validation's answer kept as an object "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) { diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 6d387c43..6fe600b6 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -94,6 +94,7 @@ "mesh-wants.wait-ended", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**", + "mesh-catalog.catalog-revision", "mesh-delivery.source-moved", "mesh-delivery.pinned", "mesh-delivery.freed", @@ -176,6 +177,7 @@ "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", + "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps",