diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go index 7b080273..511ca138 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go @@ -13,8 +13,9 @@ import ( "mesh-delivery-indexer/internal/index" ) -// catalogRead reads the indexer's modules, manifests and graph states for the seat's verbs. -func catalogRead(modules, manifests, nodes stdio.KeptState, seatDefinitions, marks views.Source) index.CatalogRead { +// catalogRead reads the indexer's view of the trunk (its modules and module graph, from the newest catalog graph +// revision) and its manifests state for the seat's verbs. +func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.Source, seatDefinitions, marks views.Source) index.CatalogRead { return index.CatalogRead{ SeatDefinitions: seatDefinitions, Marks: marks, Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) { @@ -24,11 +25,11 @@ func catalogRead(modules, manifests, nodes stdio.KeptState, seatDefinitions, mar } out := map[string]meshcatalog.ModulesByModuleValue{} for _, k := range keys { - e, err := modules.Get(k) + e, ok, err := modules.Get(k) if err != nil { return nil, fmt.Errorf("modules.%s: %w", k, err) } - if e == nil { + if !ok { continue } if m, err := meshcatalogclient.DecodeModule(e.Value); err == nil { @@ -59,11 +60,11 @@ func catalogRead(modules, manifests, nodes stdio.KeptState, seatDefinitions, mar } g := graph.Graph{} for _, k := range keys { - e, err := nodes.Get(k) + e, ok, err := nodes.Get(k) if err != nil { return nil, fmt.Errorf("graph.%s: %w", k, err) } - if e == nil { + if !ok { continue } if n, err := graph.Decode(e.Value); err == nil { @@ -76,7 +77,7 @@ func catalogRead(modules, manifests, nodes stdio.KeptState, seatDefinitions, mar } // catalogTools are the seat's verbs over the indexer's own states, served beside the indexer's own tools, since the indexer's manifest claims the seat (ADR 0604 §2). -func catalogTools() []stdio.Tool { - return index.CatalogTools(catalogRead(stdio.State("modules"), stdio.State("manifests"), stdio.State(graph.State), +func catalogTools(state func(string) views.Source) []stdio.Tool { + return index.CatalogTools(catalogRead(state("modules"), stdio.State("manifests"), state(graph.State), keptSource(index.SeatDefinitionsState), keptSource(views.MarksState))) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index e4c89f9c..3126dbdd 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -17,6 +17,7 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/busstate" "mesh-delivery-indexer/internal/index" ) @@ -275,16 +276,18 @@ func main() { asks := askTakeIn(stdio.State(index.TakeInsState), 25*time.Second) acts := meshhandactsclient.Recorder{Ask: stdio.Ask} tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts) - tools = append(tools, catalogTools()...) - tools = append(tools, viewTools(os.Getenv, keptSource, time.Now)...) + // The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go. + trunk := index.TrunkStates{Objects: bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)}, Under: keptSource} + tools = append(tools, catalogTools(trunk.State)...) + tools = append(tools, viewTools(os.Getenv, trunk.State, time.Now)...) tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, keptRead{stdio.State(index.OwnBuildStateName)}, asks, time.Now)...) tools = append(tools, index.FingerprintsTools(asks, time.Now, func(what, why string) error { _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.FingerprintsAgainTool, Verb: "fingerprints-again"}) return err }, 10*time.Second)...) tools = append(tools, index.TryCandidateTools(keptRead{stdio.State("candidates")}, keptRead{stdio.State(index.TryRecoveryState)}, asks, time.Now)...) - tools = append(tools, configReadersTool(stdio.NewWatchedStates()), busStateTool(keptSource), requirementsTool(keptSource), - packageUsersTool(keptSource), seatDefinitionTool(keptSource), outOfStepTool(keptSource), + tools = append(tools, configReadersTool(stdio.NewWatchedStates(), trunk.State("modules")), busStateTool(trunk.State), requirementsTool(keptSource), + packageUsersTool(trunk.State), seatDefinitionTool(keptSource), outOfStepTool(keptSource), holdsTool(stdio.State(index.HoldsState), asks, time.Now, acts), engineReadsTool(time.Now)) if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/revision.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/revision.go new file mode 100644 index 00000000..71407173 --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/revision.go @@ -0,0 +1,50 @@ +package main + +import ( + "context" + + "git.novox.be/novox/mesh-sdk/go/busstate" + + "mesh-delivery-indexer/internal/index" +) + +// bundleObjects is the objects catalog-revisions as the bundle reaches them through its runtime (the old objects section +// of a foundation part, reached by the module's own name). +type bundleObjects struct{ o busstate.Objects } + +func (b bundleObjects) Put(_ context.Context, name string, data []byte) error { + _, err := b.o.Put(name, data, "") + return err +} + +func (b bundleObjects) Get(_ context.Context, name string) ([]byte, error) { + d, err := b.o.Get(name) + if err != nil { + return nil, err + } + if d.Info.Name == "" || d.Info.Deleted { + return nil, index.ErrNoObject + } + return d.Data, nil +} + +func (b bundleObjects) Delete(_ context.Context, name string) error { return b.o.Delete(name) } + +func (b bundleObjects) Names(context.Context) ([]string, error) { + infos, err := b.o.List() + if err != nil { + return nil, err + } + names := []string{} + for _, i := range infos { + if !i.Deleted { + names = append(names, i.Name) + } + } + return names, nil +} + +func (b bundleObjects) Fill(context.Context) (int64, int64, error) { + f, err := b.o.Fill() + return int64(f.Bytes), int64(f.MaxBytes), err +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go index 3b287093..ecd4431e 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go @@ -37,9 +37,9 @@ func viewTools(getenv func(string) string, state func(string) views.Source, now func keptSource(name string) views.Source { return views.KeptSource{State: stdio.State(name)} } // configReadersTool is indexer_config_readers (design 32 *A value no longer read*; DLV-I10): read from one watched copy -// of the indexer's modules, manifests and config-candidates states, so an ask on each manifest-accepted reads no state +// of the indexer's manifests and config-candidates states and its view of the trunk (the newest catalog graph revision), so an ask on each manifest-accepted reads no state // whole again. -func configReadersTool(watched *stdio.WatchedStates) stdio.Tool { +func configReadersTool(watched *stdio.WatchedStates, modules views.Source) stdio.Tool { return stdio.Tool{Name: index.ConfigReadersTool, Description: "Who reads a module's config keys now (design 32): every value key and own secret any manifest of the module " + "the indexer keeps ever declared, each with its readers — the module itself where its trunk's manifest declares it, and " + @@ -53,13 +53,15 @@ func configReadersTool(watched *stdio.WatchedStates) stdio.Tool { return nil, fmt.Errorf("%s names the module", index.ConfigReadersTool) } trunk := map[string]meshcatalog.ModulesByModuleValue{} - mods, err := watched.Values("modules") + keys, err := modules.Keys() if err != nil { - return nil, fmt.Errorf("the modules state cannot be read: %w", err) + return nil, fmt.Errorf("the view of the trunk cannot be read: %w", err) } - for k, v := range mods { - if m, err := meshcatalogclient.DecodeModule(v); err == nil { - trunk[k] = m + for _, k := range keys { + if e, ok, err := modules.Get(k); err == nil && ok { + if m, err := meshcatalogclient.DecodeModule(e.Value); err == nil { + trunk[k] = m + } } } kept := map[string]meshcatalog.ManifestsByDigestValue{} diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index 2ccdf931..408b5951 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -8,12 +8,14 @@ import ( "slices" "strconv" "strings" + "time" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/sdk/graph" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/views" ) // The indexer's object bucket catalog-revisions (design 47, Two graphs; novox/hq ADR 0619 §2: declared in the old @@ -325,3 +327,91 @@ func TrunkView(ctx context.Context, o Objects) (modules, nodes Store, at int64, // MemoryStore is a store held in memory alone: the module graph's nodes, which each revision's entries carry, so the // indexer keeps no state of them on the bus (milestone 47/G). func MemoryStore() Store { return newOverlay(nil) } + +// The view of the trunk is the newest catalog graph revision the indexer's objects catalog-revisions keep (milestone +// 47/G, ADR 0656 §6): the tools bundle answers the modules and the module graph it once read from the states modules and +// graph from that revision, at its number, and says it waits while the indexer keeps none yet (ADR 0615 §6), never an +// empty catalog. + +// ErrNoRevision is a view asked before the indexer kept its first trunk revision. +var ErrNoRevision = errors.New("view-waits: the indexer keeps no catalog graph revision yet (its first start takes the trunk in); ask again on catalog-revision") + +// TrunkStates reads the view of the trunk from the newest trunk revision; under answers every other state. +type TrunkStates struct { + Objects Objects + Under func(string) views.Source +} + +// read is the view's modules and nodes, and the revision they were read at; none kept yet is ErrNoRevision. +func (t TrunkStates) read(nodes bool) (Store, int64, error) { + mods, held, at, err := TrunkView(context.Background(), t.Objects) + if err == nil && at == 0 { + err = ErrNoRevision + } + if nodes { + return held, at, err + } + return mods, at, err +} + +// state is the source of a state by its name: modules and graph from the revision, their marks at its number, every +// other the indexer's own state. +func (t TrunkStates) State(name string) views.Source { + switch name { + case "modules", graph.State: + return revisionSource{t: t, nodes: name == graph.State} + case views.MarksState: + return markSource{t: t, under: t.Under(name)} + } + return t.Under(name) +} + +// revisionSource is the modules or the module graph of the newest trunk revision, each value at its number. +type revisionSource struct { + t TrunkStates + nodes bool +} + +func (r revisionSource) Keys() ([]string, error) { + s, _, err := r.t.read(r.nodes) + if err != nil { + return nil, err + } + return s.Keys(context.Background()) +} + +func (r revisionSource) Get(key string) (*views.Entry, bool, error) { + s, at, err := r.t.read(r.nodes) + if err != nil { + return nil, false, err + } + value, _, found, err := s.Get(context.Background(), key) + if err != nil || !found { + return nil, false, err + } + return &views.Entry{Value: value, Revision: uint64(at)}, true, nil +} + +// markSource answers the mark of modules and graph as the newest trunk revision's number, which catalog-revision says +// the changes after; every other mark is the views family's. +type markSource struct { + t TrunkStates + under views.Source +} + +func (m markSource) Keys() ([]string, error) { return m.under.Keys() } + +func (m markSource) Get(state string) (*views.Entry, bool, error) { + if state != "modules" && state != graph.State { + return m.under.Get(state) + } + _, at, err := m.t.read(false) + if err != nil { + return nil, false, err + } + data, err := json.Marshal(views.Mark{V: views.Version, State: state, Revision: uint64(at), At: time.Now().UTC()}) + if err != nil { + return nil, false, err + } + return &views.Entry{Value: data, Revision: uint64(at)}, true, nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 72366ff3..c0eadfda 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -24,6 +24,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/views" "git.novox.be/novox/mesh-delivery/sdk/graph" @@ -518,6 +519,18 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // } else if raw, _, _, _ := view.Get(context.Background(), viewKey); !strings.Contains(string(raw), "a field the judge refuses") { t.Errorf("the view at a restart lost the verdict: %s", raw) } + // The tools bundle answers modules and graph from the newest trunk revision, their marks at its number; with none kept + // it says it waits (view-waits), never an empty catalog. + trunkStates := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }} + if keys, err := trunkStates.State("modules").Keys(); err != nil || len(keys) != len(builderModules(t)) { + t.Errorf("the bundle's modules = %v (%v), want every module of revision 2", keys, err) + } + if e, ok, err := trunkStates.State(views.MarksState).Get("graph"); err != nil || !ok || !strings.Contains(string(e.Value), `"revision":2`) { + t.Errorf("the bundle's mark of graph = %+v %v (%v), want revision 2", e, ok, err) + } + if _, err := (TrunkStates{Objects: newMemObjects(0)}).State("graph").Keys(); !errors.Is(err, ErrNoRevision) { + t.Errorf("the bundle's graph with no revision kept answered %v, want it to wait", err) + } // ADR 0614 §2, §3: the graph names what each module invokes and serves, and leaves out, over the whole graph, the // verb edges of two modules calling each other, from either side; a third calling one of them keeps its edge. invoking := func(name, invokes string) mod {