Catalog answers from the newest trunk revision held in memory, and module reads one module
The tools bundle's view of the trunk (KeptTrunk) is made from the half of the catalog its own follower of the seat holds (meshcatalogclient.WireCatalog on a views.Reader: the newest revision once at the start, again on the seat's event catalog-revision), once per revision, and says each read with its revision, module count and time. A read no longer lists catalog-revisions or fetches the newest revision object. The verb module reads the one module it is asked (CatalogRead.Module). The manifest consumes mesh-catalog.catalog-revision and invokes seat:mesh-catalog.revision for the follower. Builds: design 47 G §421, lead answer A-487 (03-DESIGN/01-to-be/47-milestones/G-22-lead-answers-2.md)
This commit is contained in:
@@ -38,6 +38,18 @@ func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.So
|
||||
}
|
||||
return out, nil
|
||||
},
|
||||
Module: func(name string) (meshcatalog.ModulesByModuleValue, bool, error) {
|
||||
key, err := meshcatalogclient.ModulesKey(name)
|
||||
if err != nil {
|
||||
return meshcatalog.ModulesByModuleValue{}, false, nil // a name no key holds is a module the trunk holds none of
|
||||
}
|
||||
e, ok, err := modules.Get(key)
|
||||
if err != nil || !ok {
|
||||
return meshcatalog.ModulesByModuleValue{}, false, err
|
||||
}
|
||||
m, err := meshcatalogclient.DecodeModule(e.Value)
|
||||
return m, err == nil, nil
|
||||
},
|
||||
Manifest: func(digest string) (string, error) {
|
||||
key, err := meshcatalogclient.ManifestsKey(digest)
|
||||
if err != nil {
|
||||
|
||||
@@ -17,9 +17,11 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/busstate"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
@@ -286,9 +288,19 @@ func main() {
|
||||
tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts)
|
||||
// The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go.
|
||||
revisions := bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)}
|
||||
trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: &index.KeptTrunk{Say: func(format string, a ...any) {
|
||||
// The view of the trunk is the half the bundle's follower of the seat holds (design 47 G A-487): it reads the newest
|
||||
// revision once when the runtime starts and again on the seat's event catalog-revision, like every part's follower.
|
||||
say := func(format string, a ...any) {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...)
|
||||
}}}
|
||||
}
|
||||
follow := &views.Reader{Module: os.Getenv(stdio.ServedAs), Logf: say, Acts: acts}
|
||||
half, err := meshcatalogclient.WireCatalog(follow, stdio.Ask, index.TrunkBranch, nil)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
follow.Start(context.Background())
|
||||
trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: &index.KeptTrunk{View: func() (meshcatalogclient.Half, bool) { return half.View() }, Say: say}}
|
||||
tools = append(tools, catalogTools(trunk.State)...)
|
||||
// The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly
|
||||
// the parts whose entry declares it (novox/hq ADR 0663).
|
||||
|
||||
@@ -25,6 +25,9 @@ import (
|
||||
type CatalogRead struct {
|
||||
// Modules is every value of the modules state, by module.
|
||||
Modules func() (map[string]meshcatalog.ModulesByModuleValue, error)
|
||||
// Module is the one value of the modules state of that module, and whether the state holds it: the verb module reads
|
||||
// the one module it is asked, never every module (design 47 G A-487).
|
||||
Module func(name string) (meshcatalog.ModulesByModuleValue, bool, error)
|
||||
// Manifest is the manifest kept under a digest in the manifests state, as its JSON.
|
||||
Manifest func(digest string) (string, error)
|
||||
// Graph is the module graph, by module.
|
||||
@@ -51,17 +54,26 @@ func (c CatalogRead) CatalogModulesAnswer() (catalogue.CatalogModules, error) {
|
||||
return catalogue.CatalogModules{Modules: out}, nil
|
||||
}
|
||||
|
||||
// module is one module's value: through Module when set, else out of every module's.
|
||||
func (c CatalogRead) module(name string) (meshcatalog.ModulesByModuleValue, bool, error) {
|
||||
if c.Module != nil {
|
||||
return c.Module(name)
|
||||
}
|
||||
mods, err := c.Modules()
|
||||
m, ok := mods[name]
|
||||
return m, ok, err
|
||||
}
|
||||
|
||||
// CatalogModuleAnswer is the verb module: one module whole, its manifest included.
|
||||
func (c CatalogRead) CatalogModuleAnswer(name string) (catalogue.CatalogOne, error) {
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return catalogue.CatalogOne{}, fmt.Errorf("name the module")
|
||||
}
|
||||
mods, err := c.Modules()
|
||||
m, ok, err := c.module(name)
|
||||
if err != nil {
|
||||
return catalogue.CatalogOne{}, fmt.Errorf("the indexer's modules cannot be read: %w", err)
|
||||
}
|
||||
m, ok := mods[name]
|
||||
if !ok {
|
||||
return catalogue.CatalogOne{}, fmt.Errorf("the trunk holds no module %s", name)
|
||||
}
|
||||
|
||||
@@ -438,15 +438,21 @@ func TrunkView(ctx context.Context, o Objects) (modules, nodes Store, at int64,
|
||||
// BranchView is TrunkView of a watched branch: the view read from the newest revision the bucket keeps of that branch
|
||||
// (design 47 G §291, A-93), empty and at 0 when it keeps none.
|
||||
func BranchView(ctx context.Context, o Objects, branch string) (modules, nodes Store, at int64, err error) {
|
||||
view, held := newOverlay(nil), newOverlay(nil)
|
||||
newest, any, err := newestOf(ctx, o, branch)
|
||||
if err != nil || !any {
|
||||
return view, held, 0, err
|
||||
return newOverlay(nil), newOverlay(nil), 0, err
|
||||
}
|
||||
_, entries, err := ReadRevision(ctx, o, RevisionObject(newest))
|
||||
if err != nil {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
return viewOfEntries(newest.Revision, entries)
|
||||
}
|
||||
|
||||
// viewOfEntries is the view of the entries of one trunk revision, as BranchView reads them.
|
||||
func viewOfEntries(revision int64, entries []meshcatalog.CatalogEntry) (modules, nodes Store, at int64, err error) {
|
||||
view, held := newOverlay(nil), newOverlay(nil)
|
||||
ctx, newest := context.Background(), meshcatalog.CatalogRevision{Revision: revision}
|
||||
for _, e := range entries {
|
||||
key, err := meshcatalogclient.ModulesKey(e.Module)
|
||||
if err != nil {
|
||||
@@ -496,10 +502,16 @@ type TrunkStates struct {
|
||||
Kept *KeptTrunk
|
||||
}
|
||||
|
||||
// KeptTrunk is the view of the trunk as last read, at its revision, shared by every read of a TrunkStates; Say, when set,
|
||||
// says each read of a revision and how long it took.
|
||||
// KeptTrunk is the view of the newest trunk revision, held in memory (design 47 G A-487): it is made from the half of the
|
||||
// catalog the bundle's follower of the seat holds (meshcatalogclient's Catalog, which reads the newest revision once when
|
||||
// its runtime starts and again only when it hears the seat's event catalog-revision), and a read answers from it without
|
||||
// a call to the bus or a listing of the bucket. Say, when set, says each read of a revision with its number, its module
|
||||
// count and how long it took.
|
||||
type KeptTrunk struct {
|
||||
// View is the half the follower holds; false while it has none.
|
||||
View func() (meshcatalogclient.Half, bool)
|
||||
Say func(format string, a ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
at int64
|
||||
mods Store
|
||||
@@ -518,33 +530,44 @@ func (t TrunkStates) read(nodes bool) (Store, int64, error) {
|
||||
return mods, at, err
|
||||
}
|
||||
|
||||
// view is the view of the newest trunk revision: the one kept when it is still the newest, else read whole and kept.
|
||||
// view is the view of the trunk: the one held while the follower's half is still at its revision, else made from the
|
||||
// half and held.
|
||||
func (t TrunkStates) view() (modules, nodes Store, at int64, err error) {
|
||||
ctx := context.Background()
|
||||
if t.Kept == nil {
|
||||
return TrunkView(ctx, t.Objects)
|
||||
}
|
||||
k := t.Kept
|
||||
newest, any, err := newestOf(ctx, t.Objects, TrunkBranch)
|
||||
if err != nil || !any {
|
||||
return newOverlay(nil), newOverlay(nil), 0, err
|
||||
return TrunkView(context.Background(), t.Objects)
|
||||
}
|
||||
return t.Kept.current()
|
||||
}
|
||||
|
||||
// current is the view of the half the follower holds now; a newer half than the view held is made into a view once.
|
||||
func (k *KeptTrunk) current() (Store, Store, int64, error) {
|
||||
half, ok := k.View()
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if k.mods != nil && k.at == newest.Revision {
|
||||
if !ok {
|
||||
if k.mods == nil {
|
||||
return newOverlay(nil), newOverlay(nil), 0, nil
|
||||
}
|
||||
return k.mods, k.held, k.at, nil
|
||||
}
|
||||
started := time.Now()
|
||||
mods, held, at, err := TrunkView(ctx, t.Objects)
|
||||
if k.mods != nil && k.at == half.Revision.Revision {
|
||||
return k.mods, k.held, k.at, nil
|
||||
}
|
||||
began := time.Now()
|
||||
entries := make([]meshcatalog.CatalogEntry, 0, len(half.Entries))
|
||||
for _, e := range half.Entries {
|
||||
entries = append(entries, e)
|
||||
}
|
||||
mods, held, at, err := viewOfEntries(half.Revision.Revision, entries)
|
||||
if err != nil {
|
||||
if k.Say != nil {
|
||||
k.Say("the view of the trunk: revision %d cannot be read after %s: %v", newest.Revision, time.Since(started).Round(time.Millisecond), err)
|
||||
k.Say("the view of the trunk: revision %d cannot be made: %v", half.Revision.Revision, err)
|
||||
}
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
if k.Say != nil {
|
||||
n, _ := mods.Keys(ctx)
|
||||
k.Say("the view of the trunk: read revision %d (was %d), %d modules, in %s", at, k.at, len(n), time.Since(started).Round(time.Millisecond))
|
||||
n, _ := mods.Keys(context.Background())
|
||||
k.Say("the view of the trunk: read revision %d (was %d), %d modules, in %s", at, k.at, len(n), time.Since(began).Round(time.Millisecond))
|
||||
}
|
||||
k.mods, k.held, k.at = mods, held, at
|
||||
return mods, held, at, nil
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// listedObjects counts the listings of its bucket.
|
||||
type listedObjects struct {
|
||||
*memObjects
|
||||
n atomic.Int64
|
||||
}
|
||||
|
||||
func (l *listedObjects) Names(ctx context.Context) ([]string, error) {
|
||||
l.n.Add(1)
|
||||
return l.memObjects.Names(ctx)
|
||||
}
|
||||
|
||||
// A-487: the view of the trunk is made from the half the follower holds, once per revision it holds; a read never lists
|
||||
// the bucket or asks the bus.
|
||||
func TestTheKeptTrunkIsMadeOncePerRevisionOfTheHalfHeld(t *testing.T) {
|
||||
objects := &listedObjects{memObjects: newMemObjects(0)}
|
||||
var half meshcatalogclient.Half
|
||||
held := false
|
||||
var asks int
|
||||
said := []string{}
|
||||
kept := &KeptTrunk{
|
||||
View: func() (meshcatalogclient.Half, bool) { asks++; return half, held },
|
||||
Say: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) },
|
||||
}
|
||||
modules := TrunkStates{Objects: objects, Under: func(string) views.Source { return nil }, Kept: kept}.State("modules")
|
||||
if _, err := modules.Keys(); !errors.Is(err, ErrNoRevision) {
|
||||
t.Fatalf("with no half held a read answered %v, want it to wait", err)
|
||||
}
|
||||
holds := func(revision int64, names ...string) {
|
||||
half = meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk"}, Entries: map[string]meshcatalog.CatalogEntry{}}
|
||||
for _, name := range names {
|
||||
half.Entries[name] = meshcatalog.CatalogEntry{Module: name, Repository: catalogueRepo, None: "not built"}
|
||||
}
|
||||
held = true
|
||||
}
|
||||
holds(1, "avahi")
|
||||
for i := 0; i < 50; i++ {
|
||||
if _, _, err := modules.Get("avahi"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := modules.Keys(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "read revision 1 (was 0), 1 modules, in ") {
|
||||
t.Errorf("100 reads said %q, want one read of revision 1 with its module count and time", said)
|
||||
}
|
||||
holds(2, "avahi", "ca-trust")
|
||||
if e, ok, err := modules.Get("ca-trust"); err != nil || !ok || e.Revision != 2 {
|
||||
t.Errorf("after the half moved ca-trust = %+v %v (%v), want it at revision 2", e, ok, err)
|
||||
}
|
||||
if len(said) != 2 || !strings.Contains(said[1], "read revision 2 (was 1), 2 modules, in ") {
|
||||
t.Errorf("said %q, want each read with its revision, module count and time", said)
|
||||
}
|
||||
if lists := objects.n.Load(); lists != 0 {
|
||||
t.Errorf("the reads listed the bucket %d times, want never", lists)
|
||||
}
|
||||
}
|
||||
|
||||
// A-487: the verb module reads the one module it is asked.
|
||||
func TestTheVerbModuleReadsOneModule(t *testing.T) {
|
||||
c := CatalogRead{
|
||||
Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) {
|
||||
return nil, errors.New("every module was read")
|
||||
},
|
||||
Module: func(name string) (meshcatalog.ModulesByModuleValue, bool, error) {
|
||||
return meshcatalog.ModulesByModuleValue{Manifest: "d1"}, name == "avahi", nil
|
||||
},
|
||||
Manifest: func(string) (string, error) { return `{"module":"avahi"}`, nil },
|
||||
}
|
||||
if one, err := c.CatalogModuleAnswer("avahi"); err != nil || one.Module.Module != "avahi" {
|
||||
t.Errorf("module avahi = %+v (%v)", one, err)
|
||||
}
|
||||
if _, err := c.CatalogModuleAnswer("nope"); err == nil || !strings.Contains(err.Error(), "holds no module nope") {
|
||||
t.Errorf("module nope = %v, want the trunk holds none", err)
|
||||
}
|
||||
}
|
||||
@@ -611,7 +611,18 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { //
|
||||
// A kept view reads the revision once while it stays the newest, and its first view answers in pages of values, never
|
||||
// one revision read per key (about 11 s a page live, 2026-11-14).
|
||||
reads := 0
|
||||
keptView := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{Say: func(string, ...any) { reads++ }}}
|
||||
keptView := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{Say: func(string, ...any) { reads++ }, View: func() (meshcatalogclient.Half, bool) {
|
||||
newest, _, _ := newestTrunk(context.Background(), first.Revisions)
|
||||
_, entries, err := ReadRevision(context.Background(), first.Revisions, RevisionObject(newest))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
half := meshcatalogclient.Half{Revision: newest, Entries: map[string]meshcatalog.CatalogEntry{}}
|
||||
for _, e := range entries {
|
||||
half.Entries[e.Module] = e
|
||||
}
|
||||
return half, true
|
||||
}}}
|
||||
if a, err := views.FirstView(testSelf, ModulesState, keptView.State(ModulesState), keptView.State(views.MarksState), "", time.Now()); err != nil ||
|
||||
a.Revision != 2 || len(a.Values) != len(builderModules(t)) || a.Next != "" {
|
||||
t.Errorf("the kept first view modules answered %d values at %d, next %q (%v), want every module at revision 2 in one page", len(a.Values), a.Revision, a.Next, err)
|
||||
|
||||
@@ -165,7 +165,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control
|
||||
// internal/broker/state.go:62-93).
|
||||
// The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*).
|
||||
if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-conditions.findings-asked", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**",
|
||||
if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**",
|
||||
"mesh-delivery.freed", "mesh-delivery.heads-replaced", "mesh-delivery.pinned", "mesh-delivery.pins-stated", "mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.run-started", "mesh-wants.version-delivered", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
||||
}
|
||||
@@ -178,7 +178,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths:
|
||||
// ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a
|
||||
// commit is in another (issue 1900).
|
||||
if want := []string{"seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps",
|
||||
if want := []string{"seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps",
|
||||
"seat:node-build-go.accept.check", // design 47 G A-231: the validation's run ask (§712) on the check queue
|
||||
"seat:node-build-go.output", // design 47 G A-254: a validation's answer kept as an object
|
||||
"seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) {
|
||||
|
||||
@@ -94,6 +94,7 @@
|
||||
"mesh-wants.wait-ended",
|
||||
"mesh-delivery-checker.verdict",
|
||||
"mesh-delivery-forge-bridge.forge.**",
|
||||
"mesh-catalog.catalog-revision",
|
||||
"mesh-delivery.source-moved",
|
||||
"mesh-delivery.pinned",
|
||||
"mesh-delivery.freed",
|
||||
@@ -176,6 +177,7 @@
|
||||
"seat:mesh-delivery.source",
|
||||
"seat:mesh-module-assigner.assignments",
|
||||
"seat:mesh-module-assigner.placement",
|
||||
"seat:mesh-catalog.revision",
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
"seat:mesh-watchdogs.presence",
|
||||
"seat:node-build-go.steps",
|
||||
|
||||
Reference in New Issue
Block a user