diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 16cce0a8..1c957a60 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -17,7 +17,7 @@ import ( "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/busstate" "git.novox.be/novox/mesh-sdk/go/process" @@ -288,19 +288,25 @@ func main() { tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts) // The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go. revisions := bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)} - // The view of the trunk is the half the bundle's follower of the seat holds (design 47 G A-487): it reads the newest - // revision once when the runtime starts and again on the seat's event catalog-revision, like every part's follower. + // The view of the trunk is read from the indexer's own objects once when the runtime starts and again each time the seat's + // event catalog-revision is heard (design 47 G A-487b), through the bundle's view path. say := func(format string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...) } + kept := &index.KeptTrunk{Say: say} + trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: kept} follow := &views.Reader{Module: os.Getenv(stdio.ServedAs), Logf: say, Acts: acts} - half, err := meshcatalogclient.WireCatalog(follow, stdio.Ask, index.TrunkBranch, nil) - if err != nil { + if err := follow.Adopt(trunkFollower(kept, revisions), meshcatalog.Seat, meshcatalog.EventCatalogRevision, func(ctx context.Context, data []byte) error { + var ev meshcatalog.CatalogRevisionEvent + if err := json.Unmarshal(data, &ev); err != nil { + return nil // not a revision event: offering it again changes nothing + } + return kept.Heard(ctx, revisions, ev) + }); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err) os.Exit(1) } follow.Start(context.Background()) - trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: &index.KeptTrunk{View: func() (meshcatalogclient.Half, bool) { return half.View() }, Say: say}} tools = append(tools, catalogTools(trunk.State)...) // The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly // the parts whose entry declares it (novox/hq ADR 0663). @@ -345,3 +351,15 @@ func main() { os.Exit(1) } } + +// trunkFollower is the bundle's view of the trunk as a follower of the reader: its first view is the one read of the +// indexer's objects the start makes, and answers no value (the picture is KeptTrunk's, not the reader's). +func trunkFollower(kept *index.KeptTrunk, revisions index.Objects) *views.Follower { + return &views.Follower{Owner: meshcatalog.Seat, State: "trunk-view", Tool: "catalog-revisions", + Ask: func(ctx context.Context, _ string) (views.Answer, error) { + if err := kept.Reread(ctx, revisions); err != nil { + return views.Answer{}, err + } + return views.Answer{V: views.Version, Owner: meshcatalog.Seat, State: "trunk-view", Created: time.Unix(1, 0).UTC(), Values: map[string]json.RawMessage{}, At: time.Now()}, nil + }} +} diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index 891486a8..79dfe40f 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -502,20 +502,63 @@ type TrunkStates struct { Kept *KeptTrunk } -// KeptTrunk is the view of the newest trunk revision, held in memory (design 47 G A-487): it is made from the half of the -// catalog the bundle's follower of the seat holds (meshcatalogclient's Catalog, which reads the newest revision once when -// its runtime starts and again only when it hears the seat's event catalog-revision), and a read answers from it without -// a call to the bus or a listing of the bucket. Say, when set, says each read of a revision with its number, its module -// count and how long it took. +// KeptTrunk is the view of the newest trunk revision, held in memory (design 47 G A-487, A-487b): it reads the newest +// revision from the indexer's own objects catalog-revisions once when the runtime starts (Reread) and again each time it +// hears the seat's event catalog-revision of a newer trunk revision (Heard); a read answers from the view held and never +// lists the bucket. Say, when set, says each read of a revision with its number, its module count and how long it took. type KeptTrunk struct { - // View is the half the follower holds; false while it has none. - View func() (meshcatalogclient.Half, bool) - Say func(format string, a ...any) + Say func(format string, a ...any) - mu sync.Mutex - at int64 - mods Store - held Store + read sync.Mutex // one re-read at a time; a read of the view never holds it + mu sync.Mutex + started bool + at int64 + mods Store + held Store +} + +// Reread reads the newest trunk revision of the bucket and holds its view; a bucket keeping none holds an empty view at 0. +func (k *KeptTrunk) Reread(ctx context.Context, o Objects) error { + k.read.Lock() + defer k.read.Unlock() + return k.reread(ctx, o) +} + +func (k *KeptTrunk) reread(ctx context.Context, o Objects) error { + began := time.Now() + mods, held, at, err := TrunkView(ctx, o) + if err != nil { + if k.Say != nil { + k.Say("the view of the trunk: the newest revision cannot be read after %s: %v", time.Since(began).Round(time.Millisecond), err) + } + return err + } + k.mu.Lock() + was := k.at + k.mods, k.held, k.at, k.started = mods, held, at, true + k.mu.Unlock() + if k.Say != nil { + n, _ := mods.Keys(ctx) + k.Say("the view of the trunk: read revision %d (was %d), %d modules, in %s", at, was, len(n), time.Since(began).Round(time.Millisecond)) + } + return nil +} + +// Heard is the seat's event catalog-revision: one of a trunk revision newer than the one held wakes one re-read of the +// objects; any other event is no change. A failed re-read is an error, so the event is offered again. +func (k *KeptTrunk) Heard(ctx context.Context, o Objects, e meshcatalog.CatalogRevisionEvent) error { + if e.Kind != "trunk" || !isTrunkBranch(e.Branch) { + return nil + } + k.read.Lock() + defer k.read.Unlock() + k.mu.Lock() + held := k.started && e.Revision <= k.at + k.mu.Unlock() + if held { + return nil + } + return k.reread(ctx, o) } // read is the view's modules and nodes, and the revision they were read at; none kept yet is ErrNoRevision. @@ -536,41 +579,31 @@ func (t TrunkStates) view() (modules, nodes Store, at int64, err error) { if t.Kept == nil { return TrunkView(context.Background(), t.Objects) } - return t.Kept.current() + return t.Kept.current(context.Background(), t.Objects) } -// current is the view of the half the follower holds now; a newer half than the view held is made into a view once. -func (k *KeptTrunk) current() (Store, Store, int64, error) { - half, ok := k.View() +// current is the view held; before the start's read it reads the newest revision once. +func (k *KeptTrunk) current(ctx context.Context, o Objects) (Store, Store, int64, error) { + k.mu.Lock() + started := k.started + k.mu.Unlock() + if !started { + k.read.Lock() + k.mu.Lock() + started = k.started + k.mu.Unlock() + var err error + if !started { + err = k.reread(ctx, o) + } + k.read.Unlock() + if err != nil { + return nil, nil, 0, err + } + } k.mu.Lock() defer k.mu.Unlock() - if !ok { - if k.mods == nil { - return newOverlay(nil), newOverlay(nil), 0, nil - } - return k.mods, k.held, k.at, nil - } - if k.mods != nil && k.at == half.Revision.Revision { - return k.mods, k.held, k.at, nil - } - began := time.Now() - entries := make([]meshcatalog.CatalogEntry, 0, len(half.Entries)) - for _, e := range half.Entries { - entries = append(entries, e) - } - mods, held, at, err := viewOfEntries(half.Revision.Revision, entries) - if err != nil { - if k.Say != nil { - k.Say("the view of the trunk: revision %d cannot be made: %v", half.Revision.Revision, err) - } - return nil, nil, 0, err - } - if k.Say != nil { - n, _ := mods.Keys(context.Background()) - k.Say("the view of the trunk: read revision %d (was %d), %d modules, in %s", at, k.at, len(n), time.Since(began).Round(time.Millisecond)) - } - k.mods, k.held, k.at = mods, held, at - return mods, held, at, nil + return k.mods, k.held, k.at, nil } // state is the source of a state by its name: modules and graph from the revision, their marks at its number, every diff --git a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go index 508cddeb..5513c537 100644 --- a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go +++ b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go @@ -2,13 +2,13 @@ package index import ( "context" + "encoding/json" "errors" "fmt" "strings" "sync/atomic" "testing" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-sdk/go/views" ) @@ -24,30 +24,31 @@ func (l *listedObjects) Names(ctx context.Context) ([]string, error) { return l.memObjects.Names(ctx) } -// A-487: the view of the trunk is made from the half the follower holds, once per revision it holds; a read never lists -// the bucket or asks the bus. -func TestTheKeptTrunkIsMadeOncePerRevisionOfTheHalfHeld(t *testing.T) { +// A-487, A-487b: the view of the trunk is read from the bucket once at the start and once for each heard event of a newer +// trunk revision; 100 reads between events list the bucket zero times. +func TestTheKeptTrunkIsReadAtTheStartAndOncePerHeardRevision(t *testing.T) { objects := &listedObjects{memObjects: newMemObjects(0)} - var half meshcatalogclient.Half - held := false - var asks int - said := []string{} - kept := &KeptTrunk{ - View: func() (meshcatalogclient.Half, bool) { asks++; return half, held }, - Say: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }, - } - modules := TrunkStates{Objects: objects, Under: func(string) views.Source { return nil }, Kept: kept}.State("modules") - if _, err := modules.Keys(); !errors.Is(err, ErrNoRevision) { - t.Fatalf("with no half held a read answered %v, want it to wait", err) - } - holds := func(revision int64, names ...string) { - half = meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk"}, Entries: map[string]meshcatalog.CatalogEntry{}} - for _, name := range names { - half.Entries[name] = meshcatalog.CatalogEntry{Module: name, Repository: catalogueRepo, None: "not built"} + keep := func(revision int64, modules ...string) { + t.Helper() + rev, entries := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk"}, map[string][]byte{} + for _, name := range modules { + data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: name, Repository: catalogueRepo, None: "not built"}) + digest := fmt.Sprintf("%d-%s", revision, name) + entries[digest] = data + rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: name, Digest: digest}) + } + if err := KeepRevision(context.Background(), objects, rev, entries); err != nil { + t.Fatal(err) } - held = true } - holds(1, "avahi") + said := []string{} + kept := &KeptTrunk{Say: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }} + modules := TrunkStates{Objects: objects, Under: func(string) views.Source { return nil }, Kept: kept}.State("modules") + keep(1, "avahi") + if err := kept.Reread(context.Background(), objects); err != nil { // the start's read + t.Fatal(err) + } + listed := objects.n.Load() for i := 0; i < 50; i++ { if _, _, err := modules.Get("avahi"); err != nil { t.Fatal(err) @@ -56,19 +57,35 @@ func TestTheKeptTrunkIsMadeOncePerRevisionOfTheHalfHeld(t *testing.T) { t.Fatal(err) } } - if len(said) != 1 || !strings.Contains(said[0], "read revision 1 (was 0), 1 modules, in ") { - t.Errorf("100 reads said %q, want one read of revision 1 with its module count and time", said) + if got := objects.n.Load() - listed; got != 0 { + t.Errorf("100 reads listed the bucket %d times, want none", got) + } + // a newer revision kept is no reason to read: only the event says it + keep(2, "avahi", "ca-trust") + listed = objects.n.Load() + if keys, _ := modules.Keys(); len(keys) != 1 { + t.Errorf("the view holds %v before the event, want revision 1's one module", keys) + } + for _, e := range []meshcatalog.CatalogRevisionEvent{{Kind: "candidate", Revision: 9}, {Kind: "trunk", Branch: "other", Revision: 9}, {Kind: "trunk", Revision: 1}} { + if err := kept.Heard(context.Background(), objects, e); err != nil { + t.Errorf("%+v: %v", e, err) + } + } + if got := objects.n.Load() - listed; got != 0 { + t.Errorf("events of a candidate, another branch and the revision held listed the bucket %d times, want none", got) + } + if err := kept.Heard(context.Background(), objects, meshcatalog.CatalogRevisionEvent{Kind: "trunk", Branch: "main", Revision: 2}); err != nil { + t.Fatal(err) + } + if got := objects.n.Load() - listed; got != 1 { + t.Errorf("a heard catalog-revision listed the bucket %d times, want exactly one re-read", got) } - holds(2, "avahi", "ca-trust") if e, ok, err := modules.Get("ca-trust"); err != nil || !ok || e.Revision != 2 { - t.Errorf("after the half moved ca-trust = %+v %v (%v), want it at revision 2", e, ok, err) + t.Errorf("after the event ca-trust = %+v %v (%v), want it at revision 2", e, ok, err) } if len(said) != 2 || !strings.Contains(said[1], "read revision 2 (was 1), 2 modules, in ") { t.Errorf("said %q, want each read with its revision, module count and time", said) } - if lists := objects.n.Load(); lists != 0 { - t.Errorf("the reads listed the bucket %d times, want never", lists) - } } // A-487: the verb module reads the one module it is asked. diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 375db333..491b153b 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -611,18 +611,7 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // // A kept view reads the revision once while it stays the newest, and its first view answers in pages of values, never // one revision read per key (about 11 s a page live, 2026-11-14). reads := 0 - keptView := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{Say: func(string, ...any) { reads++ }, View: func() (meshcatalogclient.Half, bool) { - newest, _, _ := newestTrunk(context.Background(), first.Revisions) - _, entries, err := ReadRevision(context.Background(), first.Revisions, RevisionObject(newest)) - if err != nil { - t.Fatal(err) - } - half := meshcatalogclient.Half{Revision: newest, Entries: map[string]meshcatalog.CatalogEntry{}} - for _, e := range entries { - half.Entries[e.Module] = e - } - return half, true - }}} + keptView := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{Say: func(string, ...any) { reads++ }}} if a, err := views.FirstView(testSelf, ModulesState, keptView.State(ModulesState), keptView.State(views.MarksState), "", time.Now()); err != nil || a.Revision != 2 || len(a.Values) != len(builderModules(t)) || a.Next != "" { t.Errorf("the kept first view modules answered %d values at %d, next %q (%v), want every module at revision 2 in one page", len(a.Values), a.Revision, a.Next, err) diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index d1003273..93082750 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -178,7 +178,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths: // ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a // commit is in another (issue 1900). - if want := []string{"seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", + if want := []string{"seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", "seat:node-build-go.accept.check", // design 47 G A-231: the validation's run ask (ยง712) on the check queue "seat:node-build-go.output", // design 47 G A-254: a validation's answer kept as an object "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) { diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 6fe600b6..b7324498 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -177,7 +177,6 @@ "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", - "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps",