diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go index d658ea70..d1b03a7e 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go @@ -15,7 +15,7 @@ import ( // catalogRead reads the indexer's view of the trunk (its modules and module graph, from the newest catalog graph // revision) and its manifests state for the seat's verbs. -func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.Source, seatDefinitions, marks views.Source) index.CatalogRead { +func catalogRead(modules views.Source, manifest func(digest string) (string, error), nodes views.Source, seatDefinitions, marks views.Source) index.CatalogRead { return index.CatalogRead{ SeatDefinitions: seatDefinitions, Marks: marks, Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) { @@ -50,21 +50,7 @@ func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.So m, err := meshcatalogclient.DecodeModule(e.Value) return m, err == nil, nil }, - Manifest: func(digest string) (string, error) { - key, err := meshcatalogclient.ManifestsKey(digest) - if err != nil { - return "", err - } - e, err := manifests.Get(key) - if err != nil { - return "", err - } - if e == nil { - return "", fmt.Errorf("the manifests state holds no %s", digest) - } - kept, err := meshcatalogclient.DecodeManifest(e.Value) - return kept.Manifest, err - }, + Manifest: manifest, Graph: func() (graph.Graph, error) { keys, err := nodes.Keys() if err != nil { @@ -89,7 +75,7 @@ func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.So } // catalogTools are the seat's verbs over the indexer's own states, served beside the indexer's own tools, since the indexer's manifest claims the seat (ADR 0604 §2). -func catalogTools(state func(string) views.Source) []stdio.Tool { - return index.CatalogTools(catalogRead(state("modules"), stdio.State("manifests"), state(graph.State), +func catalogTools(state func(string) views.Source, kept *index.KeptTrunk) []stdio.Tool { + return index.CatalogTools(catalogRead(state("modules"), kept.Manifest, state(graph.State), keptSource(index.SeatDefinitionsState), keptSource(views.MarksState))) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 1c957a60..9f7f936e 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -307,10 +307,10 @@ func main() { os.Exit(1) } follow.Start(context.Background()) - tools = append(tools, catalogTools(trunk.State)...) + tools = append(tools, catalogTools(trunk.State, kept)...) // The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly // the parts whose entry declares it (novox/hq ADR 0663). - tools = append(tools, index.RevisionTool(revisions, os.Getenv(stdio.ServedAs)), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}), + tools = append(tools, index.RevisionTool(revisions, kept, os.Getenv(stdio.ServedAs)), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}), index.OutdatedTool(&index.Indexer{Versions: keptRead{stdio.State(index.VersionsState)}, Heard: keptRead{stdio.State(index.HeardState)}, Revisions: revisions}), index.InputsReadTool(keptRead{stdio.State(index.InputsState)}, stateobjects.ForState(index.InputsState).Objects)) tools = append(tools, index.CatalogRevisionTools(revisions, asks, time.Now, func(what, why string) error { @@ -355,11 +355,11 @@ func main() { // trunkFollower is the bundle's view of the trunk as a follower of the reader: its first view is the one read of the // indexer's objects the start makes, and answers no value (the picture is KeptTrunk's, not the reader's). func trunkFollower(kept *index.KeptTrunk, revisions index.Objects) *views.Follower { - return &views.Follower{Owner: meshcatalog.Seat, State: "trunk-view", Tool: "catalog-revisions", + return &views.Follower{Owner: meshcatalog.Seat, State: "catalog-half", Tool: "catalog-revisions", Ask: func(ctx context.Context, _ string) (views.Answer, error) { if err := kept.Reread(ctx, revisions); err != nil { return views.Answer{}, err } - return views.Answer{V: views.Version, Owner: meshcatalog.Seat, State: "trunk-view", Created: time.Unix(1, 0).UTC(), Values: map[string]json.RawMessage{}, At: time.Now()}, nil + return views.Answer{V: views.Version, Owner: meshcatalog.Seat, State: "catalog-half", Created: time.Unix(1, 0).UTC(), Values: map[string]json.RawMessage{}, At: time.Now()}, nil }} } diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go b/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go index e5f21613..dd0efd45 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go @@ -45,7 +45,7 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { t.Errorf("said %+v, want main 1, then the escaped branch 1 and 2", said) } answer := func(branch string, revision int64) meshcatalog.RevisionAnswer { - a, err := Revision(ctx, ix.Revisions, "indexer", "reader", meshcatalog.RevisionInput{Branch: branch, Revision: revision}) + a, err := Revision(ctx, ix.Revisions, nil, "indexer", "reader", meshcatalog.RevisionInput{Branch: branch, Revision: revision}) if err != nil { t.Fatalf("revision of %q: %v", branch, err) } @@ -66,7 +66,7 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { // A-205: `source` answers the branch's revision taken in from that source revision; a candidate's group alone answers its // newest revision, whatever its heads. bySource := func(branch string, source int64) (meshcatalog.RevisionAnswer, error) { - return Revision(ctx, ix.Revisions, "indexer", "reader", meshcatalog.RevisionInput{Branch: branch, Source: source}) + return Revision(ctx, ix.Revisions, nil, "indexer", "reader", meshcatalog.RevisionInput{Branch: branch, Source: source}) } if a, err := bySource("main", 5); err != nil || a.Revision.Revision != 1 || a.Entries[0].Commit != "c1" { t.Errorf("main's source 5 answered %+v (%v)", a.Revision, err) @@ -89,10 +89,10 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { t.Fatal(err) } } - if a, err := Revision(ctx, ix.Revisions, "indexer", "reader", meshcatalog.RevisionInput{Group: "run-ab"}); err != nil || a.Revision.Revision != 4 || a.Revision.Heads != "h2" || a.Entries[0].Commit != "c9" { + if a, err := Revision(ctx, ix.Revisions, nil, "indexer", "reader", meshcatalog.RevisionInput{Group: "run-ab"}); err != nil || a.Revision.Revision != 4 || a.Revision.Heads != "h2" || a.Entries[0].Commit != "c9" { t.Errorf("a group alone answered %+v (%v), want its newest revision 4", a.Revision, err) } - if _, err := Revision(ctx, ix.Revisions, "indexer", "reader", meshcatalog.RevisionInput{Group: "run-none"}); err == nil { + if _, err := Revision(ctx, ix.Revisions, nil, "indexer", "reader", meshcatalog.RevisionInput{Group: "run-none"}); err == nil { t.Error("a candidate never kept was answered") } } @@ -123,7 +123,7 @@ func TestTheNodeEngineReadsTheRevisionAsTheModuleHoldingItsSeat(t *testing.T) { if err := ix.KeepBranchEntries(ctx, "main", 5, map[string]meshcatalog.CatalogEntry{"mesh-host": engine}); err != nil { t.Fatal(err) } - _, err := Revision(ctx, ix.Revisions, "indexer", c.caller, meshcatalog.RevisionInput{Branch: "main"}) + _, err := Revision(ctx, ix.Revisions, nil, "indexer", c.caller, meshcatalog.RevisionInput{Branch: "main"}) if (err == nil) != c.want { t.Errorf("caller %q: err %v, want answered %v", c.caller, err, c.want) } diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index 79dfe40f..12f15438 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -515,6 +515,11 @@ type KeptTrunk struct { at int64 mods Store held Store + rev meshcatalog.CatalogRevision + entries []meshcatalog.CatalogEntry + // manifests are the manifests the revision's entries carry whole, as JSON, by their digest as a module's value names it + // (A-487c: the verbs module and provides answer them from here, never from the manifests state). + manifests map[string]string } // Reread reads the newest trunk revision of the bucket and holds its view; a bucket keeping none holds an empty view at 0. @@ -526,16 +531,37 @@ func (k *KeptTrunk) Reread(ctx context.Context, o Objects) error { func (k *KeptTrunk) reread(ctx context.Context, o Objects) error { began := time.Now() - mods, held, at, err := TrunkView(ctx, o) - if err != nil { + fail := func(err error) error { if k.Say != nil { - k.Say("the view of the trunk: the newest revision cannot be read after %s: %v", time.Since(began).Round(time.Millisecond), err) + k.Say("the view of the trunk is BEHIND, its re-read failed after %s (the view held stays, the next heard catalog-revision reads again): %v", time.Since(began).Round(time.Millisecond), err) } return err } + newest, any, err := newestTrunk(ctx, o) + if err != nil { + return fail(err) + } + var entries []meshcatalog.CatalogEntry + mods, held, at := Store(newOverlay(nil)), Store(newOverlay(nil)), int64(0) + if any { + if _, entries, err = ReadRevision(ctx, o, RevisionObject(newest)); err != nil { + return fail(err) + } + if mods, held, at, err = viewOfEntries(newest.Revision, entries); err != nil { + return fail(err) + } + } + manifests := map[string]string{} + for _, e := range entries { + if d := meshcatalog.BareManifestDigest(e.ManifestDigest); d != "" && e.Manifest != nil { + if raw, err := json.Marshal(e.Manifest); err == nil { + manifests[d] = string(raw) + } + } + } k.mu.Lock() was := k.at - k.mods, k.held, k.at, k.started = mods, held, at, true + k.mods, k.held, k.at, k.started, k.rev, k.entries, k.manifests = mods, held, at, true, newest, entries, manifests k.mu.Unlock() if k.Say != nil { n, _ := mods.Keys(ctx) @@ -544,6 +570,27 @@ func (k *KeptTrunk) reread(ctx context.Context, o Objects) error { return nil } +// Newest is the newest trunk revision held, with a copy of its entries; false when the bucket kept none at the last read. +func (k *KeptTrunk) Newest(ctx context.Context, o Objects) (meshcatalog.CatalogRevision, []meshcatalog.CatalogEntry, bool, error) { + if _, _, _, err := k.current(ctx, o); err != nil { + return meshcatalog.CatalogRevision{}, nil, false, err + } + k.mu.Lock() + defer k.mu.Unlock() + return k.rev, slices.Clone(k.entries), k.at != 0, nil +} + +// Manifest is the manifest of that digest (as a module's value names it) the held revision's entries carry. +func (k *KeptTrunk) Manifest(digest string) (string, error) { + k.mu.Lock() + defer k.mu.Unlock() + raw, ok := k.manifests[digest] + if !ok { + return "", fmt.Errorf("the newest trunk revision's entries carry no manifest %s", digest) + } + return raw, nil +} + // Heard is the seat's event catalog-revision: one of a trunk revision newer than the one held wakes one re-read of the // objects; any other event is no change. A failed re-read is an error, so the event is offered again. func (k *KeptTrunk) Heard(ctx context.Context, o Objects, e meshcatalog.CatalogRevisionEvent) error { diff --git a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go index 5513c537..c955644c 100644 --- a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go +++ b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go @@ -16,15 +16,23 @@ import ( // listedObjects counts the listings of its bucket. type listedObjects struct { *memObjects - n atomic.Int64 + n atomic.Int64 + fail atomic.Bool } func (l *listedObjects) Names(ctx context.Context) ([]string, error) { l.n.Add(1) + if l.fail.Load() { + return nil, errors.New("the bucket does not answer") + } return l.memObjects.Names(ctx) } -// A-487, A-487b: the view of the trunk is read from the bucket once at the start and once for each heard event of a newer +func keptDigest(name string) string { + return "sha256:" + fmt.Sprintf("%064x", len(name)) +} + +// A-487, A-487b, A-487c: the view of the trunk is read from the bucket once at the start and once for each heard event of a newer // trunk revision; 100 reads between events list the bucket zero times. func TestTheKeptTrunkIsReadAtTheStartAndOncePerHeardRevision(t *testing.T) { objects := &listedObjects{memObjects: newMemObjects(0)} @@ -32,7 +40,7 @@ func TestTheKeptTrunkIsReadAtTheStartAndOncePerHeardRevision(t *testing.T) { t.Helper() rev, entries := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk"}, map[string][]byte{} for _, name := range modules { - data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: name, Repository: catalogueRepo, None: "not built"}) + data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: name, Repository: catalogueRepo, None: "not built", ManifestDigest: keptDigest(name), Manifest: map[string]any{"module": name}}) digest := fmt.Sprintf("%d-%s", revision, name) entries[digest] = data rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: name, Digest: digest}) @@ -83,9 +91,39 @@ func TestTheKeptTrunkIsReadAtTheStartAndOncePerHeardRevision(t *testing.T) { if e, ok, err := modules.Get("ca-trust"); err != nil || !ok || e.Revision != 2 { t.Errorf("after the event ca-trust = %+v %v (%v), want it at revision 2", e, ok, err) } + // A-487c: the manifests by digest and the verb revision asked of the trunk's newest answer from the view held. + listed = objects.n.Load() + if raw, err := kept.Manifest(meshcatalog.BareManifestDigest(keptDigest("ca-trust"))); err != nil || raw != `{"module":"ca-trust"}` { + t.Errorf("the held manifest = %q (%v), want the entry's manifest", raw, err) + } + if a, err := Revision(context.Background(), objects, kept, "avahi", "avahi", meshcatalog.RevisionInput{}); err != nil || a.Revision.Revision != 2 || len(a.Entries) != 2 { + t.Errorf("revision of the newest answered %+v (%v), want revision 2 whole", a, err) + } + if got := objects.n.Load() - listed; got != 0 { + t.Errorf("the manifest and the verb revision listed the bucket %d times, want none", got) + } if len(said) != 2 || !strings.Contains(said[1], "read revision 2 (was 1), 2 modules, in ") { t.Errorf("said %q, want each read with its revision, module count and time", said) } + // A-487b: a re-read that fails is said loudly, the view held stays, and the next heard event reads again. + keep(3, "avahi", "ca-trust", "dbus") + objects.fail.Store(true) + if err := kept.Heard(context.Background(), objects, meshcatalog.CatalogRevisionEvent{Kind: "trunk", Revision: 3}); err == nil { + t.Error("a re-read that failed was taken") + } + if last := said[len(said)-1]; !strings.Contains(last, "BEHIND") { + t.Errorf("a failed re-read said %q, want it said loudly", last) + } + if keys, _ := modules.Keys(); len(keys) != 2 { + t.Errorf("the view held after a failed re-read has %v, want revision 2's two modules", keys) + } + objects.fail.Store(false) + if err := kept.Heard(context.Background(), objects, meshcatalog.CatalogRevisionEvent{Kind: "trunk", Revision: 3}); err != nil { + t.Fatal(err) + } + if keys, _ := modules.Keys(); len(keys) != 3 { + t.Errorf("the next heard event left the view at %v, want revision 3's three modules", keys) + } } // A-487: the verb module reads the one module it is asked. diff --git a/modules/mesh-delivery-indexer/internal/index/revision_verb.go b/modules/mesh-delivery-indexer/internal/index/revision_verb.go index 13da59fd..c3e4a589 100644 --- a/modules/mesh-delivery-indexer/internal/index/revision_verb.go +++ b/modules/mesh-delivery-indexer/internal/index/revision_verb.go @@ -65,9 +65,20 @@ const ToolRunner = "node-tools" // answered only when the newest trunk revision's entry of its module declares the verb in invokes, and the holder is // refused too. With no trunk revision kept yet the answer is `none-yet` alone (design 47 G §530), never an empty // catalog and never an error. -func Revision(ctx context.Context, o Objects, holder, caller string, in meshcatalog.RevisionInput) (meshcatalog.RevisionAnswer, error) { +// +// kept, when not nil, is the trunk's newest revision held in memory: the trunk's newest answers from it with no read of +// the bucket (A-487c); a candidate, a branch and an older number read the bucket. +func Revision(ctx context.Context, o Objects, kept *KeptTrunk, holder, caller string, in meshcatalog.RevisionInput) (meshcatalog.RevisionAnswer, error) { var out meshcatalog.RevisionAnswer - newest, ok, err := newestTrunk(ctx, o) + var newest meshcatalog.CatalogRevision + var ok bool + var trunkEntries []meshcatalog.CatalogEntry + var err error + if kept != nil { + newest, trunkEntries, ok, err = kept.Newest(ctx, o) + } else { + newest, ok, err = newestTrunk(ctx, o) + } if err != nil { return out, fmt.Errorf("the catalog graph revisions cannot be read: %w", err) } @@ -75,9 +86,10 @@ func Revision(ctx context.Context, o Objects, holder, caller string, in meshcata // An answer is told by its fields (design 47 G §530): no trunk revision kept yet is `none-yet`, no error. return meshcatalog.RevisionAnswer{NoneYet: true}, nil } - _, trunkEntries, err := ReadRevision(ctx, o, RevisionObject(newest)) - if err != nil { - return out, err + if kept == nil { + if _, trunkEntries, err = ReadRevision(ctx, o, RevisionObject(newest)); err != nil { + return out, err + } } caller = CallerModule(trunkEntries, caller) if err := views.Refusal(meshcatalog.Seat+"."+meshcatalog.VerbRevision, RevisionCallers(trunkEntries, holder), caller); err != nil { @@ -152,13 +164,13 @@ func Revision(ctx context.Context, o Objects, holder, caller string, in meshcata // RevisionTool is the verb revision as the tool mesh-catalog.revision of the indexer's own runtime (ADR 0663 §1): the // tool runner describes it with the seat's definition, so it carries no description or input of its own. -func RevisionTool(o Objects, holder string) stdio.Tool { +func RevisionTool(o Objects, kept *KeptTrunk, holder string) stdio.Tool { return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbRevision, Run: func(args map[string]any) (any, error) { var in meshcatalog.RevisionInput if raw, err := json.Marshal(args); err == nil { _ = json.Unmarshal(raw, &in) } - return Revision(context.Background(), o, holder, stdio.CallerOf(args), in) + return Revision(context.Background(), o, kept, holder, stdio.CallerOf(args), in) }} } diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 491b153b..21804c57 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -645,22 +645,22 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // t.Fatal(err) } } - if got, err := Revision(context.Background(), verbObjects, testSelf, "n1.node-tools", meshcatalog.RevisionInput{}); err != nil || !got.NoneYet || got.Revision != nil || len(got.Entries) != 0 { + if got, err := Revision(context.Background(), verbObjects, nil, testSelf, "n1.node-tools", meshcatalog.RevisionInput{}); err != nil || !got.NoneYet || got.Revision != nil || len(got.Entries) != 0 { t.Errorf("revision with none kept answered %+v, %v, want none-yet alone with no error (design 47 G §530)", got, err) } keep(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, map[string]bool{"node-tools": true, testSelf: true}) keep(meshcatalog.CatalogRevision{Revision: 4, Kind: "candidate", Group: "feat/x", Heads: "0123456789ab"}, map[string]bool{"node-tools": true, "mesh-grants": true}) for caller, answered := range map[string]bool{"n1.node-tools": true, "n1.avahi": true, "n1.mesh-grants": true, "n1." + testSelf: true, "person.someone": false, "unproven": false} { - got, err := Revision(context.Background(), verbObjects, testSelf, caller, meshcatalog.RevisionInput{}) + got, err := Revision(context.Background(), verbObjects, nil, testSelf, caller, meshcatalog.RevisionInput{}) if (err == nil) != answered || (answered && (got.Revision.Revision != 3 || len(got.Entries) != 4 || got.Next != "")) { t.Errorf("revision to %s answered %+v (%v), want answered %v with trunk revision 3 whole", caller, got, err, answered) } } - if got, err := Revision(context.Background(), verbObjects, testSelf, "n1.node-tools", meshcatalog.RevisionInput{Group: "feat/x", Heads: "0123456789ab", After: "mesh-grants"}); err != nil || + if got, err := Revision(context.Background(), verbObjects, nil, testSelf, "n1.node-tools", meshcatalog.RevisionInput{Group: "feat/x", Heads: "0123456789ab", After: "mesh-grants"}); err != nil || got.Revision.Kind != "candidate" || len(got.Entries) != 1 || got.Entries[0].Module != "node-tools" { t.Errorf("a candidate's revision after mesh-grants answered %+v (%v), want its one entry node-tools", got, err) } - if _, err := Revision(context.Background(), verbObjects, testSelf, "n1.node-tools", meshcatalog.RevisionInput{Revision: 9}); err == nil { + if _, err := Revision(context.Background(), verbObjects, nil, testSelf, "n1.node-tools", meshcatalog.RevisionInput{Revision: 9}); err == nil { t.Error("a trunk revision not kept was answered") } // ADR 0614 §2, §3: the graph names what each module invokes and serves, and leaves out, over the whole graph, the