From b0d2d22002302df7dfb26b6745ba0138bd13c73b Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 23:04:07 +0100 Subject: [PATCH] Design 47 H1: indexer_entry reads one entry's pins and paths at a revision A reader twice could not read one entry's pins because the whole revision (1.3 MB) exceeds what the bus carries; the tool answers one module's packages, paths per artifact, fingerprint and manifest digest, at the newest trunk revision (read as entryOf reads it) or at a given one from its kept list and that one entry object. Builds: design 47 H H-A-204 (hq f977c0e2); tools-are-how-a-module-is-spoken-to --- .../cmd/mesh-delivery-indexer-tools/entry.go | 85 +++++++++++++++ .../mesh-delivery-indexer-tools/entry_test.go | 100 ++++++++++++++++++ .../cmd/mesh-delivery-indexer-tools/main.go | 2 +- .../internal/index/catalog_revisions.go | 48 +++++++++ .../internal/index/entry_verb.go | 2 + .../mesh-delivery-indexer/manifest_test.go | 4 +- modules/mesh-delivery-indexer/module.json | 1 + 7 files changed, 239 insertions(+), 3 deletions(-) create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry.go create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry_test.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry.go new file mode 100644 index 00000000..8eb969dd --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry.go @@ -0,0 +1,85 @@ +package main + +import ( + "context" + "fmt" + "strconv" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" + + "mesh-delivery-indexer/internal/index" +) + +// entryAnswer is what indexer_entry answers: the trunk revision read, the entry's packages (the pins: each other-repository +// package with the version it builds against, empty while none is published), its named paths by artifact, its fingerprint +// and its manifest's digest. +type entryAnswer struct { + Module string `json:"module"` + Revision int64 `json:"revision"` + Packages map[string]string `json:"packages"` + Paths map[string][]string `json:"paths"` + Fp string `json:"fp,omitempty"` + ManifestDigest string `json:"manifest-digest,omitempty"` +} + +// entryReadTool is indexer_entry (H-A-204; tools-are-how-a-module-is-spoken-to) over the indexer's catalog-revisions objects: +// one module's entry pins and paths at a trunk revision, small, so a pin can be read without the revision whole (1.3 MB, +// more than the bus carries). +func entryReadTool(o index.Objects) stdio.Tool { + return stdio.Tool{Name: index.EntryPinsTool, + Description: "Read one module's catalog graph entry pins: the packages it builds against with the version each is pinned to " + + "(an empty version while none is published), the paths each artifact is built from, its fingerprint and its manifest " + + "digest, at the newest trunk revision or at a trunk revision given; a small answer, for when the revision whole is too " + + "large for the bus (H-A-204).", + Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "the module whose entry is read"}, + "revision": map[string]any{"type": "integer", "description": "the trunk revision; none reads the newest"}}}, + Run: func(args map[string]any) (any, error) { + module, _ := args["module"].(string) + module = strings.TrimSpace(module) + if module == "" { + return nil, fmt.Errorf("%s names the module", index.EntryPinsTool) + } + revision, err := revisionArg(args["revision"]) + if err != nil { + return nil, err + } + rev, e, err := index.TrunkEntryAt(context.Background(), o, module, revision) + if err != nil { + return nil, err + } + out := entryAnswer{Module: e.Module, Revision: rev.Revision, Packages: e.Packages, Paths: map[string][]string{}, + Fp: e.Fp, ManifestDigest: e.ManifestDigest} + if out.Packages == nil { + out.Packages = map[string]string{} + } + if e.Inputs != nil && e.Inputs.Paths != nil { + out.Paths = e.Inputs.Paths + } + return out, nil + }} +} + +// revisionArg is the revision argument as a number: absent is 0 (the newest); a number or its digits as a string. +func revisionArg(v any) (int64, error) { + switch r := v.(type) { + case nil: + return 0, nil + case float64: + if r < 0 || r != float64(int64(r)) { + return 0, fmt.Errorf("revision %v is not a trunk revision's number", r) + } + return int64(r), nil + case string: + if strings.TrimSpace(r) == "" { + return 0, nil + } + n, err := strconv.ParseInt(strings.TrimSpace(r), 10, 64) + if err != nil || n < 0 { + return 0, fmt.Errorf("revision %q is not a trunk revision's number", r) + } + return n, nil + } + return 0, fmt.Errorf("revision %v is not a trunk revision's number", v) +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry_test.go new file mode 100644 index 00000000..aefb9942 --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry_test.go @@ -0,0 +1,100 @@ +package main + +import ( + "context" + "encoding/json" + "reflect" + "strings" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + + "mesh-delivery-indexer/internal/index" +) + +// memObjects is catalog-revisions in memory. +type memObjects map[string][]byte + +func (m memObjects) Put(_ context.Context, name string, data []byte) error { + m[name] = data + return nil +} +func (m memObjects) Get(_ context.Context, name string) ([]byte, error) { + d, ok := m[name] + if !ok { + return nil, index.ErrNoObject + } + return d, nil +} +func (m memObjects) Delete(_ context.Context, name string) error { delete(m, name); return nil } +func (m memObjects) Names(context.Context) ([]string, error) { + var out []string + for n := range m { + out = append(out, n) + } + return out, nil +} +func (memObjects) Fill(context.Context) (int64, int64, error) { return 0, 0, nil } + +// TestEntryTool: indexer_entry answers one entry's pins, paths, fingerprint and manifest digest, small, at the newest trunk +// revision and at a given one read from its kept object; an unknown module or revision is an error (H-A-204). +func TestEntryTool(t *testing.T) { + o := memObjects{} + keep := func(revision int64, entries ...meshcatalog.CatalogEntry) { + rev := meshcatalog.CatalogRevision{Branch: index.TrunkBranch, Revision: revision} + for _, e := range entries { + e.Manifest = map[string]any{"module": e.Module, "big": strings.Repeat("x", 64<<10)} // the manifest whole is never answered + data, _ := json.Marshal(e) + digest := e.Module + "-" + e.Fp + o[index.EntryObject(digest)] = data + rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: e.Module, Digest: digest}) + } + data, _ := json.Marshal(rev) + o[index.BranchObject(index.TrunkBranch, revision)] = data + } + paths := &meshcatalog.EntryInputs{Paths: map[string][]string{"mesh-grants:program": {"modules/mesh-grants", "sdk"}}} + keep(7, meshcatalog.CatalogEntry{Module: "mesh-grants", Fp: "src2:old", ManifestDigest: "sha256:a", Inputs: paths, + Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.26"}}, meshcatalog.CatalogEntry{Module: "other", Fp: "src2:o"}) + keep(9, meshcatalog.CatalogEntry{Module: "mesh-grants", Fp: "src2:new", ManifestDigest: "sha256:b", Inputs: paths, + Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.27", "busgrants": ""}}) + tool := entryReadTool(o) + if tool.Name != "indexer_entry" { + t.Fatalf("name %s", tool.Name) + } + for _, c := range []struct { + name string + args map[string]any + want entryAnswer + err string + }{ + {"newest", map[string]any{"module": "mesh-grants"}, entryAnswer{Module: "mesh-grants", Revision: 9, Fp: "src2:new", ManifestDigest: "sha256:b", + Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.27", "busgrants": ""}, Paths: paths.Paths}, ""}, + {"given", map[string]any{"module": "mesh-grants", "revision": float64(7)}, entryAnswer{Module: "mesh-grants", Revision: 7, Fp: "src2:old", ManifestDigest: "sha256:a", + Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.26"}, Paths: paths.Paths}, ""}, + {"given as digits, no paths", map[string]any{"module": "other", "revision": "7"}, entryAnswer{Module: "other", Revision: 7, Fp: "src2:o", + Packages: map[string]string{}, Paths: map[string][]string{}}, ""}, + {"not in the newest", map[string]any{"module": "other"}, entryAnswer{}, "holds no module other"}, + {"revision not kept", map[string]any{"module": "mesh-grants", "revision": float64(8)}, entryAnswer{}, "revision 8 is not kept"}, + {"no module", map[string]any{}, entryAnswer{}, "names the module"}, + } { + t.Run(c.name, func(t *testing.T) { + got, err := tool.Run(c.args) + if c.err != "" { + if err == nil || !strings.Contains(err.Error(), c.err) { + t.Fatalf("error %v, want %q", err, c.err) + } + return + } + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, c.want) { + t.Fatalf("answer %+v, want %+v", got, c.want) + } + data, _ := json.Marshal(got) + if len(data) >= 8<<10 { + t.Fatalf("answer of %d bytes, want under 8 KB", len(data)) + } + }) + } +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index c4311fa4..a2eba94a 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -193,7 +193,7 @@ func main() { tools = append(tools, catalogTools(trunk.State, kept, watched)...) // The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly // the parts whose entry declares it (novox/hq ADR 0663). - tools = append(tools, index.RevisionTool(revisions, kept), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}), + tools = append(tools, index.RevisionTool(revisions, kept), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), entryReadTool(revisions), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}), index.OutdatedTool(&index.Indexer{Versions: keptRead{stdio.State(index.VersionsState)}, Heard: keptRead{stdio.State(index.HeardState)}, Revisions: revisions}), index.InputsReadTool(keptRead{stdio.State(index.InputsState)}, stateobjects.ForState(index.InputsState).Objects)) tools = append(tools, index.CatalogRevisionTools(revisions, asks, time.Now, func(what, why string) error { diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index ec1e7717..b26e2ba7 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -210,6 +210,54 @@ func readNewestOf(ctx context.Context, o Objects, branch string) (meshcatalog.Ca } } +// TrunkEntryAt is a module's entry in a trunk revision, with that revision's list (H-A-204): for revision 0 the newest trunk +// revision read as entryOf reads it (readNewestOf); else the kept list of that trunk revision and only the module's entry +// object, so one entry is read without the revision whole. +func TrunkEntryAt(ctx context.Context, o Objects, module string, revision int64) (meshcatalog.CatalogRevision, meshcatalog.CatalogEntry, error) { + if revision == 0 { + rev, entries, found, err := readNewestOf(ctx, o, TrunkBranch) + if err != nil { + return rev, meshcatalog.CatalogEntry{}, err + } + if !found { + return rev, meshcatalog.CatalogEntry{}, errors.New("the indexer keeps no trunk revision yet") + } + for _, e := range entries { + if e.Module == module { + return rev, e, nil + } + } + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("the trunk revision %d holds no module %s", rev.Revision, module) + } + name := BranchObject(TrunkBranch, revision) + var rev meshcatalog.CatalogRevision + data, err := o.Get(ctx, name) + if errors.Is(err, ErrNoObject) { + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("the trunk revision %d is not kept", revision) + } + if err != nil { + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("revision %s: %w", name, err) + } + if err := json.Unmarshal(data, &rev); err != nil { + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("revision %s: %w", name, err) + } + for _, ref := range rev.Entries { + if ref.Module != module { + continue + } + data, err := o.Get(ctx, EntryObject(ref.Digest)) + if err != nil { + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("revision %s, the entry of %s (%s): %w", name, module, ref.Digest, err) + } + var e meshcatalog.CatalogEntry + if err := json.Unmarshal(data, &e); err != nil { + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("revision %s, the entry of %s: %w", name, module, err) + } + return rev, e, nil + } + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("the trunk revision %d holds no module %s", revision, module) +} + // changedEntries are the modules whose entry differs between two revisions' lists, one gone or added included, by name. func changedEntries(before, after []meshcatalog.EntryRef) []string { was := map[string]string{} diff --git a/modules/mesh-delivery-indexer/internal/index/entry_verb.go b/modules/mesh-delivery-indexer/internal/index/entry_verb.go index 96aee4e8..e310ac84 100644 --- a/modules/mesh-delivery-indexer/internal/index/entry_verb.go +++ b/modules/mesh-delivery-indexer/internal/index/entry_verb.go @@ -21,6 +21,8 @@ const ( EntryIndexState = "entry-index" // EntryIndexTool is the tool that reads a fingerprint's key and its object's digest (class read). EntryIndexTool = "indexer_entries" + // EntryPinsTool is the tool that reads one module's entry pins and paths at a trunk revision (class read; H-A-204). + EntryPinsTool = "indexer_entry" ) // EntryIndexValue is one key of entry-index: the digest of the entry object of the fingerprint the key names. diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 8a24e6af..6a4fc4c4 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -205,9 +205,9 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build index.FingerprintsAgainTool, // issue 1966 // milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered. index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool, - index.EntryIndexTool, index.NamedDropTool, // design 47 G §675 + index.EntryIndexTool, index.EntryPinsTool, index.NamedDropTool, // design 47 G §675, H-A-204 index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194) - if !reflect.DeepEqual(m.Tools, want) || len(want) != 42 || !slices.Contains(m.Tools, "modules") { + if !reflect.DeepEqual(m.Tools, want) || len(want) != 43 || !slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section. diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 0eeaa80c..155327c1 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -57,6 +57,7 @@ "indexer_recompute", "indexer_drop_candidate_graph", "indexer_entries", + "indexer_entry", "indexer_named_drop", "indexer_proposals", "indexer_proposal_drop",