IX29 (part 1): the seat mesh-catalog's verb entry finds an entry by its fingerprint through the state entry-index
Each revision kept writes, before it is kept, the digest of the object of every entry that has a fingerprint under that fingerprint's key; the verb entry (the tools bundle) reads the key and then the object; indexer_entries reads a key and whether its object is kept. The named set (a want or a running build keeping an entry) is not built: the event wanted carries no form for a removed want. Builds: design 47 G §675 (By fingerprint, Its tools in part), §443 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ
This commit is contained in:
@@ -288,7 +288,7 @@ func main() {
|
||||
tools = append(tools, catalogTools(trunk.State)...)
|
||||
// The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly
|
||||
// the parts whose entry declares it (novox/hq ADR 0663).
|
||||
tools = append(tools, index.RevisionTool(revisions, index.Module))
|
||||
tools = append(tools, index.RevisionTool(revisions, index.Module), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}))
|
||||
tools = append(tools, index.CatalogRevisionTools(revisions, asks, time.Now, func(what, why string) error {
|
||||
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.DropCandidateGraphTool, Verb: "drop-candidate-graph"})
|
||||
return err
|
||||
|
||||
@@ -420,7 +420,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
ReadOpen: openVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.PackagesBucket}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call),
|
||||
OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), OwnBuildState: awaitedStore{b: ownBuildState}, TryRecovery: awaitedStore{b: tryRecovery}, Graph: graph,
|
||||
SeatDefinitions: awaitedStore{b: seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats,
|
||||
Message: messageVia(calls.CallKept), Revisions: revisions, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1)
|
||||
Message: messageVia(calls.CallKept), Revisions: revisions, EntryIndex: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.EntryIndexBucket, "no entry is found by its fingerprint", logger.Printf)}, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1)
|
||||
// Each catalog revision may bring another holder of a build seat: the build modules are found again (buildmodules.go).
|
||||
SayCatalogRevision: refreshingBuildModules(sayCatalogRevision(conn, cred), &buildFollow),
|
||||
SayFill: sayFill(conn, cred),
|
||||
|
||||
@@ -100,6 +100,8 @@ type Indexer struct {
|
||||
// Revisions is the object bucket catalog-revisions (catalog_revisions.go; design 47, Two graphs); nil keeps no
|
||||
// catalog graph revision.
|
||||
Revisions Objects
|
||||
// EntryIndex is the state entry-index (entry_verb.go, design 47 G §675); nil keeps none.
|
||||
EntryIndex Store
|
||||
// SayCatalogRevision says the seat mesh-catalog's event catalog-revision for a revision kept; nil says nothing.
|
||||
SayCatalogRevision func(ctx context.Context, e meshcatalog.CatalogRevisionEvent) error
|
||||
// SayFill says the seat mesh-catalog's event filling, full or room of the catalog graph revisions, under the duplicate
|
||||
|
||||
@@ -278,6 +278,10 @@ func (ix *Indexer) keepEntries(ctx context.Context, entries map[string]meshcatal
|
||||
ix.sayFill(ctx, meshcatalog.EventFull, fmt.Sprintf("%d", src), meshcatalog.Fill{Bytes: used, Cap: capacity, Refused: refused})
|
||||
return fmt.Errorf("revision %d of source revision %d: %w", rev.Revision, src, ErrObjectsFull)
|
||||
}
|
||||
if err := ix.indexEntries(ctx, rev, entries); err != nil {
|
||||
ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err)
|
||||
return err
|
||||
}
|
||||
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
|
||||
ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err)
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// The seat mesh-catalog's verb entry finds a catalog graph entry by its build-input fingerprint (design 47 G §675, §443):
|
||||
// the indexer keeps, in its old state section entry-index, one key per fingerprint holding the digest of its entry's
|
||||
// object, written when it stores the entry; entry reads the key and then the object. The state is capped at 8 MiB.
|
||||
const (
|
||||
// EntryIndexState is the old state section of the entries' digests by fingerprint.
|
||||
EntryIndexState = "entry-index"
|
||||
// EntryIndexBucket is the bucket of the state.
|
||||
EntryIndexBucket = Module + "_" + EntryIndexState
|
||||
// EntryIndexTool is the tool that reads a fingerprint's key and its object's digest (class read).
|
||||
EntryIndexTool = "indexer_entries"
|
||||
)
|
||||
|
||||
// EntryIndexValue is one key of entry-index: the digest of the entry object of the fingerprint the key names.
|
||||
type EntryIndexValue struct {
|
||||
V int `json:"v"`
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
|
||||
// EntryIndexKey is the key of a fingerprint in entry-index, the fingerprint escaped as a key's name is (§73).
|
||||
func EntryIndexKey(fingerprint string) string { return keys.EscapeName(fingerprint) }
|
||||
|
||||
// indexEntries writes the entry-index key of each entry of rev that has a fingerprint, the digest of its object; a key that
|
||||
// already holds that digest is not written again. A fingerprint two entries carry holds the later module's, which is the
|
||||
// newest entry written.
|
||||
func (ix *Indexer) indexEntries(ctx context.Context, rev meshcatalog.CatalogRevision, entries map[string]meshcatalog.CatalogEntry) error {
|
||||
if ix.EntryIndex == nil {
|
||||
return nil
|
||||
}
|
||||
var errs []error
|
||||
for _, ref := range rev.Entries {
|
||||
fp := entries[ref.Module].Fp
|
||||
if fp == "" {
|
||||
continue
|
||||
}
|
||||
data, err := json.Marshal(EntryIndexValue{V: keys.Version, Digest: ref.Digest})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = write(ctx, ix.EntryIndex, EntryIndexKey(fp), func([]byte, bool) ([]byte, bool, error) { return data, false, nil })
|
||||
if errors.Is(err, process.ErrNotYet) {
|
||||
ix.logf("catalog graph: the state %s is not on the bus yet, so the entries of this revision are not found by their fingerprint", EntryIndexState)
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("entry-index of %s: %w", ref.Module, err))
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// Entry answers the verb entry: the entry whose key entry-index holds for the fingerprint, read from its object.
|
||||
func Entry(ctx context.Context, o Objects, index Store, in meshcatalog.EntryInput) (meshcatalog.EntryAnswer, error) {
|
||||
var out meshcatalog.EntryAnswer
|
||||
if in.Fingerprint == "" {
|
||||
return out, errors.New("name the entry's fingerprint (fingerprint)")
|
||||
}
|
||||
value, _, found, err := index.Get(ctx, EntryIndexKey(in.Fingerprint))
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("the state %s cannot be read: %w", EntryIndexState, err)
|
||||
}
|
||||
var v EntryIndexValue
|
||||
if !found || json.Unmarshal(value, &v) != nil || v.Digest == "" {
|
||||
return out, fmt.Errorf("no entry of fingerprint %s is kept", in.Fingerprint)
|
||||
}
|
||||
data, err := o.Get(ctx, EntryObject(v.Digest))
|
||||
if errors.Is(err, ErrNoObject) {
|
||||
return out, fmt.Errorf("the entry object %s of fingerprint %s is not kept", v.Digest, in.Fingerprint)
|
||||
}
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("the entry object %s: %w", v.Digest, err)
|
||||
}
|
||||
if err := json.Unmarshal(data, &out.Entry); err != nil {
|
||||
return out, fmt.Errorf("the entry object %s cannot be read: %w", v.Digest, err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// EntryTool is the verb entry as the tool mesh-catalog.entry of the indexer's own runtime, as RevisionTool is revision.
|
||||
func EntryTool(o Objects, index Store) stdio.Tool {
|
||||
return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbEntry, Run: func(args map[string]any) (any, error) {
|
||||
var in meshcatalog.EntryInput
|
||||
if raw, err := json.Marshal(args); err == nil {
|
||||
_ = json.Unmarshal(raw, &in)
|
||||
}
|
||||
return Entry(context.Background(), o, index, in)
|
||||
}}
|
||||
}
|
||||
|
||||
// EntriesAnswer is what indexer_entries answers: for a fingerprint its key and its object's digest, and whether the object
|
||||
// is kept; with none given, every fingerprint the state holds.
|
||||
type EntriesAnswer struct {
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
Kept bool `json:"kept,omitempty"`
|
||||
Keys []string `json:"keys,omitempty"`
|
||||
}
|
||||
|
||||
// EntriesTool is indexer_entries (class read): one fingerprint's key and the digest of its object, or every key of
|
||||
// entry-index with none given.
|
||||
func EntriesTool(o Objects, index Store) stdio.Tool {
|
||||
return stdio.Tool{Name: EntryIndexTool, Description: "Read the indexer's entry-index: the digest of the catalog graph entry of a fingerprint and whether its object is kept; with no fingerprint, every key it holds.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{"fingerprint": map[string]any{"type": "string", "description": "the entry's fingerprint; none lists every key"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
ctx := context.Background()
|
||||
fp, _ := args["fingerprint"].(string)
|
||||
if fp == "" {
|
||||
all, err := index.Keys(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slices.Sort(all)
|
||||
return EntriesAnswer{Keys: all}, nil
|
||||
}
|
||||
out := EntriesAnswer{Fingerprint: fp}
|
||||
value, _, found, err := index.Get(ctx, EntryIndexKey(fp))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var v EntryIndexValue
|
||||
if found && json.Unmarshal(value, &v) == nil {
|
||||
out.Digest = v.Digest
|
||||
if _, err := o.Get(ctx, EntryObject(v.Digest)); err == nil {
|
||||
out.Kept = true
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
)
|
||||
|
||||
// Design 47 G §675: the entry kept for a build-input fingerprint is found by it, through the key entry-index holds for the
|
||||
// digest of its object; a fingerprint no entry carries is answered with an error naming it.
|
||||
func TestEntryFindsAnEntryByItsFingerprint(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fp := "src2:" + strings.Repeat("b", 64)
|
||||
ix := &Indexer{Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), Now: func() time.Time { return time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) }}
|
||||
entries := map[string]meshcatalog.CatalogEntry{
|
||||
"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}},
|
||||
"none": {Module: "none", Repository: "novox/r", Commit: "c", Manifest: map[string]any{}},
|
||||
}
|
||||
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"}, entries)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.indexEntries(ctx, rev, entries); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := Entry(ctx, ix.Revisions, ix.EntryIndex, meshcatalog.EntryInput{Fingerprint: fp}); err != nil || got.Entry.Module != "avahi" {
|
||||
t.Fatalf("entry of %s: %+v %v, want avahi's", fp, got, err)
|
||||
}
|
||||
if keys, _ := ix.EntryIndex.Keys(ctx); len(keys) != 1 {
|
||||
t.Errorf("entry-index holds %v, want the one key of the entry that has a fingerprint", keys)
|
||||
}
|
||||
if _, err := Entry(ctx, ix.Revisions, ix.EntryIndex, meshcatalog.EntryInput{Fingerprint: "src2:" + strings.Repeat("c", 64)}); err == nil || !strings.Contains(err.Error(), "no entry of fingerprint") {
|
||||
t.Errorf("an unknown fingerprint: %v, want an error naming it", err)
|
||||
}
|
||||
if _, err := Entry(ctx, ix.Revisions, ix.EntryIndex, meshcatalog.EntryInput{}); err == nil {
|
||||
t.Errorf("no fingerprint was answered")
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §675: the revision the indexer keeps writes entry-index for every entry that has a fingerprint, before it keeps
|
||||
// the revision, so entry finds the entry as soon as the revision is kept.
|
||||
func TestAKeptRevisionIndexesItsEntries(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fp := "src2:" + strings.Repeat("d", 64)
|
||||
ix := &Indexer{Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), Now: func() time.Time { return time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) }}
|
||||
if err := ix.keepEntries(ctx, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := Entry(ctx, ix.Revisions, ix.EntryIndex, meshcatalog.EntryInput{Fingerprint: fp}); err != nil || got.Entry.Module != "avahi" {
|
||||
t.Fatalf("entry of %s after the revision was kept: %+v %v", fp, got, err)
|
||||
}
|
||||
}
|
||||
@@ -910,7 +910,7 @@ func (ix *Indexer) dry() *dryRun {
|
||||
Requirements: nil, Findings: nil, SeatDefinitions: nil, ReadFile: ix.ReadFile, SaySeatDefinition: nil,
|
||||
OwnBuildState: nil, underSeats: true,
|
||||
// A dry run keeps no catalog graph revision (catalog_revisions.go): its revision is the take-in's to keep.
|
||||
Revisions: nil, SayCatalogRevision: nil, SayFill: nil}
|
||||
Revisions: nil, SayCatalogRevision: nil, SayFill: nil, EntryIndex: nil}
|
||||
x.Logf = func(format string, args ...any) { ix.logf("take-in, computed and not written: "+format, args...) }
|
||||
x.Store, d.candidates = over(ix.Store)
|
||||
x.Modules, d.modules = over(ix.Modules)
|
||||
|
||||
@@ -1133,7 +1133,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", "Unjudged",
|
||||
"JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk",
|
||||
"Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn",
|
||||
"Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "OpenTries", "TryRecovery", "OwnBuildState", "Revisions", "SayCatalogRevision", "SayFill"} {
|
||||
"Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "OpenTries", "TryRecovery", "OwnBuildState", "Revisions", "SayCatalogRevision", "SayFill", "EntryIndex"} {
|
||||
named[f] = true
|
||||
}
|
||||
ty := reflect.TypeOf(Indexer{})
|
||||
|
||||
@@ -122,7 +122,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: "try-recovery", History: 1}, {Name: "own-build", History: 1}, {Name: index.PackagesState, History: index.PackagesHistory},
|
||||
{Name: views.MarksState, History: 1}, {Name: mcc.FindingsState, History: 5}, {Name: views.StatusState, History: 1},
|
||||
{Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5},
|
||||
{Name: index.PendingState, History: 1}, {Name: index.UnjudgedState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17
|
||||
{Name: index.PendingState, History: 1}, {Name: index.UnjudgedState, History: 1}, {Name: index.EntryIndexState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
// Milestone 47/G W4.2 (design 47, Two graphs; ADR 0619 §2): the catalog graph revisions are objects in the old section,
|
||||
@@ -191,8 +191,9 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
index.SeatDefinitionTool, index.SeatDefinitionsAgainTool, index.OutOfStepTool, // DLV-I19
|
||||
index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TryCandidatesTool, index.DropTryCandidateTool, index.RecoverTriesTool, index.TakeInMergeTool, index.FingerprintsAgainTool, // issue 1966; 1585/T12, T13 (ADR 0633 §5)
|
||||
// milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered.
|
||||
index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool)
|
||||
if !reflect.DeepEqual(m.Tools, want) || len(want) != 41 || slices.Contains(m.Tools, "modules") {
|
||||
index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool,
|
||||
index.EntryIndexTool) // design 47 G §675
|
||||
if !reflect.DeepEqual(m.Tools, want) || len(want) != 42 || slices.Contains(m.Tools, "modules") {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
// Design 47 G §368: version is MAJOR.MINOR (0.1 while none is published, §368); §295, §299: the catalog revisions' form and
|
||||
@@ -293,8 +294,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// ADR 0604 §2, ADR 0636 §4: it holds the seat mesh-catalog, serving its four verbs and seat-definitions from its tools
|
||||
// bundle (which stays without a request subject); ADR 0663 §1: and revision, in the same runtime of its own.
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != index.CatalogSeat || m.Claims[0].Scope != catalogue.ScopeMesh ||
|
||||
!slices.Equal(m.Claims[0].Serves, []string{"modules", "module", "provides", "dependents", index.SeatDefinitionsVerb, "revision"}) {
|
||||
t.Errorf("claims = %+v, want mesh-catalog with its six verbs", m.Claims)
|
||||
!slices.Equal(m.Claims[0].Serves, []string{"modules", "module", "provides", "dependents", index.SeatDefinitionsVerb, "revision", "entry"}) {
|
||||
t.Errorf("claims = %+v, want mesh-catalog with its seven verbs (entry: design 47 G §675)", m.Claims)
|
||||
}
|
||||
for _, e := range []string{index.SeatDefinitionChangedEvent, index.SeatDefinitionRemovedEvent} {
|
||||
if slices.Contains(m.Emits, e) {
|
||||
|
||||
@@ -13,7 +13,8 @@
|
||||
"provides",
|
||||
"dependents",
|
||||
"seat-definitions",
|
||||
"revision"
|
||||
"revision",
|
||||
"entry"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -58,7 +59,8 @@
|
||||
"indexer_catalog_revisions",
|
||||
"indexer_candidate_graphs",
|
||||
"indexer_recompute",
|
||||
"indexer_drop_candidate_graph"
|
||||
"indexer_drop_candidate_graph",
|
||||
"indexer_entries"
|
||||
],
|
||||
"consumes": [
|
||||
"go-package-registry.published",
|
||||
@@ -93,7 +95,8 @@
|
||||
{ "name": "requirements", "history": 5 },
|
||||
{ "name": "seat-definitions", "history": 5 },
|
||||
{ "name": "pending", "history": 1 },
|
||||
{ "name": "unjudged", "history": 1 }
|
||||
{ "name": "unjudged", "history": 1 },
|
||||
{ "name": "entry-index", "history": 1 }
|
||||
],
|
||||
"objects": [
|
||||
{ "name": "catalog-revisions", "max-mib": 512, "form": 1, "catch-up": "indexer_take_in" }
|
||||
|
||||
Reference in New Issue
Block a user