From ab87109ec5b573fef455edd15c3a42a057df5ed1 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 9 Nov 2026 20:17:04 +0100 Subject: [PATCH 01/15] The indexer forms a catalog graph entry per module and a revision as a list of entry digests (milestone 47/G W4.1, W4.2 form) The entry is the seat mesh-catalog's CatalogEntry filled from the module graph's node a recompute computes (build edges, provisions, verbs, the edges left out on a cycle) with where the trunk holds the module and its manifest's digest; a revision lists one entry per module by the sha256 of its bytes, so an entry two revisions share is one object (design 47, an unchanged entry is stored once). Rows in the graph test: one entry per module, and only a moved entry's digest changes. Kept as objects, numbered and said with W4.2's store and W4.3. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../internal/index/catalog_graph.go | 65 +++++++++++++++++++ .../internal/index/toolchains_test.go | 26 ++++++++ 2 files changed, 91 insertions(+) create mode 100644 modules/mesh-delivery-indexer/internal/index/catalog_graph.go diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_graph.go b/modules/mesh-delivery-indexer/internal/index/catalog_graph.go new file mode 100644 index 00000000..c87d0084 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/catalog_graph.go @@ -0,0 +1,65 @@ +package index + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "maps" + "slices" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/sdk/graph" +) + +// The catalog graph (novox/hq ADR 0656; design 47, *Two graphs, one source of truth*): at each revision one entry per +// module, as its manifest declares it, in the seat mesh-catalog's layout (its definition's CatalogEntry), and the +// revision a list of its entries' digests, each entry kept once as an object named by its digest +// (catalog-revisions: trunk., candidate.., entry.). + +// CatalogEntryLayout is the layout version an entry and a revision are written in. +const CatalogEntryLayout = 1 + +// CatalogEntryOf is the catalog graph entry of one module: the module graph's node of it as a recompute computed it +// (its build, provision and verb edges, the verb edges left out on a cycle, ADR 0614), with where the trunk holds it and +// its manifest's digest. The verdict and the acting rules are the revision's to add (they are not the node's). +func CatalogEntryOf(module string, cur meshcatalog.ModulesByModuleValue, n graph.Node) meshcatalog.CatalogEntry { + e := meshcatalog.CatalogEntry{V: CatalogEntryLayout, Module: module, Repository: cur.Repository, Path: cur.Path, + Commit: cur.Commit, Manifest: cur.Manifest, Fp: n.FP, None: n.None, Bases: n.Bases, Packages: n.Packages, + Provides: n.Provides, Requires: n.Requires, Awaits: n.Awaits, Invokes: n.Invokes, Serves: n.Serves, LeftOut: n.LeftOut} + for language, t := range n.Toolchains { + if e.Toolchains == nil { + e.Toolchains = map[string]meshcatalog.BuildEdge{} + } + e.Toolchains[language] = meshcatalog.BuildEdge{Module: t.Module, Version: t.Version, Recipe: t.Recipe} + } + return e +} + +// EncodeEntry is an entry's bytes and their sha256 in hex, the name of its object: the same entry in two revisions is +// one object (design 47, *An unchanged entry is stored once*). +func EncodeEntry(e meshcatalog.CatalogEntry) (string, []byte, error) { + data, err := json.Marshal(e) + if err != nil { + return "", nil, fmt.Errorf("the entry of %s: %w", e.Module, err) + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), data, nil +} + +// CatalogRevisionOf is a revision of entries, listed by module name with each entry's digest, and the entry objects it +// lists by digest, each once. The caller numbers the revision, names its kind (trunk or candidate) and what it was read +// from. +func CatalogRevisionOf(rev meshcatalog.CatalogRevision, entries map[string]meshcatalog.CatalogEntry) (meshcatalog.CatalogRevision, map[string][]byte, error) { + rev.V, rev.Entries = CatalogEntryLayout, make([]meshcatalog.EntryRef, 0, len(entries)) + objects := map[string][]byte{} + for _, module := range slices.Sorted(maps.Keys(entries)) { + digest, data, err := EncodeEntry(entries[module]) + if err != nil { + return meshcatalog.CatalogRevision{}, nil, err + } + objects[digest] = data + rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: module, Digest: digest}) + } + return rev, objects, nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 17667eef..804983ee 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -285,6 +285,32 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // if g := graphOf(t, ix, "other"); g.FP != "" || !strings.Contains(g.None, "ts-base") || !reflect.DeepEqual(g.Bases, []string{"ts-base"}) { t.Errorf("graph.other = %+v, want its base and its reason", g) } + // Milestone 47/G W4.1, W4.2 (design 47, Two graphs): the catalog graph entry of a module is its graph node with where the + // trunk holds it; a revision lists one entry per module by digest, an unchanged entry has the same digest in the next + // revision, and a changed one only its own. + entries := map[string]meshcatalog.CatalogEntry{} + for name := range mods.vals { + m, _ := moduleOf(t, mods, name) + entries[name] = CatalogEntryOf(name, m, graphOf(t, ix, name)) + } + if e := entries["avahi"]; e.Fp != want["avahi"] || e.Commit == "" || e.Manifest == "" || !reflect.DeepEqual(e.Bases, []string{"mesh-tools"}) || + e.Toolchains["go"] != (meshcatalog.BuildEdge{Module: "mesh-build-go", Version: "go1.27.1", Recipe: "1"}) { + t.Errorf("the catalog entry of avahi = %+v", e) + } + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk", Source: 7}, entries) + if err != nil || len(rev.Entries) != len(mods.vals) || len(objects) != len(mods.vals) || rev.V != CatalogEntryLayout { + t.Fatalf("revision %+v with %d objects (%v), want one entry per module of %d", rev, len(objects), err, len(mods.vals)) + } + moved := maps.Clone(entries) + tools := moved["mesh-tools"] + tools.Fp = "src2:" + strings.Repeat("b", 64) + moved["mesh-tools"] = tools + next, _, _ := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 2, Kind: "trunk", Source: 8}, moved) + for i, ref := range next.Entries { + if same := ref.Digest == rev.Entries[i].Digest; same == (ref.Module == "mesh-tools") { + t.Errorf("entry %s: digest %s against %s before; only mesh-tools' moves", ref.Module, ref.Digest, rev.Entries[i].Digest) + } + } graph := ix.Graph.(*memStore) written := graph.writes if err := ix.Recompute(context.Background()); err != nil || graph.writes != written { From f0b51bfc29bf246186af5801b37a542da540d79a Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 9 Nov 2026 20:24:14 +0100 Subject: [PATCH 02/15] The indexer keeps its catalog graph revisions in its object bucket catalog-revisions, each entry once by digest (milestone 47/G W4.2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design 47 (Two graphs) keeps each revision as the list of its entries' digests and each entry as one object, so an unchanged entry is stored once. The bucket is declared in the old objects section with a 512 MiB cap (ADR 0619 §2: the indexer is the foundation), opened at its first use; a revision's list is written only after its entries, and a revision read back with an entry lost is an error naming the module. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-indexer/main.go | 2 +- .../cmd/mesh-delivery-indexer/objects.go | 109 ++++++++++++++++ .../internal/index/candidates.go | 3 + .../internal/index/catalog_revisions.go | 120 ++++++++++++++++++ .../internal/index/helpers_test.go | 74 ++++++++++- .../internal/index/pending.go | 4 +- .../internal/index/pending_test.go | 2 +- .../internal/index/toolchains_test.go | 21 +++ .../mesh-delivery-indexer/manifest_test.go | 11 +- modules/mesh-delivery-indexer/module.json | 3 + 10 files changed, 344 insertions(+), 5 deletions(-) create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go create mode 100644 modules/mesh-delivery-indexer/internal/index/catalog_revisions.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 01e3af36..bf432663 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -434,7 +434,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro ReadOpen: openVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.PackagesBucket}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), OwnBuildState: awaitedStore{b: ownBuildState}, TryRecovery: awaitedStore{b: tryRecovery}, Graph: awaitedStore{b: graph}, SeatDefinitions: awaitedStore{b: seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats, - Message: messageVia(calls.CallKept), // a kept call: a read, called again through a transport fault (ADR 0668 §1) + Message: messageVia(calls.CallKept), Revisions: &busObjects{js: js, local: index.CatalogRevisionsObjects}, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1) // The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear; // until then every merge is taken in at once, as before. Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.PendingBucket, "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)}, diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go new file mode 100644 index 00000000..abc08a3c --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go @@ -0,0 +1,109 @@ +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "sync" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + + "mesh-delivery-indexer/internal/index" +) + +// indexerModule is the module this program runs as, whose object buckets it opens by their local names. +const indexerModule = "mesh-delivery-indexer" + +// busObjects is one of the indexer's own object buckets in the old `objects` section (novox/hq ADR 0619 §2), which the +// grants part creates: it is opened at its first use and kept; until the bucket exists every call is an error saying so, +// and the next call opens it again (no timer: the next revision or tool call asks). +type busObjects struct { + js jetstream.JetStream + local string + mu sync.Mutex + o jetstream.ObjectStore +} + +func (b *busObjects) open(ctx context.Context) (jetstream.ObjectStore, error) { + b.mu.Lock() + defer b.mu.Unlock() + if b.o != nil { + return b.o, nil + } + o, err := b.js.ObjectStore(ctx, catalogue.ObjectsBucket(indexerModule, b.local)) + if err != nil { + return nil, fmt.Errorf("the object bucket %s is not open (the grants part creates it from the manifest's objects): %w", b.local, err) + } + b.o = o + return o, nil +} + +func (b *busObjects) Put(ctx context.Context, name string, data []byte) error { + o, err := b.open(ctx) + if err != nil { + return err + } + _, err = o.Put(ctx, jetstream.ObjectMeta{Name: name}, bytes.NewReader(data)) + return err +} + +func (b *busObjects) Get(ctx context.Context, name string) ([]byte, error) { + o, err := b.open(ctx) + if err != nil { + return nil, err + } + data, err := o.GetBytes(ctx, name) + if errors.Is(err, jetstream.ErrObjectNotFound) { + return nil, index.ErrNoObject + } + return data, err +} + +func (b *busObjects) Delete(ctx context.Context, name string) error { + o, err := b.open(ctx) + if err != nil { + return err + } + if err := o.Delete(ctx, name); err != nil && !errors.Is(err, jetstream.ErrObjectNotFound) { + return err + } + return nil +} + +func (b *busObjects) Names(ctx context.Context) ([]string, error) { + o, err := b.open(ctx) + if err != nil { + return nil, err + } + infos, err := o.List(ctx) + if errors.Is(err, jetstream.ErrNoObjectsFound) { + return []string{}, nil + } + if err != nil { + return nil, err + } + names := make([]string, 0, len(infos)) + for _, i := range infos { + names = append(names, i.Name) + } + return names, nil +} + +func (b *busObjects) Fill(ctx context.Context) (int64, int64, error) { + o, err := b.open(ctx) + if err != nil { + return 0, 0, err + } + st, err := o.Status(ctx) + if err != nil { + return 0, 0, err + } + info, err := b.js.Stream(ctx, "OBJ_"+st.Bucket()) + if err != nil { + return int64(st.Size()), 0, err + } + return int64(st.Size()), info.CachedInfo().Config.MaxBytes, nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 54463a6b..8e9c5b81 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -97,6 +97,9 @@ type Indexer struct { ReadFile func(ctx context.Context, repository, sha, path string) ([]byte, error) // SaySeatDefinition says seat-definition-changed (seatdefs.go); nil says nothing. SaySeatDefinition func(ctx context.Context, c SeatDefinitionChanged) error + // Revisions is the object bucket catalog-revisions (catalog_revisions.go; design 47, Two graphs); nil keeps no + // catalog graph revision. + Revisions Objects // Graph is the module graph's state (GraphBucket, graph.go; novox/hq ADR 0603 §2); nil writes no graph. Graph Store // Files asks the forge for every blob under each directory at a commit, by directory then path relative to it, each diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go new file mode 100644 index 00000000..3a457422 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -0,0 +1,120 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +// The indexer's object bucket catalog-revisions (design 47, Two graphs; novox/hq ADR 0619 §2: declared in the old +// `objects` section, its cap there): a revision is its list of entries' digests under trunk. or +// candidate.., and each entry one object entry., written once and shared by every revision that +// lists it. + +// CatalogRevisionsObjects is the bucket's local name in the indexer's manifest. +const CatalogRevisionsObjects = "catalog-revisions" + +// ErrNoObject is an object the bucket does not hold. +var ErrNoObject = errors.New("no such object") + +// Objects is an object bucket as the indexer keeps its revisions in it. +type Objects interface { + // Put stores data whole under name, replacing one of that name. + Put(ctx context.Context, name string, data []byte) error + // Get is the object's bytes; ErrNoObject when it holds none of that name. + Get(ctx context.Context, name string) ([]byte, error) + // Delete removes the object; one it does not hold is no error. + Delete(ctx context.Context, name string) error + // Names is every object's name; none is an empty list. + Names(ctx context.Context) ([]string, error) + // Fill is the bucket's bytes used and its cap in bytes. + Fill(ctx context.Context) (used, capacity int64, err error) +} + +// EntryObject is the object name of the entry of that digest. +func EntryObject(digest string) string { return "entry." + digest } + +// RevisionObject is the object name of a revision: trunk., or candidate.. for a candidate, +// which keeps its newest revision only. +func RevisionObject(rev meshcatalog.CatalogRevision) string { + if rev.Kind == "candidate" { + return "candidate." + rev.Group + "." + rev.Heads + } + return "trunk." + strconv.FormatInt(rev.Revision, 10) +} + +// TrunkRevisionOf is the revision number a trunk object's name holds, and whether it is one. +func TrunkRevisionOf(name string) (int64, bool) { + n, ok := strings.CutPrefix(name, "trunk.") + if !ok { + return 0, false + } + r, err := strconv.ParseInt(n, 10, 64) + return r, err == nil +} + +// KeepRevision stores a revision as CatalogRevisionOf made it: each entry object the bucket does not hold yet, then the +// revision's list, so a list is never kept whose entries are not; an entry already held is not written again. +func KeepRevision(ctx context.Context, o Objects, rev meshcatalog.CatalogRevision, entries map[string][]byte) error { + held, err := o.Names(ctx) + if err != nil { + return fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err) + } + have := make(map[string]bool, len(held)) + for _, n := range held { + have[n] = true + } + for _, ref := range rev.Entries { + name := EntryObject(ref.Digest) + if have[name] { + continue + } + data, ok := entries[ref.Digest] + if !ok { + return fmt.Errorf("revision %s lists the entry of %s (%s), which it does not carry", RevisionObject(rev), ref.Module, ref.Digest) + } + if err := o.Put(ctx, name, data); err != nil { + return fmt.Errorf("the entry of %s in %s: %w", ref.Module, CatalogRevisionsObjects, err) + } + have[name] = true + } + list, err := json.Marshal(rev) + if err != nil { + return fmt.Errorf("revision %s: %w", RevisionObject(rev), err) + } + if err := o.Put(ctx, RevisionObject(rev), list); err != nil { + return fmt.Errorf("revision %s in %s: %w", RevisionObject(rev), CatalogRevisionsObjects, err) + } + return nil +} + +// ReadRevision is the revision kept under name and its entries whole, in the revision's order; an entry it lists that +// the bucket lost is an error naming it, never an entry left out. +func ReadRevision(ctx context.Context, o Objects, name string) (meshcatalog.CatalogRevision, []meshcatalog.CatalogEntry, error) { + var rev meshcatalog.CatalogRevision + data, err := o.Get(ctx, name) + if err != nil { + return rev, nil, fmt.Errorf("revision %s: %w", name, err) + } + if err := json.Unmarshal(data, &rev); err != nil { + return rev, nil, fmt.Errorf("revision %s: %w", name, err) + } + entries := make([]meshcatalog.CatalogEntry, 0, len(rev.Entries)) + for _, ref := range rev.Entries { + data, err := o.Get(ctx, EntryObject(ref.Digest)) + if err != nil { + return rev, nil, fmt.Errorf("revision %s, the entry of %s (%s): %w", name, ref.Module, ref.Digest, err) + } + var e meshcatalog.CatalogEntry + if err := json.Unmarshal(data, &e); err != nil { + return rev, nil, fmt.Errorf("revision %s, the entry of %s: %w", name, ref.Module, err) + } + entries = append(entries, e) + } + return rev, entries, nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/helpers_test.go b/modules/mesh-delivery-indexer/internal/index/helpers_test.go index 59cb9423..4c458ce4 100644 --- a/modules/mesh-delivery-indexer/internal/index/helpers_test.go +++ b/modules/mesh-delivery-indexer/internal/index/helpers_test.go @@ -1,6 +1,12 @@ package index -import "encoding/json" +import ( + "context" + "encoding/json" + "maps" + "slices" + "sync" +) // rawList is the list a kept predefined value holds, as it was written. func rawList(value []byte) json.RawMessage { @@ -10,3 +16,69 @@ func rawList(value []byte) json.RawMessage { _ = json.Unmarshal(value, &kept) return kept.List } + +// memObjects is an object bucket in memory with a cap: a put past the cap is refused as the bus's DiscardNew refuses it. +type memObjects struct { + mu sync.Mutex + objs map[string][]byte + puts int + capacity int64 +} + +func newMemObjects(capacity int64) *memObjects { + return &memObjects{objs: map[string][]byte{}, capacity: capacity} +} + +func (m *memObjects) used() int64 { + var n int64 + for _, d := range m.objs { + n += int64(len(d)) + } + return n +} + +func (m *memObjects) Put(_ context.Context, name string, data []byte) error { + m.mu.Lock() + defer m.mu.Unlock() + if m.capacity > 0 && m.used()-int64(len(m.objs[name]))+int64(len(data)) > m.capacity { + return errObjectsFullForTest + } + m.objs[name] = slices.Clone(data) + m.puts++ + return nil +} + +func (m *memObjects) Get(_ context.Context, name string) ([]byte, error) { + m.mu.Lock() + defer m.mu.Unlock() + d, ok := m.objs[name] + if !ok { + return nil, ErrNoObject + } + return slices.Clone(d), nil +} + +func (m *memObjects) Delete(_ context.Context, name string) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.objs, name) + return nil +} + +func (m *memObjects) Names(context.Context) ([]string, error) { + m.mu.Lock() + defer m.mu.Unlock() + return slices.Sorted(maps.Keys(m.objs)), nil +} + +func (m *memObjects) Fill(context.Context) (int64, int64, error) { + m.mu.Lock() + defer m.mu.Unlock() + return m.used(), m.capacity, nil +} + +var errObjectsFullForTest = &objectsFullError{} + +type objectsFullError struct{} + +func (*objectsFullError) Error() string { return "maximum bytes exceeded" } diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index 0f6515c6..242886a9 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -884,7 +884,9 @@ func (ix *Indexer) dry() *dryRun { // requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is // weighed (Holds and Counted nil hold nothing); ReadFile only reads. Requirements: nil, Findings: nil, SeatDefinitions: nil, ReadFile: ix.ReadFile, SaySeatDefinition: nil, - Holds: nil, Counted: nil, AskEngines: nil, HoldBounds: nil, OwnBuildState: nil, underSeats: true} + Holds: nil, Counted: nil, AskEngines: nil, HoldBounds: nil, OwnBuildState: nil, underSeats: true, + // A dry run keeps no catalog graph revision (catalog_revisions.go): its revision is the take-in's to keep. + Revisions: nil} x.Logf = func(format string, args ...any) { ix.logf("take-in, computed and not written: "+format, args...) } x.Store, d.candidates = over(ix.Store) x.Modules, d.modules = over(ix.Modules) diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index 0d62d0c5..efa3a483 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -970,7 +970,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", "Unjudged", "JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk", "Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", - "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "Holds", "Counted", "AskEngines", "HoldBounds", "OpenTries", "TryRecovery", "OwnBuildState"} { + "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "Holds", "Counted", "AskEngines", "HoldBounds", "OpenTries", "TryRecovery", "OwnBuildState", "Revisions"} { named[f] = true } ty := reflect.TypeOf(Indexer{}) diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 804983ee..73f7a37d 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -311,6 +311,27 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // t.Errorf("entry %s: digest %s against %s before; only mesh-tools' moves", ref.Module, ref.Digest, rev.Entries[i].Digest) } } + // W4.2's store: both revisions kept in catalog-revisions hold one object per entry once, the moved entry twice, and + // each list; a revision read back is its entries whole; an entry the bucket lost is an error, never left out. + kept := newMemObjects(0) + _, nextObjects, _ := CatalogRevisionOf(next, moved) + if err := KeepRevision(context.Background(), kept, rev, objects); err != nil { + t.Fatal(err) + } + if err := KeepRevision(context.Background(), kept, next, nextObjects); err != nil { + t.Fatal(err) + } + if names, _ := kept.Names(context.Background()); len(names) != len(mods.vals)+1+2 || kept.puts != len(names) { + t.Errorf("catalog-revisions holds %v after %d puts, want each entry once, the moved one twice, and two lists", names, kept.puts) + } + if back, got, err := ReadRevision(context.Background(), kept, "trunk.2"); err != nil || back.Source != 8 || len(got) != len(mods.vals) || + !slices.ContainsFunc(got, func(e meshcatalog.CatalogEntry) bool { return e.Module == "mesh-tools" && e.Fp == tools.Fp }) { + t.Errorf("trunk.2 read back = %+v, %d entries, %v", back, len(got), err) + } + _ = kept.Delete(context.Background(), EntryObject(next.Entries[0].Digest)) + if _, _, err := ReadRevision(context.Background(), kept, "trunk.2"); err == nil || !strings.Contains(err.Error(), next.Entries[0].Module) { + t.Errorf("trunk.2 with a lost entry read without error naming it: %v", err) + } graph := ix.Graph.(*memStore) written := graph.writes if err := ix.Recompute(context.Background()); err != nil || graph.writes != written { diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index cb7db957..34b28b91 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -57,7 +57,11 @@ type manifest struct { Slug string `json:"slug"` RunsAs string `json:"runs-as"` Streams []stream `json:"streams"` - Claims []struct { + Objects []struct { + Name string `json:"name"` + MaxMiB int `json:"max-mib"` + } `json:"objects"` + Claims []struct { Name string `json:"name"` Scope string `json:"scope"` Serves []string `json:"serves"` @@ -119,6 +123,11 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build {Name: index.HoldsState, History: 5}, {Name: index.EngineReadsState, History: 1}, {Name: index.HoldBoundsState, History: 1, KeyTTL: true}, {Name: index.PendingState, History: 1}, {Name: index.UnjudgedState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17, ADR 0632 §2 t.Errorf("state = %+v, want %+v", m.State, want) } + // Milestone 47/G W4.2 (design 47, Two graphs; ADR 0619 §2): the catalog graph revisions are objects in the old section, + // capped at 512 MiB, under the local name the indexer opens. + if len(m.Objects) != 1 || m.Objects[0].Name != index.CatalogRevisionsObjects || m.Objects[0].MaxMiB != 512 { + t.Errorf("objects = %+v, want %s capped at 512 MiB", m.Objects, index.CatalogRevisionsObjects) + } // D-11 (ADR 0634 §1): the holders and buckets the indexer names (internal/index/names.go) are its manifest's: its own // name and states with their revisions, and the states it reads, each as `.`. if index.Module != m.Module { diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 0392d211..b56cb192 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -99,6 +99,9 @@ { "name": "pending", "history": 1 }, { "name": "unjudged", "history": 1 } ], + "objects": [ + { "name": "catalog-revisions", "max-mib": 512 } + ], "emits": [ "take-in.pending", "take-in.taken", From bd067ae7793ef948d16044ebdcb659f2d5453ef9 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 9 Nov 2026 20:27:32 +0100 Subject: [PATCH 03/15] The indexer keeps a catalog graph revision at each merge and each moved fingerprint, and says catalog-revision (milestone 47/G W4.3, its writing part) Design 47 (Two graphs) keeps a catalog graph revision for each source revision taken in and for a fingerprint moved with no merge. The recompute that writes the module graph now keeps the revision it makes, numbered after the newest kept trunk revision, and says the seat mesh-catalog's event catalog-revision naming the changed modules; a recompute that moves no entry keeps none, and objects not yet on the bus keep none, said in the log. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-indexer/main.go | 1 + .../cmd/mesh-delivery-indexer/objects.go | 32 ++++++- .../internal/index/candidates.go | 2 + .../internal/index/catalog_revisions.go | 96 +++++++++++++++++++ .../internal/index/graph.go | 37 +++---- .../internal/index/pending.go | 2 +- .../internal/index/pending_test.go | 2 +- .../internal/index/toolchains.go | 5 +- .../internal/index/toolchains_test.go | 22 ++++- 9 files changed, 177 insertions(+), 22 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index bf432663..5d60a336 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -435,6 +435,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), OwnBuildState: awaitedStore{b: ownBuildState}, TryRecovery: awaitedStore{b: tryRecovery}, Graph: awaitedStore{b: graph}, SeatDefinitions: awaitedStore{b: seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats, Message: messageVia(calls.CallKept), Revisions: &busObjects{js: js, local: index.CatalogRevisionsObjects}, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1) + SayCatalogRevision: sayCatalogRevision(conn, cred), // The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear; // until then every merge is taken in at once, as before. Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.PendingBucket, "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)}, diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go index abc08a3c..bb6ae703 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go @@ -3,13 +3,19 @@ package main import ( "bytes" "context" + "encoding/json" "errors" "fmt" "sync" + "time" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-indexer/internal/index" ) @@ -34,8 +40,11 @@ func (b *busObjects) open(ctx context.Context) (jetstream.ObjectStore, error) { return b.o, nil } o, err := b.js.ObjectStore(ctx, catalogue.ObjectsBucket(indexerModule, b.local)) + if errors.Is(err, jetstream.ErrBucketNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) { + return nil, fmt.Errorf("the object bucket %s is not on the bus yet (the grants part creates it from the manifest's objects): %w", b.local, process.ErrNotYet) + } if err != nil { - return nil, fmt.Errorf("the object bucket %s is not open (the grants part creates it from the manifest's objects): %w", b.local, err) + return nil, fmt.Errorf("the object bucket %s is not open: %w", b.local, err) } b.o = o return o, nil @@ -107,3 +116,24 @@ func (b *busObjects) Fill(ctx context.Context) (int64, int64, error) { } return int64(st.Size()), info.CachedInfo().Config.MaxBytes, nil } + +// sayCatalogRevision publishes the seat mesh-catalog's event catalog-revision under the duplicate id of its kind and +// number, so a revision said twice is heard once (design 47, Two graphs). +func sayCatalogRevision(conn *nats.Conn, cred process.Account) func(context.Context, meshcatalog.CatalogRevisionEvent) error { + return func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error { + data, err := json.Marshal(e) + if err != nil { + return err + } + msg := nats.NewMsg(seatEventSubject(meshcatalog.EventCatalogRevision)) + id := fmt.Sprintf("%s-%s-%d%s", meshcatalog.EventCatalogRevision, e.Kind, e.Revision, e.Heads) + msg.Header.Set("x-event-id", id) + msg.Header.Set("x-source", cred.Module) + msg.Header.Set("x-node", cred.Node) + msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano)) + msg.Header.Set("content-type", "application/json") + msg.Header.Set(nats.MsgIdHdr, id) + msg.Data = data + return conn.PublishMsg(msg) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 8e9c5b81..57d8f35d 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -100,6 +100,8 @@ type Indexer struct { // Revisions is the object bucket catalog-revisions (catalog_revisions.go; design 47, Two graphs); nil keeps no // catalog graph revision. Revisions Objects + // SayCatalogRevision says the seat mesh-catalog's event catalog-revision for a revision kept; nil says nothing. + SayCatalogRevision func(ctx context.Context, e meshcatalog.CatalogRevisionEvent) error // Graph is the module graph's state (GraphBucket, graph.go; novox/hq ADR 0603 §2); nil writes no graph. Graph Store // Files asks the forge for every blob under each directory at a commit, by directory then path relative to it, each diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index 3a457422..ccb52f58 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -5,10 +5,13 @@ import ( "encoding/json" "errors" "fmt" + "slices" "strconv" "strings" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/sdk/graph" + "git.novox.be/novox/mesh-sdk/go/process" ) // The indexer's object bucket catalog-revisions (design 47, Two graphs; novox/hq ADR 0619 §2: declared in the old @@ -118,3 +121,96 @@ func ReadRevision(ctx context.Context, o Objects, name string) (meshcatalog.Cata } return rev, entries, nil } + +// newestTrunk is the newest trunk revision the bucket keeps, its list only; none is a zero revision and false. +func newestTrunk(ctx context.Context, o Objects) (meshcatalog.CatalogRevision, bool, error) { + names, err := o.Names(ctx) + if err != nil { + return meshcatalog.CatalogRevision{}, false, err + } + var newest int64 + for _, n := range names { + if r, ok := TrunkRevisionOf(n); ok && r > newest { + newest = r + } + } + if newest == 0 { + return meshcatalog.CatalogRevision{}, false, nil + } + data, err := o.Get(ctx, "trunk."+strconv.FormatInt(newest, 10)) + if err != nil { + return meshcatalog.CatalogRevision{}, false, err + } + var rev meshcatalog.CatalogRevision + if err := json.Unmarshal(data, &rev); err != nil { + return rev, false, fmt.Errorf("trunk.%d: %w", newest, err) + } + return rev, true, nil +} + +// changedEntries are the modules whose entry differs between two revisions' lists, one gone or added included, by name. +func changedEntries(before, after []meshcatalog.EntryRef) []string { + was := map[string]string{} + for _, r := range before { + was[r.Module] = r.Digest + } + var changed []string + for _, r := range after { + if d, ok := was[r.Module]; !ok || d != r.Digest { + changed = append(changed, r.Module) + } + delete(was, r.Module) + } + for m := range was { + changed = append(changed, m) + } + slices.Sort(changed) + return changed +} + +// keepTrunkRevision keeps the catalog graph revision a merge's or a recompute's module graph makes (design 47, Two +// graphs: a revision for each source revision taken in, and for a fingerprint moved with no merge), numbered after the +// newest kept, and says catalog-revision; a graph whose entries are the newest revision's makes none. A revision not +// kept is said in the log and returned, so the take-in that made it is not said taken. +func (ix *Indexer) keepTrunkRevision(ctx context.Context, edges map[string]graphEdges, whole graph.Graph) error { + if ix.Revisions == nil { + return nil + } + entries := make(map[string]meshcatalog.CatalogEntry, len(edges)) + for name, e := range edges { + entries[name] = CatalogEntryOf(name, e.cur, whole[name]) + } + ix.mu.Lock() + src := int64(ix.source.rev) + ix.mu.Unlock() + newest, any, err := newestTrunk(ctx, ix.Revisions) + if errors.Is(err, process.ErrNotYet) { + ix.logf("catalog graph: no revision is kept: its objects %s are not on the bus yet", CatalogRevisionsObjects) + return nil + } + if err != nil { + ix.logf("catalog graph: no revision is kept: the newest is not read: %v", err) + return fmt.Errorf("the newest catalog graph revision: %w", err) + } + at := ix.now().UTC() + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: newest.Revision + 1, Kind: "trunk", Source: src, At: &at}, entries) + if err != nil { + return err + } + changed := changedEntries(newest.Entries, rev.Entries) + if any && len(changed) == 0 { + return nil + } + if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { + ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err) + return err + } + ix.logf("catalog graph: revision %d of source revision %d kept, %d entries, %d changed", rev.Revision, src, len(rev.Entries), len(changed)) + if ix.SayCatalogRevision != nil { + if err := ix.SayCatalogRevision(ctx, meshcatalog.CatalogRevisionEvent{V: CatalogEntryLayout, Revision: rev.Revision, Kind: rev.Kind, + Source: src, At: &at, Changed: changed, Entries: int64(len(rev.Entries))}); err != nil { + ix.logf("catalog graph: revision %d is not said: %v", rev.Revision, err) + } + } + return nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/graph.go b/modules/mesh-delivery-indexer/internal/index/graph.go index 199f9f7d..2d37097f 100644 --- a/modules/mesh-delivery-indexer/internal/index/graph.go +++ b/modules/mesh-delivery-indexer/internal/index/graph.go @@ -81,22 +81,7 @@ func (ix *Indexer) writeGraph(ctx context.Context, edges map[string]graphEdges) GraphMost, len(names), names[GraphMost], len(names)-GraphMost-1) names = names[:GraphMost] } - providers := func(provision string) []catalogue.Manifest { - var out []catalogue.Manifest - for _, name := range slices.Sorted(maps.Keys(edges)) { - if slices.Contains(edges[name].manifest.Offers(), provision) { - out = append(out, edges[name].manifest) - } - } - return out - } - // The verb edges on a cycle are left out over the whole graph (ADR 0614 §3), every module it knows, so a module past - // GraphMost still closes the cycle it lies on. - whole := graph.Graph{} - for name, e := range edges { - whole[name] = graphNode(e, providers) - } - whole.LeaveOut() + whole := wholeGraph(edges) for _, name := range names { key, err := graph.Key(name) if err != nil { @@ -141,3 +126,23 @@ func (ix *Indexer) writeGraph(ctx context.Context, edges map[string]graphEdges) } return nil } + +// wholeGraph is the module graph of every module of edges, the verb edges on a cycle left out over the whole graph +// (ADR 0614 §3), so a module past GraphMost still closes the cycle it lies on. +func wholeGraph(edges map[string]graphEdges) graph.Graph { + providers := func(provision string) []catalogue.Manifest { + var out []catalogue.Manifest + for _, name := range slices.Sorted(maps.Keys(edges)) { + if slices.Contains(edges[name].manifest.Offers(), provision) { + out = append(out, edges[name].manifest) + } + } + return out + } + whole := graph.Graph{} + for name, e := range edges { + whole[name] = graphNode(e, providers) + } + whole.LeaveOut() + return whole +} diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index 242886a9..4756fd1e 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -886,7 +886,7 @@ func (ix *Indexer) dry() *dryRun { Requirements: nil, Findings: nil, SeatDefinitions: nil, ReadFile: ix.ReadFile, SaySeatDefinition: nil, Holds: nil, Counted: nil, AskEngines: nil, HoldBounds: nil, OwnBuildState: nil, underSeats: true, // A dry run keeps no catalog graph revision (catalog_revisions.go): its revision is the take-in's to keep. - Revisions: nil} + Revisions: nil, SayCatalogRevision: nil} x.Logf = func(format string, args ...any) { ix.logf("take-in, computed and not written: "+format, args...) } x.Store, d.candidates = over(ix.Store) x.Modules, d.modules = over(ix.Modules) diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index efa3a483..6a82b105 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -970,7 +970,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", "Unjudged", "JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk", "Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", - "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "Holds", "Counted", "AskEngines", "HoldBounds", "OpenTries", "TryRecovery", "OwnBuildState", "Revisions"} { + "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "Holds", "Counted", "AskEngines", "HoldBounds", "OpenTries", "TryRecovery", "OwnBuildState", "Revisions", "SayCatalogRevision"} { named[f] = true } ty := reflect.TypeOf(Indexer{}) diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 2ffb0617..0f8b0f29 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -372,7 +372,10 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc if err := ix.writeOverlay(ctx, overlay, finals, w); err != nil { return err } - return ix.writeGraph(ctx, edges) + if err := ix.writeGraph(ctx, edges); err != nil { + return err + } + return ix.keepTrunkRevision(ctx, edges, wholeGraph(edges)) } // writeOverlay writes the records of a merge, each once and only if it differs from what the bucket holds; finals has diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 73f7a37d..5b9821dc 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -332,16 +332,34 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // if _, _, err := ReadRevision(context.Background(), kept, "trunk.2"); err == nil || !strings.Contains(err.Error(), next.Entries[0].Module) { t.Errorf("trunk.2 with a lost entry read without error naming it: %v", err) } + // W4.3: a recompute keeps a trunk revision of the module graph and says catalog-revision; one that moves no entry keeps + // none; a module leaving makes the next revision, naming it changed. + ix.Revisions = newMemObjects(0) + var revisions []meshcatalog.CatalogRevisionEvent + ix.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error { + revisions = append(revisions, e) + return nil + } + if err := ix.Recompute(context.Background()); err != nil || len(revisions) != 1 || revisions[0].Revision != 1 || revisions[0].Kind != "trunk" || + revisions[0].Entries != int64(len(mods.vals)) || len(revisions[0].Changed) != len(mods.vals) { + t.Fatalf("the first recompute said %+v (%v), want trunk revision 1 with every module", revisions, err) + } graph := ix.Graph.(*memStore) written := graph.writes - if err := ix.Recompute(context.Background()); err != nil || graph.writes != written { - t.Errorf("a recompute that moves nothing wrote the graph %d times (%v)", graph.writes-written, err) + if err := ix.Recompute(context.Background()); err != nil || graph.writes != written || len(revisions) != 1 { + t.Errorf("a recompute that moves nothing wrote the graph %d times and said %d revisions (%v)", graph.writes-written, len(revisions), err) } // A module the modules state no longer holds (a merge dropped it, issue 1900) leaves the graph at the next recompute. delete(mods.vals, "other") if err := ix.Recompute(context.Background()); err != nil { t.Fatal(err) } + if len(revisions) != 2 || revisions[1].Revision != 2 || !reflect.DeepEqual(revisions[1].Changed, []string{"other"}) { + t.Errorf("after other left, said %+v, want revision 2 naming other changed", revisions) + } + if _, kept, err := ReadRevision(context.Background(), ix.Revisions, "trunk.2"); err != nil || len(kept) != len(mods.vals) { + t.Errorf("trunk.2 holds %d entries (%v), want %d", len(kept), err, len(mods.vals)) + } if _, held := graph.vals["other"]; held { t.Error("graph.other is still held after its module went") } From 6ff7aeb5cf75b9f4f22c044bb13c25da4221ab29 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 9 Nov 2026 20:30:01 +0100 Subject: [PATCH 04/15] The indexer fills the catalog graph entry's main from the module's manifest (milestone 47/G W3.1's main, W4.3) The entry's main is the from of the Go artifact the module's process runs, which the check's first tier builds when the module acts. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../internal/index/catalog_graph.go | 29 +++++++++++++++++-- .../internal/index/catalog_revisions.go | 2 +- .../internal/index/toolchains_test.go | 13 ++++++++- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_graph.go b/modules/mesh-delivery-indexer/internal/index/catalog_graph.go index c87d0084..eed3191a 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_graph.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_graph.go @@ -10,6 +10,7 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/sdk/graph" + "git.novox.be/novox/mesh-sdk/go/catalogue" ) // The catalog graph (novox/hq ADR 0656; design 47, *Two graphs, one source of truth*): at each revision one entry per @@ -22,11 +23,12 @@ const CatalogEntryLayout = 1 // CatalogEntryOf is the catalog graph entry of one module: the module graph's node of it as a recompute computed it // (its build, provision and verb edges, the verb edges left out on a cycle, ADR 0614), with where the trunk holds it and -// its manifest's digest. The verdict and the acting rules are the revision's to add (they are not the node's). -func CatalogEntryOf(module string, cur meshcatalog.ModulesByModuleValue, n graph.Node) meshcatalog.CatalogEntry { +// its manifest's digest and the main package its process runs (MainOf). The verdict and the acting rules are the revision's to add (they are not the node's). +func CatalogEntryOf(module string, cur meshcatalog.ModulesByModuleValue, m catalogue.Manifest, n graph.Node) meshcatalog.CatalogEntry { e := meshcatalog.CatalogEntry{V: CatalogEntryLayout, Module: module, Repository: cur.Repository, Path: cur.Path, Commit: cur.Commit, Manifest: cur.Manifest, Fp: n.FP, None: n.None, Bases: n.Bases, Packages: n.Packages, - Provides: n.Provides, Requires: n.Requires, Awaits: n.Awaits, Invokes: n.Invokes, Serves: n.Serves, LeftOut: n.LeftOut} + Provides: n.Provides, Requires: n.Requires, Awaits: n.Awaits, Invokes: n.Invokes, Serves: n.Serves, LeftOut: n.LeftOut, + Main: MainOf(m)} for language, t := range n.Toolchains { if e.Toolchains == nil { e.Toolchains = map[string]meshcatalog.BuildEdge{} @@ -63,3 +65,24 @@ func CatalogRevisionOf(rev meshcatalog.CatalogRevision, entries map[string]meshc } return rev, objects, nil } + +// MainOf is the Go main package the check's first tier builds when the module acts (the seat mesh-catalog's entry +// field main): the from of the first Go artifact a process resource of the manifest runs, as the manifest names it; +// empty when it runs none. +func MainOf(m catalogue.Manifest) string { + if m.Build == nil { + return "" + } + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) != "process" { + continue + } + name := fmt.Sprint(r["artifact"]) + for _, a := range m.Build.Artifacts { + if a.Name == name && a.Language == "go" && a.From != "" { + return a.From + } + } + } + return "" +} diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index ccb52f58..cab54e63 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -178,7 +178,7 @@ func (ix *Indexer) keepTrunkRevision(ctx context.Context, edges map[string]graph } entries := make(map[string]meshcatalog.CatalogEntry, len(edges)) for name, e := range edges { - entries[name] = CatalogEntryOf(name, e.cur, whole[name]) + entries[name] = CatalogEntryOf(name, e.cur, e.manifest, whole[name]) } ix.mu.Lock() src := int64(ix.source.rev) diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 5b9821dc..db367a8c 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "maps" + "os" "reflect" "slices" "strings" @@ -18,6 +19,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/process" @@ -291,7 +293,16 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // entries := map[string]meshcatalog.CatalogEntry{} for name := range mods.vals { m, _ := moduleOf(t, mods, name) - entries[name] = CatalogEntryOf(name, m, graphOf(t, ix, name)) + entries[name] = CatalogEntryOf(name, m, catalogue.Manifest{}, graphOf(t, ix, name)) + } + // W3.1's main: the Go main package the module's process runs, read from its manifest (here the indexer's own); a + // manifest that runs no process names none. + own, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + if m, err := catalogue.Decode(own); err != nil || MainOf(m) != "cmd/mesh-delivery-indexer" || MainOf(catalogue.Manifest{}) != "" { + t.Errorf("the indexer's main = %q (%v), want cmd/mesh-delivery-indexer", MainOf(m), err) } if e := entries["avahi"]; e.Fp != want["avahi"] || e.Commit == "" || e.Manifest == "" || !reflect.DeepEqual(e.Bases, []string{"mesh-tools"}) || e.Toolchains["go"] != (meshcatalog.BuildEdge{Module: "mesh-build-go", Version: "go1.27.1", Recipe: "1"}) { From 9ab20f14f82cf2bc91769b7950ae1c84a0edbbeb Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 9 Nov 2026 20:38:23 +0100 Subject: [PATCH 05/15] The catalog graph entry carries what the next revision is computed from and its package versions (milestone 47/G W3.1, inputs) The lead's ruling on finding 1: an entry gains inputs (what its fingerprint is computed from, the toolchains used, problems, warnings, named paths, the source revisions it was taken in and its fingerprint moved at) and packages becomes name to version, so the indexer's view of the trunk can be read back from the newest trunk revision alone (ModuleValueOf). A revision's entries are the values the recompute wrote, not the values it started from. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../internal/index/catalog_graph.go | 41 ++++++++++++++++--- .../internal/index/catalog_revisions.go | 12 +++++- .../internal/index/toolchains_test.go | 13 ++++++ 3 files changed, 60 insertions(+), 6 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_graph.go b/modules/mesh-delivery-indexer/internal/index/catalog_graph.go index eed3191a..a9fd8e87 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_graph.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_graph.go @@ -6,11 +6,13 @@ import ( "encoding/json" "fmt" "maps" + "reflect" "slices" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/sdk/graph" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/keys" ) // The catalog graph (novox/hq ADR 0656; design 47, *Two graphs, one source of truth*): at each revision one entry per @@ -22,13 +24,24 @@ import ( const CatalogEntryLayout = 1 // CatalogEntryOf is the catalog graph entry of one module: the module graph's node of it as a recompute computed it -// (its build, provision and verb edges, the verb edges left out on a cycle, ADR 0614), with where the trunk holds it and -// its manifest's digest and the main package its process runs (MainOf). The verdict and the acting rules are the revision's to add (they are not the node's). +// (its build, provision and verb edges, the verb edges left out on a cycle, ADR 0614), with where the trunk holds it, +// its manifest's digest and verdict, the main package its process runs (MainOf), the package versions it builds against +// and what the indexer computes its next revision from (Inputs: the lead's ruling on milestone 47/G finding 1), so that +// ModuleValueOf gives the module's value back from the entry alone. The acting rules are the revision's to add. func CatalogEntryOf(module string, cur meshcatalog.ModulesByModuleValue, m catalogue.Manifest, n graph.Node) meshcatalog.CatalogEntry { e := meshcatalog.CatalogEntry{V: CatalogEntryLayout, Module: module, Repository: cur.Repository, Path: cur.Path, - Commit: cur.Commit, Manifest: cur.Manifest, Fp: n.FP, None: n.None, Bases: n.Bases, Packages: n.Packages, - Provides: n.Provides, Requires: n.Requires, Awaits: n.Awaits, Invokes: n.Invokes, Serves: n.Serves, LeftOut: n.LeftOut, - Main: MainOf(m)} + Commit: cur.Commit, Manifest: cur.Manifest, Verdict: cur.Verdict, Fp: n.FP, None: n.None, Bases: n.Bases, + Packages: maps.Clone(cur.Packages), Provides: n.Provides, Requires: n.Requires, Awaits: n.Awaits, Invokes: n.Invokes, + Serves: n.Serves, LeftOut: n.LeftOut, Main: MainOf(m)} + in := meshcatalog.EntryInputs{Toolchains: maps.Clone(cur.Toolchains), Paths: maps.Clone(cur.Paths), Problems: cur.Problems, + Warnings: cur.Warnings, Unknown: cur.Unknown, Src: cur.Src, FpSrc: cur.FpSrc} + if cur.Inputs2 != nil { + from := meshcatalog.FingerprintInputs(*cur.Inputs2) + in.From = &from + } + if !reflect.ValueOf(in).IsZero() { + e.Inputs = &in + } for language, t := range n.Toolchains { if e.Toolchains == nil { e.Toolchains = map[string]meshcatalog.BuildEdge{} @@ -38,6 +51,24 @@ func CatalogEntryOf(module string, cur meshcatalog.ModulesByModuleValue, m catal return e } +// ModuleValueOf is the module's value as the indexer computes with it, read back from its catalog graph entry: the +// inverse of CatalogEntryOf, by which the indexer's view of the trunk is the newest trunk revision's entries (the lead's +// ruling on milestone 47/G finding 1; ADR 0656 §6: one form of the fact). The old line's fields (the old-form +// fingerprint, the reader needs) are not kept. +func ModuleValueOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue { + v := meshcatalog.ModulesByModuleValue{V: keys.Version, Repository: e.Repository, Path: e.Path, Commit: e.Commit, Manifest: e.Manifest, + Verdict: e.Verdict, Fp2: e.Fp, None2: e.None, Packages: maps.Clone(e.Packages)} + if in := e.Inputs; in != nil { + v.Toolchains, v.Paths, v.Problems, v.Warnings, v.Unknown = maps.Clone(in.Toolchains), maps.Clone(in.Paths), in.Problems, in.Warnings, in.Unknown + v.Src, v.FpSrc = in.Src, in.FpSrc + if in.From != nil { + from := meshcatalog.ModulesByModuleValueInputs2(*in.From) + v.Inputs2 = &from + } + } + return v +} + // EncodeEntry is an entry's bytes and their sha256 in hex, the name of its object: the same entry in two revisions is // one object (design 47, *An unchanged entry is stored once*). func EncodeEntry(e meshcatalog.CatalogEntry) (string, []byte, error) { diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index cab54e63..256e16ff 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -176,9 +176,19 @@ func (ix *Indexer) keepTrunkRevision(ctx context.Context, edges map[string]graph if ix.Revisions == nil { return nil } + // Each entry is the module's value as this recompute left it in modules (its fingerprint, toolchains, package versions + // and inputs written), so the revision holds what the next one is computed from. + held, err := ix.heldModules(ctx) + if err != nil { + return err + } entries := make(map[string]meshcatalog.CatalogEntry, len(edges)) for name, e := range edges { - entries[name] = CatalogEntryOf(name, e.cur, e.manifest, whole[name]) + cur, ok := held[name] + if !ok { + cur = e.cur + } + entries[name] = CatalogEntryOf(name, cur, e.manifest, whole[name]) } ix.mu.Lock() src := int64(ix.source.rev) diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index db367a8c..8d1d2753 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -308,6 +308,19 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // e.Toolchains["go"] != (meshcatalog.BuildEdge{Module: "mesh-build-go", Version: "go1.27.1", Recipe: "1"}) { t.Errorf("the catalog entry of avahi = %+v", e) } + // W3.1's inputs (the lead's ruling on finding 1): an entry carries what the next revision is computed from (the + // fingerprint's inputs, the toolchains used, the package versions), so the module's value read back from it is the + // value modules held, but for the old line's fields. + for name := range mods.vals { + m, _ := moduleOf(t, mods, name) + m.Fp, m.None, m.Needs = "", "", nil + if back := ModuleValueOf(entries[name]); !reflect.DeepEqual(back, m) { + t.Errorf("%s read back from its entry = %+v, want %+v", name, back, m) + } + } + if e := entries["avahi"]; e.Inputs == nil || e.Inputs.From == nil || e.Inputs.Toolchains["go"].Version != "go1.27.1" { + t.Errorf("the catalog entry of avahi carries inputs %+v, want what its fingerprint is computed from", e.Inputs) + } rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk", Source: 7}, entries) if err != nil || len(rev.Entries) != len(mods.vals) || len(objects) != len(mods.vals) || rev.V != CatalogEntryLayout { t.Fatalf("revision %+v with %d objects (%v), want one entry per module of %d", rev, len(objects), err, len(mods.vals)) From e0d54881389b09bb4635f3233af084762454ec8a Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 9 Nov 2026 20:39:34 +0100 Subject: [PATCH 06/15] The indexer reads its view of the trunk from the newest trunk revision (milestone 47/G W4.3, second part: the view's read) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The lead's ruling on finding 1: modules becomes an in-memory view of the newest trunk revision read from catalog-revisions at start, one form of the fact (ADR 0656 §6). TrunkView gives that view as a store in memory and the revision it was read at; an empty bucket is an empty view. Not yet wired in main. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../internal/index/catalog_revisions.go | 32 +++++++++++++++++++ .../internal/index/toolchains_test.go | 17 ++++++++++ 2 files changed, 49 insertions(+) diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index 256e16ff..20a88327 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -9,6 +9,7 @@ import ( "strconv" "strings" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/sdk/graph" "git.novox.be/novox/mesh-sdk/go/process" @@ -224,3 +225,34 @@ func (ix *Indexer) keepTrunkRevision(ctx context.Context, edges map[string]graph } return nil } + +// TrunkView is the indexer's view of the trunk read from the newest trunk revision the bucket keeps (the lead's ruling +// on milestone 47/G finding 1; ADR 0656 §6, one form of the fact): a store in memory holding each module's value as +// ModuleValueOf reads it back from its entry, under its modules key, and the revision it was read at (0 when the bucket +// keeps none yet: the view starts empty and the first merge taken in fills it). A lost entry is an error naming the +// module (ReadRevision), never a module left out of the view. +func TrunkView(ctx context.Context, o Objects) (Store, int64, error) { + view := newOverlay(nil) + newest, any, err := newestTrunk(ctx, o) + if err != nil || !any { + return view, 0, err + } + _, entries, err := ReadRevision(ctx, o, RevisionObject(newest)) + if err != nil { + return nil, 0, err + } + for _, e := range entries { + key, err := meshcatalogclient.ModulesKey(e.Module) + if err != nil { + return nil, 0, fmt.Errorf("trunk.%d: the entry of %q: %w", newest.Revision, e.Module, err) + } + data, err := meshcatalogclient.EncodeModule(ModuleValueOf(e)) + if err != nil { + return nil, 0, fmt.Errorf("trunk.%d: the entry of %s: %w", newest.Revision, e.Module, err) + } + if err := view.Create(ctx, key, data); err != nil { + return nil, 0, fmt.Errorf("trunk.%d: the entry of %s: %w", newest.Revision, e.Module, err) + } + } + return view, newest.Revision, nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 8d1d2753..e3abfa5e 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -352,6 +352,23 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // !slices.ContainsFunc(got, func(e meshcatalog.CatalogEntry) bool { return e.Module == "mesh-tools" && e.Fp == tools.Fp }) { t.Errorf("trunk.2 read back = %+v, %d entries, %v", back, len(got), err) } + // W4.3's second part: the view of the trunk read at start is the newest trunk revision, each module's value as its + // entry gives it back; a bucket that keeps none is an empty view at 0. + view, at, err := TrunkView(context.Background(), kept) + if err != nil || at != 2 { + t.Fatalf("the view is read at %d (%v), want 2", at, err) + } + for name, e := range moved { + key, _ := meshcatalogclient.ModulesKey(name) + raw, _, found, _ := view.Get(context.Background(), key) + got, err := meshcatalogclient.DecodeModule(raw) + if want := ModuleValueOf(e); !found || err != nil || !reflect.DeepEqual(got, want) { + t.Errorf("the view's %s = %+v (%v), want %+v", name, got, err, want) + } + } + if empty, at, err := TrunkView(context.Background(), newMemObjects(0)); err != nil || at != 0 || func() bool { k, _ := empty.Keys(context.Background()); return len(k) != 0 }() { + t.Errorf("an empty bucket's view is read at %d (%v), want an empty view at 0", at, err) + } _ = kept.Delete(context.Background(), EntryObject(next.Entries[0].Digest)) if _, _, err := ReadRevision(context.Background(), kept, "trunk.2"); err == nil || !strings.Contains(err.Error(), next.Entries[0].Module) { t.Errorf("trunk.2 with a lost entry read without error naming it: %v", err) From 80bc3a408464436822307b7e08acc6fdee53abbd Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 9 Nov 2026 20:56:35 +0100 Subject: [PATCH 07/15] The indexer runs on its view of the trunk read from the newest catalog graph revision, keeps it after every write and takes the trunk in at a first start (milestone 47/G W4.3, the view wired) main.go reads the view and the module graph's nodes (NodeOf) from the newest trunk revision instead of the modules and graph states; every write outside a recompute (the judge pass's verdicts, a build's imports, the source stamp, the modules the trunk dropped, the indexer's own entry a held revision writes) ends in KeepView, so a restart loses nothing; a revision lists every module the view holds, one whose manifest cannot be read too. At a first start whose objects keep no trunk revision the indexer takes in the head the gate's source names for every repository by the forge bridge's merged message and keeps revision 1 (the lead's ruling A). The whole-module tests read the view from the revisions. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-delivery-indexer/integration_test.go | 103 ++++++++++++++--- .../cmd/mesh-delivery-indexer/main.go | 31 +++-- .../internal/index/candidates.go | 15 ++- .../internal/index/catalog_graph.go | 22 ++++ .../internal/index/catalog_revisions.go | 109 ++++++++++++++---- .../internal/index/imports.go | 7 ++ .../internal/index/modules.go | 7 ++ .../internal/index/pending.go | 5 + .../internal/index/source.go | 90 +++++++++++++++ .../internal/index/toolchains_test.go | 77 ++++++++++++- .../internal/index/verdicts.go | 4 + 11 files changed, 417 insertions(+), 53 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index 2d127854..efafb1e5 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -39,6 +39,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-delivery/sdk/graph" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-indexer/internal/index" @@ -86,7 +87,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 }, func(ctx context.Context, js jetstream.JetStream) { // The states and the stream the controller makes from the indexer's manifest (old sections, ADR 0595 §11). for bucket, history := range map[string]uint8{index.PendingBucket: 1, index.VerdictsBucket: index.VerdictsHistory, - index.GraphBucket: 1, index.TakenBucket: 1, index.PackagesBucket: 1, + index.TakenBucket: 1, index.PackagesBucket: 1, index.SeatDefinitionsBucket: 1, index.RequirementsBucket: 1, findingsBucket: 1, index.HoldsBucket: 1, index.UnjudgedBucket: 1} { if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket, History: history}); err != nil { t.Fatal(err) @@ -262,6 +263,13 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 named := held.WaitsFor get := func(bucket, key string) ([]byte, bool) { t.Helper() + if bucket == index.ModulesBucket { + e, err := revisionView{js: js}.Get(ctx, key) + if err != nil { + return nil, false + } + return e.Value(), true + } kv, err := js.KeyValue(ctx, bucket) if err != nil { t.Fatal(err) @@ -284,9 +292,6 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 if _, ok := get(index.VerdictsBucket, own.Manifest); !ok { t.Errorf("the verdict on the indexer's manifest %s is not written", own.Manifest) } - if gk, _ := graph.Key(ix); !func() bool { _, ok := get(index.GraphBucket, gk); return ok }() { - t.Errorf("the indexer's key %s of the module graph is not written", gk) - } repoKey, _ := trunk.RepositoryKey(m1.Repository) if _, ok := get(index.ModulesBucket, "a"); ok { t.Errorf("modules.a is written while the revision is pending") @@ -471,7 +476,75 @@ var acceptedHeard atomic.Int64 // startIndexerRun starts the bus, the stream FORGE with the indexer's durable consumer, the buckets, lets put fill // mesh-controller_builds, and runs the program; restart starts it again. mesh-delivery-gate_source is never created. -func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue, func()) { +// revisionView reads the indexer's view of the trunk as the indexer itself does at a restart (milestone 47/G): the newest +// trunk revision its objects catalog-revisions keep, a module by its key. +type revisionView struct{ js jetstream.JetStream } + +func (v revisionView) Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error) { + view, _, _, err := index.TrunkView(ctx, &busObjects{js: v.js, local: index.CatalogRevisionsObjects}) + if err != nil { + return nil, err + } + value, rev, found, err := view.Get(ctx, key) + if err != nil { + return nil, err + } + if !found { + return nil, jetstream.ErrKeyNotFound + } + return viewEntry{key: key, value: value, revision: rev}, nil +} + +// History is the module's value in every trunk revision the objects keep that holds it, oldest first. +func (v revisionView) History(ctx context.Context, key string) ([]jetstream.KeyValueEntry, error) { + o := &busObjects{js: v.js, local: index.CatalogRevisionsObjects} + names, err := o.Names(ctx) + if err != nil { + return nil, err + } + var revs []int64 + for _, n := range names { + if r, ok := index.TrunkRevisionOf(n); ok { + revs = append(revs, r) + } + } + slices.Sort(revs) + var out []jetstream.KeyValueEntry + for _, r := range revs { + _, entries, err := index.ReadRevision(ctx, o, fmt.Sprintf("trunk.%d", r)) + if err != nil { + return nil, err + } + for _, e := range entries { + if e.Module != key { + continue + } + data, err := meshcatalogclient.EncodeModule(index.ModuleValueOf(e)) + if err != nil { + return nil, err + } + out = append(out, viewEntry{key: key, value: data, revision: uint64(r)}) + } + } + return out, nil +} + +// viewEntry is one module's value of the view. +type viewEntry struct { + key string + value []byte + revision uint64 +} + +func (e viewEntry) Bucket() string { return index.CatalogRevisionsObjects } +func (e viewEntry) Key() string { return e.key } +func (e viewEntry) Value() []byte { return e.value } +func (e viewEntry) Revision() uint64 { return e.revision } +func (e viewEntry) Created() time.Time { return time.Time{} } +func (e viewEntry) Delta() uint64 { return 0 } +func (e viewEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut } + +func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, revisionView, func()) { dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i") if err != nil { t.Fatal(err) @@ -526,10 +599,11 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before . if err != nil { t.Fatal(err) } - modules, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ModulesBucket, History: index.ModulesHistory}) - if err != nil { + // The indexer's objects catalog-revisions, which the grants part creates from its manifest's old objects section. + if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: catalogue.ObjectsBucket(indexerModule, index.CatalogRevisionsObjects)}); err != nil { t.Fatal(err) } + modules := revisionView{js: js} // The bucket of the manifests, made by the controller from the manifest's state entry manifests (history 1, issue 1449). if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ManifestsBucket, History: index.ManifestsHistory}); err != nil { t.Fatal(err) @@ -583,7 +657,7 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before . } // startIndexer is startIndexerRun for a test that never starts the program again. -func startIndexer(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue) { +func startIndexer(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, revisionView) { js, ctx, candidates, modules, _ := startIndexerRun(t, put, before...) return js, ctx, candidates, modules } @@ -960,17 +1034,16 @@ func TestIndexerJudgesAnUnreadableVerdictAgainAtStart(t *testing.T) { // 1923/AC if _, err := manifests.Put(ctx, mkey, mdata); err != nil { t.Fatal(err) } - stale, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "89abcdef0123456789abcdef0123456789abcdef", Path: "modules/avahi", + // The view the indexer starts from is the newest trunk revision its objects keep (milestone 47/G). + stale := meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "89abcdef0123456789abcdef0123456789abcdef", Path: "modules/avahi", Manifest: digest, None2: "the manifest cannot be read: claims node-build-go, which is a seat in the mesh's own namespace that it does not define", - Inputs2: &meshcatalog.ModulesByModuleValueInputs2{Tree: "tree-avahi", Locks: map[string]string{"go.sum": goSum}}}) + Inputs2: &meshcatalog.ModulesByModuleValueInputs2{Tree: "tree-avahi", Locks: map[string]string{"go.sum": goSum}}} + rev, objects, err := index.CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"}, + map[string]meshcatalog.CatalogEntry{"avahi": index.CatalogEntryOf("avahi", stale, catalogue.Manifest{}, graph.Node{None: stale.None2})}) if err != nil { t.Fatal(err) } - held, err := js.KeyValue(ctx, index.ModulesBucket) - if err != nil { - t.Fatal(err) - } - if _, err := held.Put(ctx, "avahi", stale); err != nil { + if err := index.KeepRevision(ctx, &busObjects{js: js, local: index.CatalogRevisionsObjects}, rev, objects); err != nil { t.Fatal(err) } kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ToolchainsBucket("mesh-build-go"), History: 1}) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 5d60a336..610725e7 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -375,9 +375,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - modules, err := process.EssentialBucket(ctx, js, "indexer", index.ModulesBucket, "write a module's fingerprint") - if err != nil { - return err + // The view of the trunk is the newest trunk revision of the catalog graph the objects keep (milestone 47/G, ADR 0656 §6: + // one form of the fact), read once here and kept as a revision after every write; objects not on the bus yet, or none + // kept, start it empty, and the first start's catch-up takes the trunk in once the gate's source is read (catchUp). + revisions := &busObjects{js: js, local: index.CatalogRevisionsObjects} + modules, graph, at, err := index.TrunkView(ctx, revisions) + switch { + case errors.Is(err, process.ErrNotYet): + logger.Printf("the view of the trunk starts empty: %v; the trunk is taken in once they are there and the gate's source is read", err) + modules, graph = index.MemoryStore(), index.MemoryStore() + case err != nil: + return fmt.Errorf("the view of the trunk cannot be read from the newest catalog graph revision: %w", err) + case at == 0: + logger.Printf("the view of the trunk starts empty: %s keeps no trunk revision yet; the trunk is taken in once the gate's source is read", index.CatalogRevisionsObjects) + default: + logger.Printf("the view of the trunk read from trunk revision %d", at) } manifests, err := process.EssentialBucket(ctx, js, "indexer", index.ManifestsBucket, "keep a module's manifest") if err != nil { @@ -413,8 +425,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // What each consumer's go.sum requires of each mesh Go module (DLV-I7): a state this delivery brings, bound when it // appears; until then indexer_requirements answers none, and the builder's floor waits on its answer. requirements := process.AwaitBucket(ctx, js, conn, index.RequirementsBucket, "what consumers require is not kept", logger.Printf) - // The module graph (ADR 0603 §2): a state this delivery brings, bound when it appears; until then no graph is written. - graph := process.AwaitBucket(ctx, js, conn, index.GraphBucket, "no module graph is written", logger.Printf) + // The module graph (ADR 0603 §2) is held in memory, read with the view: each revision's entries carry its nodes. ownBuildState := process.AwaitBucket(ctx, js, conn, index.OwnBuildBucket, "which build this indexer runs is not kept", logger.Printf) // The manifests a take-in wrote and no judge pass judged yet (issue 1966): a state this delivery brings, bound when it // appears; until then a take-in's new manifests are kept for their judging in its memory alone. @@ -427,14 +438,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro seatBus := &seatBus{defs: awaitedSource{ctx, seatDefs}, marks: awaitedSource{ctx, marks}} seatReader, saySeats := followSeatList(seatBus, saySeatDefinition(js, cred), logger.Printf) readWhenBound(seatReader, seatDefs, marks) - ix := &index.Indexer{Findings: awaitedStore{b: findings}, OldSections: awaitedStore{b: oldSections}, Unjudged: awaitedStore{b: unjudged}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known, + ix := &index.Indexer{Findings: awaitedStore{b: findings}, OldSections: awaitedStore{b: oldSections}, Unjudged: awaitedStore{b: unjudged}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known, Verdicts: awaitedStore{b: verdicts}, SayRefused: sayRefused(conn, cred, logger.Printf), SayAccepted: sayAccepted(conn, cred), Predefined: awaitedStore{b: predefined}, ConfigCandidates: awaitedStore{b: configCandidates}, Logf: logger.Printf, Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), ReadOpen: openVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.PackagesBucket}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), - OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), OwnBuildState: awaitedStore{b: ownBuildState}, TryRecovery: awaitedStore{b: tryRecovery}, Graph: awaitedStore{b: graph}, + OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), OwnBuildState: awaitedStore{b: ownBuildState}, TryRecovery: awaitedStore{b: tryRecovery}, Graph: graph, SeatDefinitions: awaitedStore{b: seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats, - Message: messageVia(calls.CallKept), Revisions: &busObjects{js: js, local: index.CatalogRevisionsObjects}, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1) + Message: messageVia(calls.CallKept), Revisions: revisions, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1) SayCatalogRevision: sayCatalogRevision(conn, cred), // The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear; // until then every merge is taken in at once, as before. @@ -728,10 +739,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // said up to; a first view (the tools bundle) answers at that mark. startViews(ctx, js, cred, marks, findings, - map[string]readyOf{"manifests": boundNow(manifests), "modules": boundNow(modules), "candidates": boundNow(candidates), + map[string]readyOf{"manifests": boundNow(manifests), "candidates": boundNow(candidates), index.VerdictsState: awaited(verdicts), index.PredefinedState: awaited(predefined), index.ConfigCandidatesState: awaited(configCandidates), index.PackagesState: awaited(packages), - graphState: awaited(graph), index.HeardState: awaited(heard.b), "taken": awaited(taken), "findings": awaited(findings), + index.HeardState: awaited(heard.b), "taken": awaited(taken), "findings": awaited(findings), index.SeatDefinitionsState: awaited(seatDefs)}, errs, logger.Printf) cc, err := process.HearEvents(cons, func(m jetstream.Msg) { handle(ctx, m, ix, heard, logger) diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 57d8f35d..5d9888dd 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -169,12 +169,15 @@ type Indexer struct { failedAt map[string]string // the head of a repository whose tree lacks a directory asked, by repository: not asked again until the head changes // OwnBuild answers the fingerprint of the build this program runs, from the node-engine's seat; nil knows none. OwnBuild func(ctx context.Context) (string, error) - underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go) - judgeMu sync.Mutex // one judge pass at a time (verdicts.go) - ownMu sync.Mutex // guards ownFP - ownFP string // the fingerprint of the build this program runs, fixed once at the start (FixOwnBuild) - ownFixed bool // FixOwnBuild asked already: it asks once - ownSince time.Time // when the build was asked and not known, with ownWhy + underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go) + judgeMu sync.Mutex // one judge pass at a time (verdicts.go) + ownMu sync.Mutex // guards ownFP + keepMu sync.Mutex // one catalog graph revision kept at a time (keepEntries) + catchingUp atomic.Bool // the first start's catch-up runs: its merges keep no revision until it ends (CatchUp) + catchUpWG sync.WaitGroup // the first start's catch-up in the background (kickCatchUp), for tests to wait on + ownFP string // the fingerprint of the build this program runs, fixed once at the start (FixOwnBuild) + ownFixed bool // FixOwnBuild asked already: it asks once + ownSince time.Time // when the build was asked and not known, with ownWhy ownWhy string // OwnBuildState keeps which build this process knows it runs, or since when and why not (FixOwnBuild); nil keeps none. OwnBuildState Store diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_graph.go b/modules/mesh-delivery-indexer/internal/index/catalog_graph.go index a9fd8e87..d1e6f001 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_graph.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_graph.go @@ -33,6 +33,10 @@ func CatalogEntryOf(module string, cur meshcatalog.ModulesByModuleValue, m catal Commit: cur.Commit, Manifest: cur.Manifest, Verdict: cur.Verdict, Fp: n.FP, None: n.None, Bases: n.Bases, Packages: maps.Clone(cur.Packages), Provides: n.Provides, Requires: n.Requires, Awaits: n.Awaits, Invokes: n.Invokes, Serves: n.Serves, LeftOut: n.LeftOut, Main: MainOf(m)} + if n.FP == "" && n.None == "" { + // A module the recompute computed no node for (its manifest cannot be read) keeps what its value holds. + e.Fp, e.None = cur.Fp2, cur.None2 + } in := meshcatalog.EntryInputs{Toolchains: maps.Clone(cur.Toolchains), Paths: maps.Clone(cur.Paths), Problems: cur.Problems, Warnings: cur.Warnings, Unknown: cur.Unknown, Src: cur.Src, FpSrc: cur.FpSrc} if cur.Inputs2 != nil { @@ -69,6 +73,24 @@ func ModuleValueOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue return v } +// NodeOf is the node of the module graph an entry carries, as CatalogEntryOf took it in: the graph the indexer holds in +// memory is read back from the newest trunk revision at its start (TrunkView), so a revision kept before its first +// recompute loses no edge. +func NodeOf(e meshcatalog.CatalogEntry) graph.Node { + n := graph.Node{V: graph.Version, FP: e.Fp, None: e.None, Repository: e.Repository, Path: e.Path, Bases: e.Bases, + Provides: e.Provides, Requires: e.Requires, Awaits: e.Awaits, Invokes: e.Invokes, Serves: e.Serves, LeftOut: e.LeftOut} + if len(e.Packages) > 0 { + n.Packages = slices.Sorted(maps.Keys(e.Packages)) + } + for language, t := range e.Toolchains { + if n.Toolchains == nil { + n.Toolchains = map[string]graph.Toolchain{} + } + n.Toolchains[language] = graph.Toolchain{Module: t.Module, Version: t.Version, Recipe: t.Recipe} + } + return n +} + // EncodeEntry is an entry's bytes and their sha256 in hex, the name of its object: the same entry in two revisions is // one object (design 47, *An unchanged entry is stored once*). func EncodeEntry(e meshcatalog.CatalogEntry) (string, []byte, error) { diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index 20a88327..2ccdf931 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -12,6 +12,7 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/sdk/graph" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/process" ) @@ -183,14 +184,67 @@ func (ix *Indexer) keepTrunkRevision(ctx context.Context, edges map[string]graph if err != nil { return err } - entries := make(map[string]meshcatalog.CatalogEntry, len(edges)) - for name, e := range edges { - cur, ok := held[name] - if !ok { - cur = e.cur - } - entries[name] = CatalogEntryOf(name, cur, e.manifest, whole[name]) + // Every module the view holds is an entry, one the recompute computed no edges for (a manifest it cannot read) too. + entries := make(map[string]meshcatalog.CatalogEntry, len(held)) + for name, cur := range held { + entries[name] = CatalogEntryOf(name, cur, edges[name].manifest, whole[name]) } + for name, e := range edges { + if _, ok := held[name]; !ok { + entries[name] = CatalogEntryOf(name, e.cur, e.manifest, whole[name]) + } + } + return ix.keepEntries(ctx, entries) +} + +// KeepView keeps the view of the trunk as a trunk revision after a write to it outside a recompute (the lead's ruling on +// milestone 47/G: the judge pass's verdicts, a build's reported imports, the source stamp, the modules the trunk no longer +// holds, the indexer's own entry a held revision writes): each entry is the module's value in the view with its node of +// the module graph in memory, so what the view holds is never lost at a restart. A view whose entries are the newest +// revision's keeps none. +func (ix *Indexer) KeepView(ctx context.Context) error { + if ix.Revisions == nil || ix.Modules == nil || ix.catchingUp.Load() { + return nil + } + held, err := ix.heldModules(ctx) + if err != nil { + return err + } + nodes := graph.Graph{} + if ix.Graph != nil { + keys, err := ix.Graph.Keys(ctx) + if err != nil && !errors.Is(err, process.ErrNotYet) { + return fmt.Errorf("the module graph: %w", err) + } + for _, k := range keys { + if value, _, found, err := ix.Graph.Get(ctx, k); err == nil && found { + if n, err := graph.Decode(value); err == nil { + nodes[k] = n + } + } + } + } + entries := make(map[string]meshcatalog.CatalogEntry, len(held)) + for name, cur := range held { + var m catalogue.Manifest + if cur.Manifest != "" { + if _, read, err := ix.manifestRaw(ctx, cur.Manifest); err == nil { + m = read + } + } + entries[name] = CatalogEntryOf(name, cur, m, nodes[name]) + } + return ix.keepEntries(ctx, entries) +} + +// keepEntries keeps entries as the trunk revision after the newest, says catalog-revision, and keeps none when no entry +// moved; one keep at a time, so two never take the same number. +func (ix *Indexer) keepEntries(ctx context.Context, entries map[string]meshcatalog.CatalogEntry) error { + if ix.catchingUp.Load() { + return nil // the catch-up of a first start keeps its one revision at its end (CatchUp) + } + ix.keepMu.Lock() + defer ix.keepMu.Unlock() ix.mu.Lock() src := int64(ix.source.rev) ix.mu.Unlock() @@ -209,8 +263,8 @@ func (ix *Indexer) keepTrunkRevision(ctx context.Context, edges map[string]graph return err } changed := changedEntries(newest.Entries, rev.Entries) - if any && len(changed) == 0 { - return nil + if (any && len(changed) == 0) || (!any && len(entries) == 0) { + return nil // nothing moved; an empty first revision is never kept, so the first start's catch-up still runs } if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err) @@ -228,31 +282,46 @@ func (ix *Indexer) keepTrunkRevision(ctx context.Context, edges map[string]graph // TrunkView is the indexer's view of the trunk read from the newest trunk revision the bucket keeps (the lead's ruling // on milestone 47/G finding 1; ADR 0656 §6, one form of the fact): a store in memory holding each module's value as -// ModuleValueOf reads it back from its entry, under its modules key, and the revision it was read at (0 when the bucket -// keeps none yet: the view starts empty and the first merge taken in fills it). A lost entry is an error naming the -// module (ReadRevision), never a module left out of the view. -func TrunkView(ctx context.Context, o Objects) (Store, int64, error) { - view := newOverlay(nil) +// ModuleValueOf reads it back from its entry, under its modules key, a store in memory of each module's node of the module +// graph (NodeOf), and the revision it was read at (0 when the bucket keeps none yet: both start empty and the first start's +// catch-up fills them). A lost entry is an error naming the module (ReadRevision), never a module left out of the view. +func TrunkView(ctx context.Context, o Objects) (modules, nodes Store, at int64, err error) { + view, held := newOverlay(nil), newOverlay(nil) newest, any, err := newestTrunk(ctx, o) if err != nil || !any { - return view, 0, err + return view, held, 0, err } _, entries, err := ReadRevision(ctx, o, RevisionObject(newest)) if err != nil { - return nil, 0, err + return nil, nil, 0, err } for _, e := range entries { key, err := meshcatalogclient.ModulesKey(e.Module) if err != nil { - return nil, 0, fmt.Errorf("trunk.%d: the entry of %q: %w", newest.Revision, e.Module, err) + return nil, nil, 0, fmt.Errorf("trunk.%d: the entry of %q: %w", newest.Revision, e.Module, err) } data, err := meshcatalogclient.EncodeModule(ModuleValueOf(e)) if err != nil { - return nil, 0, fmt.Errorf("trunk.%d: the entry of %s: %w", newest.Revision, e.Module, err) + return nil, nil, 0, fmt.Errorf("trunk.%d: the entry of %s: %w", newest.Revision, e.Module, err) } if err := view.Create(ctx, key, data); err != nil { - return nil, 0, fmt.Errorf("trunk.%d: the entry of %s: %w", newest.Revision, e.Module, err) + return nil, nil, 0, fmt.Errorf("trunk.%d: the entry of %s: %w", newest.Revision, e.Module, err) + } + gk, err := graph.Key(e.Module) + if err != nil { + continue // a module the graph cannot name has no node, as writeGraph writes none + } + node, err := graph.Encode(NodeOf(e)) + if err != nil { + return nil, nil, 0, fmt.Errorf("trunk.%d: the node of %s: %w", newest.Revision, e.Module, err) + } + if err := held.Create(ctx, gk, node); err != nil { + return nil, nil, 0, fmt.Errorf("trunk.%d: the node of %s: %w", newest.Revision, e.Module, err) } } - return view, newest.Revision, nil + return view, held, newest.Revision, nil } + +// MemoryStore is a store held in memory alone: the module graph's nodes, which each revision's entries carry, so the +// indexer keeps no state of them on the bus (milestone 47/G). +func MemoryStore() Store { return newOverlay(nil) } diff --git a/modules/mesh-delivery-indexer/internal/index/imports.go b/modules/mesh-delivery-indexer/internal/index/imports.go index 50095db6..acd6e152 100644 --- a/modules/mesh-delivery-indexer/internal/index/imports.go +++ b/modules/mesh-delivery-indexer/internal/index/imports.go @@ -27,6 +27,13 @@ import ( // Imports that cannot be resolved (a package the forge cannot list) are kept with no ImportForm: the next recompute that // resolves them writes the import form, one rebuild too many, never a stale module. func (ix *Indexer) ApplyBuildEnd(ctx context.Context, value []byte) error { + if err := ix.applyBuildEnd(ctx, value); err != nil { + return err + } + return ix.KeepView(ctx) // the imports kept with the view (KeepView), so a restart reads them back +} + +func (ix *Indexer) applyBuildEnd(ctx context.Context, value []byte) error { if !builds.IsBuildEntry(value) { return nil } diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 219cbe85..ffe1ffce 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -292,6 +292,13 @@ func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[ // message (applyMerged). A repository whose trunk cannot be read, or reads as holding no module at all, is left as it is // and said: a failure never empties the bucket. func (ix *Indexer) Reconcile(ctx context.Context) error { + if err := ix.reconcile(ctx); err != nil { + return err + } + return ix.KeepView(ctx) // the modules dropped leave the view's next revision too +} + +func (ix *Indexer) reconcile(ctx context.Context) error { if ix.Modules == nil || ix.Trunk == nil { return nil } diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index 4756fd1e..c732188f 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -1189,6 +1189,11 @@ func (ix *Indexer) holdFor(ctx context.Context, ready []PendingEntry, names, own } } } + // The indexer's own entry the held revision names is kept as the trunk revision after the newest, every other entry as + // it was (milestone 47/G W4.7: a merged revision naming a newer indexer replaces only the indexer's entry). + if err := ix.KeepView(ctx); err != nil { + return fmt.Errorf("the revision with the indexer's own entry: %w", err) + } t := revisionWait(takein.TakeInPending, ready, names) t.Refused = refused var ended, unmarked []PendingEntry diff --git a/modules/mesh-delivery-indexer/internal/index/source.go b/modules/mesh-delivery-indexer/internal/index/source.go index 62123afe..8ff4f934 100644 --- a/modules/mesh-delivery-indexer/internal/index/source.go +++ b/modules/mesh-delivery-indexer/internal/index/source.go @@ -2,9 +2,14 @@ package index import ( "context" + "errors" "fmt" "maps" "slices" + "strings" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" @@ -56,6 +61,9 @@ func (ix *Indexer) ApplySource(ctx context.Context, a meshdelivery.SourceAnswer) if ix.Modules == nil { return ix.RecomputeCandidates(ctx) } + if err := ix.kickCatchUp(ctx, s.Heads); err != nil { + return err + } // A registered module the forge could not read is read again at each new source revision (issue 1945), in the // background: a forge that does not answer must not hold this revision. One taken in later is stamped when it is written // (applyManifests reads srcOf). @@ -100,6 +108,88 @@ func (ix *Indexer) ApplySource(ctx context.Context, a meshdelivery.SourceAnswer) return fmt.Errorf("modules.%s: %w", key, err) } } + // The stamps are kept with the view (KeepView), so a restart reads them back. + if err := ix.KeepView(ctx); err != nil { + return err + } // A trunk that moved makes the previews made onto the old one out of date. return ix.RecomputeCandidates(ctx) } + +// kickCatchUp takes the trunk in at a first start, when the objects keep no trunk revision yet (the lead's ruling A on +// milestone 47/G: the catch-up path): the head the source names for every repository, by the merged message the forge +// bridge gives for it, each applied as its merge, and revision 1 kept from them all at its end. Until then the revision +// the tools answer waits, never an empty catalog. The catch-up runs off the source handler, in its own goroutine, said +// when it ends (lead-47's ruling R1 on the rebase): a bridge that does not answer holds the kept call (ADR 0668) to its +// bound, and must not hold this source revision's stamps and the pulls' candidates with it. A repository whose message is +// not read keeps no revision, said: the next source revision runs the catch-up again (no timer). Objects not on the bus +// yet run none, said, until then. +func (ix *Indexer) kickCatchUp(ctx context.Context, heads map[string]string) error { + if ix.Revisions == nil || ix.Message == nil || len(heads) == 0 { + return nil + } + if _, any, err := newestTrunk(ctx, ix.Revisions); errors.Is(err, process.ErrNotYet) { + ix.logf("catalog graph: the trunk is not taken in yet: its objects %s are not on the bus", CatalogRevisionsObjects) + return nil + } else if err != nil || any { + return err + } + if !ix.catchingUp.CompareAndSwap(false, true) { + return nil + } + heads = maps.Clone(heads) + ix.catchUpWG.Add(1) + go func() { + defer ix.catchUpWG.Done() + unread := ix.takeInHeads(ctx, heads) + ix.catchingUp.Store(false) + if len(unread) > 0 { + ix.logf("catalog graph: the first start's catch-up ended and no first revision is kept: %d repositories were not taken in (%s); taken in again at the next source revision", + len(unread), strings.Join(unread, ", ")) + return + } + if err := ix.KeepView(ctx); err != nil { + ix.logf("catalog graph: the first start's catch-up took in the trunk of %d repositories, but its revision could not be kept: %v; taken in again at the next source revision", len(heads), err) + return + } + ix.logf("catalog graph: the first start's catch-up ended: the trunk of %d repositories taken in", len(heads)) + }() + return nil +} + +// takeInHeads applies the forge bridge's merged message of each repository's head, as its merge, and answers the +// repositories whose message was not read or not applied. +func (ix *Indexer) takeInHeads(ctx context.Context, heads map[string]string) []string { + var unread []string + for _, repository := range slices.Sorted(maps.Keys(heads)) { + head := heads[repository] + if head == "" { + continue + } + f, err := ix.mergedAt(ctx, repository, head) + if err == nil { + _, err = ix.ApplyMessage(ctx, f) + } + if err != nil { + unread = append(unread, repository) + ix.logf("catalog graph: the trunk of %s at %.12s is not taken in: %v", repository, head, err) + } + } + return unread +} + +// mergedAt is the forge bridge's merged message of a repository's head. +func (ix *Indexer) mergedAt(ctx context.Context, repository, head string) (forgemsg.Forge, error) { + data, err := ix.Message(ctx, repository, head, 0) + if err != nil { + return forgemsg.Forge{}, fmt.Errorf("the forge bridge could not answer: %w", err) + } + f, err := forgemsg.DecodeForge(data) + if err != nil { + return f, fmt.Errorf("the forge bridge's answer is not a forge message: %w", err) + } + if f.Kind != forgemsg.ForgeMerged || f.Merge != head || !trunk.SameRepository(f.Repository, repository) { + return f, fmt.Errorf("the forge bridge's answer is not the merge of %s at %s", repository, head) + } + return f, nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index e3abfa5e..72366ff3 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -2,6 +2,7 @@ package index import ( "context" + "encoding/json" "errors" "fmt" "maps" @@ -9,6 +10,7 @@ import ( "reflect" "slices" "strings" + "sync" "testing" "time" @@ -354,7 +356,7 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // } // W4.3's second part: the view of the trunk read at start is the newest trunk revision, each module's value as its // entry gives it back; a bucket that keeps none is an empty view at 0. - view, at, err := TrunkView(context.Background(), kept) + view, nodes, at, err := TrunkView(context.Background(), kept) if err != nil || at != 2 { t.Fatalf("the view is read at %d (%v), want 2", at, err) } @@ -365,8 +367,16 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // if want := ModuleValueOf(e); !found || err != nil || !reflect.DeepEqual(got, want) { t.Errorf("the view's %s = %+v (%v), want %+v", name, got, err, want) } + // The node of the module graph the entry carries is read back with the view (NodeOf), so a revision kept before the + // first recompute after a start loses no edge. + gk, _ := graph.Key(name) + rawNode, _, found, _ := nodes.Get(context.Background(), gk) + if back, err := graph.Decode(rawNode); !found || err != nil || !reflect.DeepEqual(NodeOf(e), back) || back.FP != e.Fp || + !reflect.DeepEqual(back.Bases, e.Bases) || !reflect.DeepEqual(CatalogEntryOf(name, ModuleValueOf(e), catalogue.Manifest{}, back).Bases, e.Bases) { + t.Errorf("the view's node of %s = %+v (%v), want %+v", name, back, err, NodeOf(e)) + } } - if empty, at, err := TrunkView(context.Background(), newMemObjects(0)); err != nil || at != 0 || func() bool { k, _ := empty.Keys(context.Background()); return len(k) != 0 }() { + if empty, _, at, err := TrunkView(context.Background(), newMemObjects(0)); err != nil || at != 0 || func() bool { k, _ := empty.Keys(context.Background()); return len(k) != 0 }() { t.Errorf("an empty bucket's view is read at %d (%v), want an empty view at 0", at, err) } _ = kept.Delete(context.Background(), EntryObject(next.Entries[0].Digest)) @@ -445,6 +455,69 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // t.Errorf("%s moved with a build of build-agent: fp2 = %q, want %s", name, m.Fp2, fp) } } + // The lead's ruling A on milestone 47/G: a first start whose objects keep no trunk revision takes in the head the source + // names for every repository, by the forge bridge's merged message, and keeps revision 1 with every module; a message + // the bridge does not give keeps none, and the next source revision takes the trunk in again. + first, _, merged := newFormIndexerUnmerged(t, builderModules(t)...) + first.Revisions = newMemObjects(0) + var firstSaid []meshcatalog.CatalogRevisionEvent + first.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error { + firstSaid = append(firstSaid, e) + return nil + } + // Lead-47's ruling R1: the catch-up runs off the source handler, so a bridge that has not answered yet holds no source + // revision (the first message is answered only once ApplySource returned; held in the handler it is answered as failed). + answers, applied := 0, make(chan struct{}) + first.Message = func(_ context.Context, repository, head string, _ int) ([]byte, error) { + if answers++; answers == 1 { + select { + case <-applied: + return nil, errors.New("the forge bridge does not answer") + case <-time.After(10 * time.Second): + return nil, errors.New("the catch-up held the source handler") + } + } + if repository != catalogueRepo || head != merged.Merge { + return nil, fmt.Errorf("no merge of %s at %s", repository, head) + } + return json.Marshal(merged) + } + heads := meshdelivery.SourceByMainValue{Heads: map[string]string{catalogueRepo: merged.Merge}} + var firstLog []string + var logMu sync.Mutex + first.Logf = func(f string, a ...any) { + logMu.Lock() + firstLog = append(firstLog, fmt.Sprintf(f, a...)) + logMu.Unlock() + } + err = first.ApplySource(context.Background(), 5, heads) + close(applied) + first.catchUpWG.Wait() + if err != nil || len(firstSaid) != 0 || !strings.Contains(strings.Join(firstLog, "\n"), "the forge bridge does not answer") { + t.Fatalf("a catch-up whose message is not read said %+v (%v), log %q, want no revision and the bridge's silence said", firstSaid, err, firstLog) + } + err = first.ApplySource(context.Background(), 6, heads) + first.catchUpWG.Wait() + if err != nil || len(firstSaid) != 1 || firstSaid[0].Revision != 1 || + firstSaid[0].Entries != int64(len(builderModules(t))) || firstSaid[0].Source != 6 { + t.Fatalf("the first start's catch-up said %+v (%v), want trunk revision 1 of source 6 with every module", firstSaid, err) + } + // KeepView: a write to the view outside a recompute (here a verdict carried into one module) is the next revision. + viewKey, _ := meshcatalogclient.ModulesKey("avahi") + _ = write(context.Background(), first.Modules, viewKey, func(value []byte, _ bool) ([]byte, bool, error) { + m, _ := meshcatalogclient.DecodeModule(value) + m.Verdict, m.Problems = meshcatalogclient.VerdictRefused, []string{"a field the judge refuses"} + data, err := meshcatalogclient.EncodeModule(m) + return data, false, err + }) + if err := first.KeepView(context.Background()); err != nil || len(firstSaid) != 2 || !reflect.DeepEqual(firstSaid[1].Changed, []string{"avahi"}) { + t.Errorf("KeepView after a write said %+v (%v), want revision 2 naming avahi", firstSaid, err) + } + if view, _, at, err := TrunkView(context.Background(), first.Revisions); err != nil || at != 2 { + t.Errorf("the view at a restart is read at %d (%v), want 2", at, err) + } else if raw, _, _, _ := view.Get(context.Background(), viewKey); !strings.Contains(string(raw), "a field the judge refuses") { + t.Errorf("the view at a restart lost the verdict: %s", raw) + } // ADR 0614 §2, §3: the graph names what each module invokes and serves, and leaves out, over the whole graph, the // verb edges of two modules calling each other, from either side; a third calling one of them keeps its edge. invoking := func(name, invokes string) mod { diff --git a/modules/mesh-delivery-indexer/internal/index/verdicts.go b/modules/mesh-delivery-indexer/internal/index/verdicts.go index 24ce8159..22bc5764 100644 --- a/modules/mesh-delivery-indexer/internal/index/verdicts.go +++ b/modules/mesh-delivery-indexer/internal/index/verdicts.go @@ -252,6 +252,10 @@ func (ix *Indexer) judgeOverHeld(ctx context.Context, fresh map[string]bool, ove ix.logf("judge: modules.%s does not carry its verdict: %v", name, err) } } + // The verdicts carried are kept with the view (KeepView), so a restart reads them back. + if err := ix.KeepView(ctx); err != nil { + ix.logf("judge: the verdicts carried are not kept in a catalog graph revision: %v", err) + } if over != nil { return out, nil } From 66dd7b3a3f81e508efb1436d8d000ad6a9fe9696 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 9 Nov 2026 20:59:09 +0100 Subject: [PATCH 08/15] The indexer's tools answer the modules and the module graph from the newest catalog graph revision and wait while none is kept (milestone 47/G W4.3, the graph state's readers) The tools bundle reads the objects catalog-revisions through its runtime (index.TrunkStates over busstate): the first views, read tools and seat verbs of modules and graph, indexer_config_readers, indexer_bus_state and indexer_package_users; the marks of modules and graph are the revision's number, and a view asked before the first revision says view-waits, never an empty catalog. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-delivery-indexer-tools/catalog.go | 17 ++-- .../cmd/mesh-delivery-indexer-tools/main.go | 11 ++- .../mesh-delivery-indexer-tools/revision.go | 50 +++++++++++ .../cmd/mesh-delivery-indexer-tools/views.go | 16 ++-- .../internal/index/catalog_revisions.go | 90 +++++++++++++++++++ .../internal/index/toolchains_test.go | 13 +++ 6 files changed, 178 insertions(+), 19 deletions(-) create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/revision.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go index 7b080273..511ca138 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go @@ -13,8 +13,9 @@ import ( "mesh-delivery-indexer/internal/index" ) -// catalogRead reads the indexer's modules, manifests and graph states for the seat's verbs. -func catalogRead(modules, manifests, nodes stdio.KeptState, seatDefinitions, marks views.Source) index.CatalogRead { +// catalogRead reads the indexer's view of the trunk (its modules and module graph, from the newest catalog graph +// revision) and its manifests state for the seat's verbs. +func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.Source, seatDefinitions, marks views.Source) index.CatalogRead { return index.CatalogRead{ SeatDefinitions: seatDefinitions, Marks: marks, Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) { @@ -24,11 +25,11 @@ func catalogRead(modules, manifests, nodes stdio.KeptState, seatDefinitions, mar } out := map[string]meshcatalog.ModulesByModuleValue{} for _, k := range keys { - e, err := modules.Get(k) + e, ok, err := modules.Get(k) if err != nil { return nil, fmt.Errorf("modules.%s: %w", k, err) } - if e == nil { + if !ok { continue } if m, err := meshcatalogclient.DecodeModule(e.Value); err == nil { @@ -59,11 +60,11 @@ func catalogRead(modules, manifests, nodes stdio.KeptState, seatDefinitions, mar } g := graph.Graph{} for _, k := range keys { - e, err := nodes.Get(k) + e, ok, err := nodes.Get(k) if err != nil { return nil, fmt.Errorf("graph.%s: %w", k, err) } - if e == nil { + if !ok { continue } if n, err := graph.Decode(e.Value); err == nil { @@ -76,7 +77,7 @@ func catalogRead(modules, manifests, nodes stdio.KeptState, seatDefinitions, mar } // catalogTools are the seat's verbs over the indexer's own states, served beside the indexer's own tools, since the indexer's manifest claims the seat (ADR 0604 §2). -func catalogTools() []stdio.Tool { - return index.CatalogTools(catalogRead(stdio.State("modules"), stdio.State("manifests"), stdio.State(graph.State), +func catalogTools(state func(string) views.Source) []stdio.Tool { + return index.CatalogTools(catalogRead(state("modules"), stdio.State("manifests"), state(graph.State), keptSource(index.SeatDefinitionsState), keptSource(views.MarksState))) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index e4c89f9c..3126dbdd 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -17,6 +17,7 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/busstate" "mesh-delivery-indexer/internal/index" ) @@ -275,16 +276,18 @@ func main() { asks := askTakeIn(stdio.State(index.TakeInsState), 25*time.Second) acts := meshhandactsclient.Recorder{Ask: stdio.Ask} tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts) - tools = append(tools, catalogTools()...) - tools = append(tools, viewTools(os.Getenv, keptSource, time.Now)...) + // The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go. + trunk := index.TrunkStates{Objects: bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)}, Under: keptSource} + tools = append(tools, catalogTools(trunk.State)...) + tools = append(tools, viewTools(os.Getenv, trunk.State, time.Now)...) tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, keptRead{stdio.State(index.OwnBuildStateName)}, asks, time.Now)...) tools = append(tools, index.FingerprintsTools(asks, time.Now, func(what, why string) error { _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.FingerprintsAgainTool, Verb: "fingerprints-again"}) return err }, 10*time.Second)...) tools = append(tools, index.TryCandidateTools(keptRead{stdio.State("candidates")}, keptRead{stdio.State(index.TryRecoveryState)}, asks, time.Now)...) - tools = append(tools, configReadersTool(stdio.NewWatchedStates()), busStateTool(keptSource), requirementsTool(keptSource), - packageUsersTool(keptSource), seatDefinitionTool(keptSource), outOfStepTool(keptSource), + tools = append(tools, configReadersTool(stdio.NewWatchedStates(), trunk.State("modules")), busStateTool(trunk.State), requirementsTool(keptSource), + packageUsersTool(trunk.State), seatDefinitionTool(keptSource), outOfStepTool(keptSource), holdsTool(stdio.State(index.HoldsState), asks, time.Now, acts), engineReadsTool(time.Now)) if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/revision.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/revision.go new file mode 100644 index 00000000..71407173 --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/revision.go @@ -0,0 +1,50 @@ +package main + +import ( + "context" + + "git.novox.be/novox/mesh-sdk/go/busstate" + + "mesh-delivery-indexer/internal/index" +) + +// bundleObjects is the objects catalog-revisions as the bundle reaches them through its runtime (the old objects section +// of a foundation part, reached by the module's own name). +type bundleObjects struct{ o busstate.Objects } + +func (b bundleObjects) Put(_ context.Context, name string, data []byte) error { + _, err := b.o.Put(name, data, "") + return err +} + +func (b bundleObjects) Get(_ context.Context, name string) ([]byte, error) { + d, err := b.o.Get(name) + if err != nil { + return nil, err + } + if d.Info.Name == "" || d.Info.Deleted { + return nil, index.ErrNoObject + } + return d.Data, nil +} + +func (b bundleObjects) Delete(_ context.Context, name string) error { return b.o.Delete(name) } + +func (b bundleObjects) Names(context.Context) ([]string, error) { + infos, err := b.o.List() + if err != nil { + return nil, err + } + names := []string{} + for _, i := range infos { + if !i.Deleted { + names = append(names, i.Name) + } + } + return names, nil +} + +func (b bundleObjects) Fill(context.Context) (int64, int64, error) { + f, err := b.o.Fill() + return int64(f.Bytes), int64(f.MaxBytes), err +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go index 3b287093..ecd4431e 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go @@ -37,9 +37,9 @@ func viewTools(getenv func(string) string, state func(string) views.Source, now func keptSource(name string) views.Source { return views.KeptSource{State: stdio.State(name)} } // configReadersTool is indexer_config_readers (design 32 *A value no longer read*; DLV-I10): read from one watched copy -// of the indexer's modules, manifests and config-candidates states, so an ask on each manifest-accepted reads no state +// of the indexer's manifests and config-candidates states and its view of the trunk (the newest catalog graph revision), so an ask on each manifest-accepted reads no state // whole again. -func configReadersTool(watched *stdio.WatchedStates) stdio.Tool { +func configReadersTool(watched *stdio.WatchedStates, modules views.Source) stdio.Tool { return stdio.Tool{Name: index.ConfigReadersTool, Description: "Who reads a module's config keys now (design 32): every value key and own secret any manifest of the module " + "the indexer keeps ever declared, each with its readers — the module itself where its trunk's manifest declares it, and " + @@ -53,13 +53,15 @@ func configReadersTool(watched *stdio.WatchedStates) stdio.Tool { return nil, fmt.Errorf("%s names the module", index.ConfigReadersTool) } trunk := map[string]meshcatalog.ModulesByModuleValue{} - mods, err := watched.Values("modules") + keys, err := modules.Keys() if err != nil { - return nil, fmt.Errorf("the modules state cannot be read: %w", err) + return nil, fmt.Errorf("the view of the trunk cannot be read: %w", err) } - for k, v := range mods { - if m, err := meshcatalogclient.DecodeModule(v); err == nil { - trunk[k] = m + for _, k := range keys { + if e, ok, err := modules.Get(k); err == nil && ok { + if m, err := meshcatalogclient.DecodeModule(e.Value); err == nil { + trunk[k] = m + } } } kept := map[string]meshcatalog.ManifestsByDigestValue{} diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index 2ccdf931..408b5951 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -8,12 +8,14 @@ import ( "slices" "strconv" "strings" + "time" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/sdk/graph" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/views" ) // The indexer's object bucket catalog-revisions (design 47, Two graphs; novox/hq ADR 0619 §2: declared in the old @@ -325,3 +327,91 @@ func TrunkView(ctx context.Context, o Objects) (modules, nodes Store, at int64, // MemoryStore is a store held in memory alone: the module graph's nodes, which each revision's entries carry, so the // indexer keeps no state of them on the bus (milestone 47/G). func MemoryStore() Store { return newOverlay(nil) } + +// The view of the trunk is the newest catalog graph revision the indexer's objects catalog-revisions keep (milestone +// 47/G, ADR 0656 §6): the tools bundle answers the modules and the module graph it once read from the states modules and +// graph from that revision, at its number, and says it waits while the indexer keeps none yet (ADR 0615 §6), never an +// empty catalog. + +// ErrNoRevision is a view asked before the indexer kept its first trunk revision. +var ErrNoRevision = errors.New("view-waits: the indexer keeps no catalog graph revision yet (its first start takes the trunk in); ask again on catalog-revision") + +// TrunkStates reads the view of the trunk from the newest trunk revision; under answers every other state. +type TrunkStates struct { + Objects Objects + Under func(string) views.Source +} + +// read is the view's modules and nodes, and the revision they were read at; none kept yet is ErrNoRevision. +func (t TrunkStates) read(nodes bool) (Store, int64, error) { + mods, held, at, err := TrunkView(context.Background(), t.Objects) + if err == nil && at == 0 { + err = ErrNoRevision + } + if nodes { + return held, at, err + } + return mods, at, err +} + +// state is the source of a state by its name: modules and graph from the revision, their marks at its number, every +// other the indexer's own state. +func (t TrunkStates) State(name string) views.Source { + switch name { + case "modules", graph.State: + return revisionSource{t: t, nodes: name == graph.State} + case views.MarksState: + return markSource{t: t, under: t.Under(name)} + } + return t.Under(name) +} + +// revisionSource is the modules or the module graph of the newest trunk revision, each value at its number. +type revisionSource struct { + t TrunkStates + nodes bool +} + +func (r revisionSource) Keys() ([]string, error) { + s, _, err := r.t.read(r.nodes) + if err != nil { + return nil, err + } + return s.Keys(context.Background()) +} + +func (r revisionSource) Get(key string) (*views.Entry, bool, error) { + s, at, err := r.t.read(r.nodes) + if err != nil { + return nil, false, err + } + value, _, found, err := s.Get(context.Background(), key) + if err != nil || !found { + return nil, false, err + } + return &views.Entry{Value: value, Revision: uint64(at)}, true, nil +} + +// markSource answers the mark of modules and graph as the newest trunk revision's number, which catalog-revision says +// the changes after; every other mark is the views family's. +type markSource struct { + t TrunkStates + under views.Source +} + +func (m markSource) Keys() ([]string, error) { return m.under.Keys() } + +func (m markSource) Get(state string) (*views.Entry, bool, error) { + if state != "modules" && state != graph.State { + return m.under.Get(state) + } + _, at, err := m.t.read(false) + if err != nil { + return nil, false, err + } + data, err := json.Marshal(views.Mark{V: views.Version, State: state, Revision: uint64(at), At: time.Now().UTC()}) + if err != nil { + return nil, false, err + } + return &views.Entry{Value: data, Revision: uint64(at)}, true, nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 72366ff3..c0eadfda 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -24,6 +24,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/views" "git.novox.be/novox/mesh-delivery/sdk/graph" @@ -518,6 +519,18 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // } else if raw, _, _, _ := view.Get(context.Background(), viewKey); !strings.Contains(string(raw), "a field the judge refuses") { t.Errorf("the view at a restart lost the verdict: %s", raw) } + // The tools bundle answers modules and graph from the newest trunk revision, their marks at its number; with none kept + // it says it waits (view-waits), never an empty catalog. + trunkStates := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }} + if keys, err := trunkStates.State("modules").Keys(); err != nil || len(keys) != len(builderModules(t)) { + t.Errorf("the bundle's modules = %v (%v), want every module of revision 2", keys, err) + } + if e, ok, err := trunkStates.State(views.MarksState).Get("graph"); err != nil || !ok || !strings.Contains(string(e.Value), `"revision":2`) { + t.Errorf("the bundle's mark of graph = %+v %v (%v), want revision 2", e, ok, err) + } + if _, err := (TrunkStates{Objects: newMemObjects(0)}).State("graph").Keys(); !errors.Is(err, ErrNoRevision) { + t.Errorf("the bundle's graph with no revision kept answered %v, want it to wait", err) + } // ADR 0614 §2, §3: the graph names what each module invokes and serves, and leaves out, over the whole graph, the // verb edges of two modules calling each other, from either side; a third calling one of them keeps its edge. invoking := func(name, invokes string) mod { From a732cb0cb2f332fbbb2c518ac1923522db89431d Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 9 Nov 2026 21:01:34 +0100 Subject: [PATCH 09/15] The indexer declares no modules or graph state: its view of the trunk is the catalog graph revisions (milestone 47/G W4.3, the manifest) The states modules and graph leave the manifest and the names the indexer keeps; their first views and read tools are answered from the newest revision, whose changes catalog-revision says. The contract keeps the modules type until its readers moved (the lead's ruling). The manifest validator, built at mesh-delivery's trunk, accepts the indexer in both cases, change and trunk, and no verdict differs. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-indexer/integration_test.go | 14 +++++++------- .../mesh-delivery-indexer/internal/index/names.go | 6 ------ .../mesh-delivery-indexer/internal/index/views.go | 2 -- modules/mesh-delivery-indexer/manifest_test.go | 12 ++++++------ modules/mesh-delivery-indexer/module.json | 2 -- 5 files changed, 13 insertions(+), 23 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index efafb1e5..16f3ec69 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -263,7 +263,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 named := held.WaitsFor get := func(bucket, key string) ([]byte, bool) { t.Helper() - if bucket == index.ModulesBucket { + if bucket == index.ModulesState { // the view of the trunk, from the newest catalog graph revision e, err := revisionView{js: js}.Get(ctx, key) if err != nil { return nil, false @@ -281,7 +281,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 return e.Value(), true } // Only what building and moving the named indexer needs is written. - raw, ok := get(index.ModulesBucket, ix) + raw, ok := get(index.ModulesState, ix) own, err := meshcatalogclient.DecodeModule(raw) if !ok || err != nil || own.Fp2 != named || own.Commit != m1.Merge { t.Fatalf("modules.%s = %+v (%v), want the named build %s at the merge", ix, own, err, named) @@ -293,7 +293,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 t.Errorf("the verdict on the indexer's manifest %s is not written", own.Manifest) } repoKey, _ := trunk.RepositoryKey(m1.Repository) - if _, ok := get(index.ModulesBucket, "a"); ok { + if _, ok := get(index.ModulesState, "a"); ok { t.Errorf("modules.a is written while the revision is pending") } if _, ok := get(index.TakenBucket, repoKey); ok { @@ -353,7 +353,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 break } } - if raw, ok := get(index.ModulesBucket, "a"); !ok || !strings.Contains(string(raw), m1.Merge) { + if raw, ok := get(index.ModulesState, "a"); !ok || !strings.Contains(string(raw), m1.Merge) { t.Errorf("modules.a = %s after the take-in at the start, want it at the merge %s", raw, m1.Merge) } if _, ok := get(index.TakenBucket, repoKey); !ok { @@ -362,7 +362,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 if len(w) != 1 || w[0].Key != groupKey || len(w[0].Keys) != 1 || w[0].Keys[0] != g1Key { t.Fatalf("indexer_pending after the start = %+v, want only the group gathered, its first member %s", w, g1Key) } - if _, ok := get(index.ModulesBucket, "c"); ok { + if _, ok := get(index.ModulesState, "c"); ok { t.Errorf("modules.c is written while its group is gathered") } @@ -381,8 +381,8 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 } var hasC, hasK bool for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && !(hasC && hasK && len(w) == 0); time.Sleep(50 * time.Millisecond) { - _, hasC = get(index.ModulesBucket, "c") - _, hasK = get(index.ModulesBucket, "k") + _, hasC = get(index.ModulesState, "c") + _, hasK = get(index.ModulesState, "k") w = waits() } if !hasC || !hasK || len(w) != 0 { diff --git a/modules/mesh-delivery-indexer/internal/index/names.go b/modules/mesh-delivery-indexer/internal/index/names.go index e3d860d9..7a53689a 100644 --- a/modules/mesh-delivery-indexer/internal/index/names.go +++ b/modules/mesh-delivery-indexer/internal/index/names.go @@ -1,7 +1,5 @@ package index -import "git.novox.be/novox/mesh-delivery/sdk/graph" - // The holders, states and buckets the indexer names, taken from its own manifest (module.json: its name, its `state`, its // `reads`) and from the manifests of the holders it reads; a unit row checks them against the manifest // (TestTheNamesAreTheManifests). No contract names a holder (novox/hq ADR 0625 §4, ADR 0634 §1). @@ -14,13 +12,9 @@ const ( CandidatesBucket = Module + "_" + CandidatesState CandidatesHistory = 10 ModulesState = "modules" - ModulesBucket = Module + "_" + ModulesState - ModulesHistory = 20 ManifestsState = "manifests" ManifestsBucket = Module + "_" + ManifestsState ManifestsHistory = 1 - // GraphBucket is the bucket of the state graph (sdk/graph names the state, not its holder). - GraphBucket = Module + "_" + graph.State // VerdictsState and VerdictsBucket are in verdicts.go. VerdictsHistory = 5 PredefinedState = "predefined" diff --git a/modules/mesh-delivery-indexer/internal/index/views.go b/modules/mesh-delivery-indexer/internal/index/views.go index 7bcffe38..88bebe00 100644 --- a/modules/mesh-delivery-indexer/internal/index/views.go +++ b/modules/mesh-delivery-indexer/internal/index/views.go @@ -43,13 +43,11 @@ type ViewedTool struct { // ViewedStates are the states the indexer's views family says, each by its bucket: the ones its readers read. var ViewedStates = map[string]string{ "manifests": ManifestsBucket, - "modules": ModulesBucket, VerdictsState: VerdictsBucket, PredefinedState: PredefinedBucket, ConfigCandidatesState: ConfigCandidatesBucket, "candidates": CandidatesBucket, PackagesState: PackagesBucket, - graph.State: GraphBucket, HeardState: HeardBucket, "taken": TakenBucket, mcc.FindingsState: mcc.FindingsBucket(Module), diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 34b28b91..209bfd62 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -16,7 +16,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" - "git.novox.be/novox/mesh-delivery/sdk/graph" "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-indexer/internal/index" @@ -117,8 +116,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build t.Errorf("emits = %v, want the two take-in events (the verdict events are the seat mesh-catalog's)", m.Emits) } // 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard. - if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: "try-recovery", History: 1}, {Name: "own-build", History: 1}, {Name: index.PackagesState, History: index.PackagesHistory}, - {Name: graph.State, History: graph.History}, {Name: views.MarksState, History: 1}, {Name: mcc.FindingsState, History: 5}, {Name: views.StatusState, History: 1}, + if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: "try-recovery", History: 1}, {Name: "own-build", History: 1}, {Name: index.PackagesState, History: index.PackagesHistory}, + {Name: views.MarksState, History: 1}, {Name: mcc.FindingsState, History: 5}, {Name: views.StatusState, History: 1}, {Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5}, {Name: index.HoldsState, History: 5}, {Name: index.EngineReadsState, History: 1}, {Name: index.HoldBoundsState, History: 1, KeyTTL: true}, {Name: index.PendingState, History: 1}, {Name: index.UnjudgedState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17, ADR 0632 §2 t.Errorf("state = %+v, want %+v", m.State, want) @@ -133,8 +132,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build if index.Module != m.Module { t.Errorf("the indexer names itself %q, its manifest's module is %q", index.Module, m.Module) } - for bucket, history := range map[string]int{index.CandidatesBucket: index.CandidatesHistory, index.ModulesBucket: index.ModulesHistory, index.ManifestsBucket: index.ManifestsHistory, - index.VerdictsBucket: index.VerdictsHistory, index.PackagesBucket: index.PackagesHistory, index.TakenBucket: 1, index.PredefinedBucket: 5, index.ConfigCandidatesBucket: 5, index.GraphBucket: graph.History} { + for bucket, history := range map[string]int{index.CandidatesBucket: index.CandidatesHistory, index.ManifestsBucket: index.ManifestsHistory, + index.VerdictsBucket: index.VerdictsHistory, index.PackagesBucket: index.PackagesHistory, index.TakenBucket: 1, index.PredefinedBucket: 5, index.ConfigCandidatesBucket: 5} { if !slices.ContainsFunc(m.State, func(st state) bool { return m.Module+"_"+st.Name == bucket && st.History == history }) { t.Errorf("the indexer names its bucket %s with %d revisions, its manifest's states are %+v", bucket, history, m.State) } @@ -177,7 +176,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build want := []string{"indexer_take_in", "indexer_old_sections", "indexer_verdicts", "indexer_judge"} for _, v := range append(append([]index.ViewedTool{}, index.FirstViewTools...), index.ReadTools...) { want = append(want, v.Tool) - if _, ok := index.ViewedStates[v.State]; !ok { + // Milestone 47/G: modules and graph are the newest catalog graph revision's, which catalog-revision says the changes of. + if _, ok := index.ViewedStates[v.State]; !ok && v.State != index.ModulesState && v.State != "graph" { t.Errorf("%s answers the state %s, which the views family does not say", v.Tool, v.State) } } diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index b56cb192..c3a0a3c7 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -77,7 +77,6 @@ { "name": "candidates", "history": 10 }, { "name": "heard", "history": 64 }, { "name": "manifests", "history": 1 }, - { "name": "modules", "history": 20 }, { "name": "predefined", "history": 5 }, { "name": "config-candidates", "history": 5 }, { "name": "taken", "history": 1 }, @@ -87,7 +86,6 @@ { "name": "try-recovery", "history": 1 }, { "name": "own-build", "history": 1 }, { "name": "packages", "history": 5 }, - { "name": "graph", "history": 5 }, { "name": "view-marks", "history": 1 }, { "name": "findings", "history": 5 }, { "name": "first-views", "history": 1 }, From 64afa0045740a77d651b8fd5edd4d087ee01df48 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 9 Nov 2026 21:08:58 +0100 Subject: [PATCH 10/15] The indexer's catch-up tool takes no arguments and is named beside its catalog graph revisions (milestone 47/G W4.14) The node agent calls a restored part's catch-up tool with no arguments, so indexer_take_in with neither module nor repository takes in the head the gate's source names for every repository, as the first start's catch-up does, and keeps the view as the next revision when an entry moved. The manifest names it beside the catalog graph revisions, form 1 (design 47, Reverted). Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-indexer-tools/main.go | 12 +++++-- .../cmd/mesh-delivery-indexer/takein.go | 4 +++ .../internal/index/registered.go | 2 ++ .../internal/index/source.go | 36 +++++++++++++++++++ .../internal/index/takein.go | 19 +++++++++- .../internal/index/toolchains_test.go | 18 ++++++++++ .../mesh-delivery-indexer/manifest_test.go | 14 +++++--- modules/mesh-delivery-indexer/module.json | 2 +- 8 files changed, 97 insertions(+), 10 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 3126dbdd..2889ddc7 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -57,7 +57,10 @@ func Tools(ask asker, now func() time.Time, read stdio.KeptState, counts func() "source names for that repository, for a merge whose merged message the indexer never heard (the forge bridge has no " + "tool): a head whose trunk holds no module.json is taken in as a merge that leaves none (every module of the repository " + "is dropped, one another repository's trunk holds handed over from there first, listed in handed and dropped); a head " + - "that holds modules is answered \"take each module in by name\".", + "that holds modules is answered \"take each module in by name\". With neither module nor repository it is the indexer's " + + "catch-up tool, named in its manifest beside the catalog graph revisions: it takes in the head the gate's source names for " + + "every repository, as merges, and keeps the view as the next revision when an entry moved; the node agent calls it so " + + "after restoring them. Answers the repositories taken in and, in unread, those whose merged message was not read.", Input: map[string]any{"type": "object", "properties": map[string]any{ "module": map[string]any{"type": "string", "description": "the module's name, as the catalogue registers it"}, "repository": map[string]any{"type": "string", "description": "instead of module: the repository (owner/name) whose head is taken in"}, @@ -66,9 +69,12 @@ func Tools(ask asker, now func() time.Time, read stdio.KeptState, counts func() var key string var value []byte var err error - if str(args, "repository") != "" && str(args, "module") == "" { + switch { + case str(args, "repository") == "" && str(args, "module") == "": + key, value, err = index.NewCatchUp(str(args, "why"), stdio.CallerOf(args), now()) + case str(args, "repository") != "" && str(args, "module") == "": key, value, err = index.NewTakeInRepository(str(args, "repository"), str(args, "why"), stdio.CallerOf(args), now()) - } else { + default: key, value, err = index.NewTakeIn(str(args, "module"), str(args, "why"), stdio.CallerOf(args), now()) } if err != nil { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go index af5f2e8a..37822709 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go @@ -15,6 +15,7 @@ import ( type takerIn interface { TakeIn(ctx context.Context, module string) index.TakenIn TakeInRepository(ctx context.Context, repository string) index.TakenIn + CatchUp(ctx context.Context) index.TakenIn JudgeAgain(ctx context.Context, module, digest string) index.TakenIn SayAcceptedAgain(ctx context.Context, module string) index.TakenIn SeatDefinitionsAgain(ctx context.Context, module string, pull int, head string) index.TakenIn @@ -60,6 +61,9 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream } else if a.Pending != "" { asked, r = "hand take-in of what waits", ix.TakeInPendingByHand(ctx, a.Pending, a.By, a.Why) a.Module = a.Pending + } else if a.CatchUp { + asked, r = "catch-up of every repository", ix.CatchUp(ctx) + a.Module = r.Module } else if a.Repository != "" { asked, r = "take-in of a repository", ix.TakeInRepository(ctx, a.Repository) a.Module = a.Repository diff --git a/modules/mesh-delivery-indexer/internal/index/registered.go b/modules/mesh-delivery-indexer/internal/index/registered.go index 2f30556c..5bf426d7 100644 --- a/modules/mesh-delivery-indexer/internal/index/registered.go +++ b/modules/mesh-delivery-indexer/internal/index/registered.go @@ -220,6 +220,8 @@ type TakenIn struct { // and deleted. Handed []string `json:"handed,omitempty"` Dropped []string `json:"dropped,omitempty"` + // Unread are the repositories a catch-up (CatchUp) could not take in: their merged message was not read. + Unread []string `json:"unread,omitempty"` // Fingerprints are the modules whose fingerprint a recompute by hand moved (FingerprintsAgain), each old and new. Fingerprints []FingerprintMove `json:"fingerprints,omitempty"` } diff --git a/modules/mesh-delivery-indexer/internal/index/source.go b/modules/mesh-delivery-indexer/internal/index/source.go index 8ff4f934..702990d1 100644 --- a/modules/mesh-delivery-indexer/internal/index/source.go +++ b/modules/mesh-delivery-indexer/internal/index/source.go @@ -178,6 +178,42 @@ func (ix *Indexer) takeInHeads(ctx context.Context, heads map[string]string) []s return unread } +// CatchUp is the indexer's catch-up tool's work (indexer_take_in with no module and no repository, the catch-up its +// manifest names beside the catalog graph revisions): after a restore point of them is restored, the node agent calls it +// with no arguments, and the indexer takes in the head the gate's source names for every repository, as the first +// start's catch-up does, whatever revision the objects keep, then keeps the view as the next revision when an entry +// moved. Its answer names the repositories taken in and those whose merged message could not be read. +func (ix *Indexer) CatchUp(ctx context.Context) TakenIn { + out := TakenIn{Module: "catch-up"} + ix.mu.Lock() + heads := maps.Clone(ix.source.heads) + ix.mu.Unlock() + switch { + case len(heads) == 0: + out.Answer = "the gate's source names no head yet, so nothing is taken in; the next source revision is taken in as it comes" + return out + case ix.Modules == nil || ix.Message == nil: + out.Answer = "the indexer holds no view of the trunk or reads no merged message, so nothing is taken in" + return out + case !ix.catchingUp.CompareAndSwap(false, true): + out.Answer = "a catch-up runs already; its end keeps the revision" + return out + } + unread := ix.takeInHeads(ctx, heads) + ix.catchingUp.Store(false) + if err := ix.KeepView(ctx); err != nil { + out.Answer = "the repositories were taken in but the revision could not be kept: " + err.Error() + return out + } + out.Stored = len(unread) == 0 + out.Unread = unread + out.Answer = fmt.Sprintf("the heads the gate's source names for %d repositories taken in", len(heads)-len(unread)) + if len(unread) > 0 { + out.Answer += fmt.Sprintf("; %d not (%s): their merged message was not read, and the call may be made again", len(unread), strings.Join(unread, ", ")) + } + return out +} + // mergedAt is the forge bridge's merged message of a repository's head. func (ix *Indexer) mergedAt(ctx context.Context, repository, head string) (forgemsg.Forge, error) { data, err := ix.Message(ctx, repository, head, 0) diff --git a/modules/mesh-delivery-indexer/internal/index/takein.go b/modules/mesh-delivery-indexer/internal/index/takein.go index 89505157..d6a9d524 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein.go +++ b/modules/mesh-delivery-indexer/internal/index/takein.go @@ -32,6 +32,9 @@ type TakeInAsk struct { // Repository asks the head the gate's source names for a repository to be taken in (TakeInRepository) instead of a module. Repository string `json:"repository,omitempty"` Why string `json:"why,omitempty"` + // CatchUp asks the head the gate's source names for every repository to be taken in (CatchUp): the catch-up tool, + // indexer_take_in called with no module and no repository, as the node agent calls it after a restore. + CatchUp bool `json:"catch-up,omitempty"` // Judge asks the manifests to be judged again now (indexer_judge_again) instead of the module to be taken in. Judge bool `json:"judge,omitempty"` // Digest names, with Judge, the one manifest digest of the module to judge (indexer_judge with a digest). @@ -83,7 +86,7 @@ func DecodeTakeIn(data []byte) (TakeInAsk, error) { if a.V != 1 { return a, fmt.Errorf("a takeins value of version %d, not 1", a.V) } - if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && a.TryCandidate == "" && !a.RecoverTries && !a.Fingerprints { + if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && a.TryCandidate == "" && !a.RecoverTries && !a.Fingerprints && !a.CatchUp { return a, errors.New("a takeins value naming no module, repository, pending wait, merge or try") } return a, nil @@ -114,6 +117,20 @@ func NewTakeInRepository(repository, why, by string, at time.Time) (string, []by return "repository." + key, data, err } +// CatchUpKey is the takeins key of a catch-up: one at a time, a second written over an unanswered first. +const CatchUpKey = "catch-up" + +// NewCatchUp is the value indexer_take_in writes when it names no module and no repository: the catch-up of every +// repository the gate's source names (CatchUp). +func NewCatchUp(why, by string, at time.Time) (string, []byte, error) { + why = strings.TrimSpace(why) + if why == "" { + why = "the catch-up after a restore" + } + data, err := json.Marshal(TakeInAsk{V: 1, CatchUp: true, Why: why, By: by, At: at.UTC()}) + return CatchUpKey, data, err +} + // TakeInMergeTool takes one merge in by hand, its merged message asked of the forge bridge (ADR 0633 §5, issue 1585). const TakeInMergeTool = "indexer_take_in_merge" diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index c0eadfda..d20d2798 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -519,6 +519,24 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // } else if raw, _, _, _ := view.Get(context.Background(), viewKey); !strings.Contains(string(raw), "a field the judge refuses") { t.Errorf("the view at a restart lost the verdict: %s", raw) } + // W4.14: the catch-up tool (indexer_take_in with no module and no repository, as the node agent calls it after a + // restore) takes in the head the source names for every repository again, whatever revision is kept, and keeps none + // when nothing moved; a repository whose message is not read is named in unread. + if key, value, err := NewCatchUp("", "node-engine@ace", time.Now()); err != nil || key != CatchUpKey { + t.Fatalf("NewCatchUp = %q (%v), want key %q", key, err, CatchUpKey) + } else if a, err := DecodeTakeIn(value); err != nil || !a.CatchUp { + t.Fatalf("a catch-up ask decodes to %+v (%v), want a catch-up", a, err) + } + asked := answers + if got := first.CatchUp(context.Background()); !got.Stored || answers != asked+1 || len(firstSaid) != 2 || len(got.Unread) != 0 { + t.Errorf("the catch-up answered %+v after %d asks, said %d revisions, want the one repository taken in, no new revision", got, answers-asked, len(firstSaid)) + } + first.Message = func(context.Context, string, string, int) ([]byte, error) { + return nil, errors.New("the forge bridge does not answer") + } + if got := first.CatchUp(context.Background()); got.Stored || !reflect.DeepEqual(got.Unread, []string{catalogueRepo}) { + t.Errorf("a catch-up whose message is not read answered %+v, want %s unread", got, catalogueRepo) + } // The tools bundle answers modules and graph from the newest trunk revision, their marks at its number; with none kept // it says it waits (view-waits), never an empty catalog. trunkStates := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 209bfd62..25a07565 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -57,8 +57,10 @@ type manifest struct { RunsAs string `json:"runs-as"` Streams []stream `json:"streams"` Objects []struct { - Name string `json:"name"` - MaxMiB int `json:"max-mib"` + Name string `json:"name"` + MaxMiB int `json:"max-mib"` + Form int `json:"form"` + CatchUp string `json:"catch-up"` } `json:"objects"` Claims []struct { Name string `json:"name"` @@ -123,9 +125,11 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build t.Errorf("state = %+v, want %+v", m.State, want) } // Milestone 47/G W4.2 (design 47, Two graphs; ADR 0619 §2): the catalog graph revisions are objects in the old section, - // capped at 512 MiB, under the local name the indexer opens. - if len(m.Objects) != 1 || m.Objects[0].Name != index.CatalogRevisionsObjects || m.Objects[0].MaxMiB != 512 { - t.Errorf("objects = %+v, want %s capped at 512 MiB", m.Objects, index.CatalogRevisionsObjects) + // capped at 512 MiB, under the local name the indexer opens. W4.14 (design 47, Reverted): their form is 1, and the + // catch-up tool a restore of them is followed by is indexer_take_in, which the bundle serves. + if len(m.Objects) != 1 || m.Objects[0].Name != index.CatalogRevisionsObjects || m.Objects[0].MaxMiB != 512 || + m.Objects[0].Form != 1 || m.Objects[0].CatchUp != index.TakeInTool || !slices.Contains(m.Tools, index.TakeInTool) { + t.Errorf("objects = %+v, want %s capped at 512 MiB, form 1, catch-up %s (a tool of its own)", m.Objects, index.CatalogRevisionsObjects, index.TakeInTool) } // D-11 (ADR 0634 §1): the holders and buckets the indexer names (internal/index/names.go) are its manifest's: its own // name and states with their revisions, and the states it reads, each as `.`. diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index c3a0a3c7..ef26c524 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -98,7 +98,7 @@ { "name": "unjudged", "history": 1 } ], "objects": [ - { "name": "catalog-revisions", "max-mib": 512 } + { "name": "catalog-revisions", "max-mib": 512, "form": 1, "catch-up": "indexer_take_in" } ], "emits": [ "take-in.pending", From 5595403215ecda453c00f9bda8a4ce00ce7d3216 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 9 Nov 2026 21:10:12 +0100 Subject: [PATCH 11/15] The indexer opens its catalog graph revisions under the module its account runs as, not a fixed name (milestone 47/G W4.2) A module's name in its own code logic is a fixed name the runtime already says: the bus account the process runs as names it. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-delivery-indexer/integration_test.go | 8 +++---- .../cmd/mesh-delivery-indexer/main.go | 2 +- .../cmd/mesh-delivery-indexer/main_test.go | 7 ++++++ .../cmd/mesh-delivery-indexer/objects.go | 23 ++++++++++++------- 4 files changed, 27 insertions(+), 13 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index 16f3ec69..e1d58357 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -481,7 +481,7 @@ var acceptedHeard atomic.Int64 type revisionView struct{ js jetstream.JetStream } func (v revisionView) Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error) { - view, _, _, err := index.TrunkView(ctx, &busObjects{js: v.js, local: index.CatalogRevisionsObjects}) + view, _, _, err := index.TrunkView(ctx, newBusObjects(v.js, testAccount, index.CatalogRevisionsObjects)) if err != nil { return nil, err } @@ -497,7 +497,7 @@ func (v revisionView) Get(ctx context.Context, key string) (jetstream.KeyValueEn // History is the module's value in every trunk revision the objects keep that holds it, oldest first. func (v revisionView) History(ctx context.Context, key string) ([]jetstream.KeyValueEntry, error) { - o := &busObjects{js: v.js, local: index.CatalogRevisionsObjects} + o := newBusObjects(v.js, testAccount, index.CatalogRevisionsObjects) names, err := o.Names(ctx) if err != nil { return nil, err @@ -600,7 +600,7 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before . t.Fatal(err) } // The indexer's objects catalog-revisions, which the grants part creates from its manifest's old objects section. - if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: catalogue.ObjectsBucket(indexerModule, index.CatalogRevisionsObjects)}); err != nil { + if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: newBusObjects(nil, testAccount, index.CatalogRevisionsObjects).bucket()}); err != nil { t.Fatal(err) } modules := revisionView{js: js} @@ -1043,7 +1043,7 @@ func TestIndexerJudgesAnUnreadableVerdictAgainAtStart(t *testing.T) { // 1923/AC if err != nil { t.Fatal(err) } - if err := index.KeepRevision(ctx, &busObjects{js: js, local: index.CatalogRevisionsObjects}, rev, objects); err != nil { + if err := index.KeepRevision(ctx, newBusObjects(js, testAccount, index.CatalogRevisionsObjects), rev, objects); err != nil { t.Fatal(err) } kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ToolchainsBucket("mesh-build-go"), History: 1}) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 610725e7..9faf2d03 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -378,7 +378,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The view of the trunk is the newest trunk revision of the catalog graph the objects keep (milestone 47/G, ADR 0656 §6: // one form of the fact), read once here and kept as a revision after every write; objects not on the bus yet, or none // kept, start it empty, and the first start's catch-up takes the trunk in once the gate's source is read (catchUp). - revisions := &busObjects{js: js, local: index.CatalogRevisionsObjects} + revisions := newBusObjects(js, cred, index.CatalogRevisionsObjects) modules, graph, at, err := index.TrunkView(ctx, revisions) switch { case errors.Is(err, process.ErrNotYet): diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go index 1450b161..11264cd3 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go @@ -79,6 +79,9 @@ func (m *memStore) Delete(_ context.Context, k string, _ uint64) error { delete( const head = "0123456789abcdef0123456789abcdef01234567" +// testAccount is the bus account the tests run the indexer as. +var testAccount = process.Account{Node: "n1", Module: "mesh-delivery-indexer"} + func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part) // Review round 2: the indexer's views emitter writes its finding view-unsaid (Emitter.Findings set). if em := viewsEmitter(context.Background(), nil, process.Account{}, nil, &process.AwaitedBucket{}, nil, t.Logf); em.Findings == nil { @@ -143,6 +146,10 @@ func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part) t.Errorf("answered leaves %v, presence asked %v", held.vals, asked) } }) + // Milestone 47/G: the object buckets are the module's own by the name its account runs as, never a fixed name. + if b := newBusObjects(nil, process.Account{Module: "renamed-indexer"}, index.CatalogRevisionsObjects).bucket(); b != "renamed-indexer_catalog-revisions" { + t.Errorf("the catalog graph revisions open as %s, want the account's module's", b) + } t.Run("not a forge message: terminated and logged", func(t *testing.T) { // 1238 (never redelivered) var logs bytes.Buffer st := &memStore{vals: map[string][]byte{}} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go index bb6ae703..025301db 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go @@ -20,26 +20,33 @@ import ( "mesh-delivery-indexer/internal/index" ) -// indexerModule is the module this program runs as, whose object buckets it opens by their local names. -const indexerModule = "mesh-delivery-indexer" - // busObjects is one of the indexer's own object buckets in the old `objects` section (novox/hq ADR 0619 §2), which the // grants part creates: it is opened at its first use and kept; until the bucket exists every call is an error saying so, // and the next call opens it again (no timer: the next revision or tool call asks). type busObjects struct { - js jetstream.JetStream - local string - mu sync.Mutex - o jetstream.ObjectStore + js jetstream.JetStream + // module is the module this program runs as, read from its bus account (process.Account.Module), never a fixed name. + module string + local string + mu sync.Mutex + o jetstream.ObjectStore } +// newBusObjects is the object bucket of this local name of the module the account runs as. +func newBusObjects(js jetstream.JetStream, cred process.Account, local string) *busObjects { + return &busObjects{js: js, module: cred.Module, local: local} +} + +// bucket is the bus's name of the object bucket: the module's own name and the local name. +func (b *busObjects) bucket() string { return catalogue.ObjectsBucket(b.module, b.local) } + func (b *busObjects) open(ctx context.Context) (jetstream.ObjectStore, error) { b.mu.Lock() defer b.mu.Unlock() if b.o != nil { return b.o, nil } - o, err := b.js.ObjectStore(ctx, catalogue.ObjectsBucket(indexerModule, b.local)) + o, err := b.js.ObjectStore(ctx, b.bucket()) if errors.Is(err, jetstream.ErrBucketNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) { return nil, fmt.Errorf("the object bucket %s is not on the bus yet (the grants part creates it from the manifest's objects): %w", b.local, process.ErrNotYet) } From deb88b17b96775519d30e99b927ea3fa7d839ed6 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 9 Nov 2026 21:17:00 +0100 Subject: [PATCH 12/15] The indexer answers the seat mesh-catalog's verb revision in its own runtime, to exactly the parts that declare it; modules ends as a first view (milestone 47/G W4.4) ADR 0663: the catalog half is read through revision, answered by the tools bundle from the catalog graph revisions before it asks anything, to the parts whose entry in the newest trunk revision declares seat:mesh-catalog.revision in invokes, the holder refused too; none kept, every reader waits. The first view modules, its callers list and its tool leave the manifest; the assigner declares revision instead (its code moves in W5.10); the grants part keeps its read of modules until W8. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-delivery-assigner/manifest_test.go | 3 +- modules/mesh-delivery-assigner/module.json | 2 +- .../cmd/mesh-delivery-indexer-tools/main.go | 6 +- .../internal/index/revision_verb.go | 102 ++++++++++++++++++ .../internal/index/toolchains_test.go | 38 +++++++ .../internal/index/views.go | 5 +- .../mesh-delivery-indexer/manifest_test.go | 28 +++-- modules/mesh-delivery-indexer/module.json | 5 +- 8 files changed, 174 insertions(+), 15 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/revision_verb.go diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index c1ceeaea..856f996a 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -126,7 +126,8 @@ func TestManifest(t *testing.T) { if !slices.Equal(m.Consumes, []string{"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-indexer.views.graph", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", buckets.TellerModule + "." + tellerevents.AskEnded, "node-engine.manifest-reads-changed"}) { t.Errorf("consumes = %v, want the indexer's views of graph, manifests and modules, the teller's ask-ended and the engines' reads changed", m.Consumes) } - if want := []string{"mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", + if want := []string{"mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests", + "seat:mesh-catalog.revision", // ADR 0663 §2: which modules exist "seat:mesh-catalog.seat-definitions", // ADR 0636 §4: the seat list "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone "seat:node-backup.backed-up", "seat:node-backup.now", diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index e594b25c..729c8655 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -50,7 +50,7 @@ "invokes": [ "mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests", - "mesh-delivery-indexer.modules", + "seat:mesh-catalog.revision", "seat:mesh-catalog.seat-definitions", "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 2889ddc7..7b2ba739 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -283,8 +283,12 @@ func main() { acts := meshhandactsclient.Recorder{Ask: stdio.Ask} tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts) // The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go. - trunk := index.TrunkStates{Objects: bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)}, Under: keptSource} + revisions := bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)} + trunk := index.TrunkStates{Objects: revisions, Under: keptSource} tools = append(tools, catalogTools(trunk.State)...) + // The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly + // the parts whose entry declares it (novox/hq ADR 0663). + tools = append(tools, index.RevisionTool(revisions, index.Module)) tools = append(tools, viewTools(os.Getenv, trunk.State, time.Now)...) tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, keptRead{stdio.State(index.OwnBuildStateName)}, asks, time.Now)...) tools = append(tools, index.FingerprintsTools(asks, time.Now, func(what, why string) error { diff --git a/modules/mesh-delivery-indexer/internal/index/revision_verb.go b/modules/mesh-delivery-indexer/internal/index/revision_verb.go new file mode 100644 index 00000000..82ffac84 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/revision_verb.go @@ -0,0 +1,102 @@ +package index + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "strconv" + "strings" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// RevisionInvoke is how a part declares the seat mesh-catalog's verb revision in its manifest's invokes, the one way a +// part reads the catalog half (novox/hq ADR 0663 §2). +const RevisionInvoke = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbRevision + +// RevisionCallers are the parts the newest trunk revision's entries say declare the verb revision in their invokes, +// sorted, the holder itself never among them (ADR 0663 §2: granted to exactly the parts that declare it). +func RevisionCallers(entries []meshcatalog.CatalogEntry, holder string) stdio.Callers { + var out stdio.Callers + for _, e := range entries { + if e.Module != holder && slices.Contains(e.Invokes, RevisionInvoke) { + out = append(out, e.Module) + } + } + slices.Sort(out) + return out +} + +// Revision answers the seat mesh-catalog's verb revision from the indexer's own objects (ADR 0663 §1, §3): one catalog +// graph revision, the trunk's newest when the input names none, a kept trunk revision by its number, or a candidate's by +// its group and heads, with one page of its entries whole, by module name. The caller is the runtime's proof: it is +// answered only when the newest trunk revision's entry of its module declares the verb in invokes, and the holder is +// refused too. With no trunk revision kept yet nobody is answered and the reader waits (ErrNoRevision), never an empty +// catalog. +func Revision(ctx context.Context, o Objects, holder, caller string, in meshcatalog.RevisionInput) (meshcatalog.RevisionAnswer, error) { + var out meshcatalog.RevisionAnswer + newest, ok, err := newestTrunk(ctx, o) + if err != nil { + return out, fmt.Errorf("the catalog graph revisions cannot be read: %w", err) + } + if !ok { + return out, ErrNoRevision + } + _, trunkEntries, err := ReadRevision(ctx, o, RevisionObject(newest)) + if err != nil { + return out, err + } + if err := views.Refusal(meshcatalog.Seat+"."+meshcatalog.VerbRevision, RevisionCallers(trunkEntries, holder), caller); err != nil { + return out, fmt.Errorf("%w (novox/hq ADR 0663 §2: a part reads it by declaring %s in invokes)", err, RevisionInvoke) + } + name := RevisionObject(newest) + switch { + case in.Group != "" || in.Heads != "": + if in.Group == "" || in.Heads == "" { + return out, fmt.Errorf("a candidate's revision is named by its group and its heads, both") + } + name = RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: in.Group, Heads: in.Heads}) + case in.Revision != 0 && in.Revision != newest.Revision: + name = "trunk." + strconv.FormatInt(in.Revision, 10) + } + rev, entries := newest, trunkEntries + if name != RevisionObject(newest) { + if rev, entries, err = ReadRevision(ctx, o, name); err != nil { + return out, err + } + if in.Revision != 0 && rev.Revision != in.Revision { + return out, fmt.Errorf("the candidate %s.%s keeps its newest revision only, %d, not %d", in.Group, in.Heads, rev.Revision, in.Revision) + } + } + slices.SortFunc(entries, func(a, b meshcatalog.CatalogEntry) int { return strings.Compare(a.Module, b.Module) }) + out.Revision, out.Entries = rev, []meshcatalog.CatalogEntry{} + size := 0 + for _, e := range entries { + if e.Module <= in.After { + continue + } + data, _ := json.Marshal(e) + if len(out.Entries) > 0 && size+len(data) > views.PageBytes { + out.Next = out.Entries[len(out.Entries)-1].Module + break + } + size += len(data) + out.Entries = append(out.Entries, e) + } + return out, nil +} + +// RevisionTool is the verb revision as the tool mesh-catalog.revision of the indexer's own runtime (ADR 0663 §1): the +// tool runner describes it with the seat's definition, so it carries no description or input of its own. +func RevisionTool(o Objects, holder string) stdio.Tool { + return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbRevision, Run: func(args map[string]any) (any, error) { + var in meshcatalog.RevisionInput + if raw, err := json.Marshal(args); err == nil { + _ = json.Unmarshal(raw, &in) + } + return Revision(context.Background(), o, holder, stdio.CallerOf(args), in) + }} +} diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index d20d2798..5de024cf 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -549,6 +549,44 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // if _, err := (TrunkStates{Objects: newMemObjects(0)}).State("graph").Keys(); !errors.Is(err, ErrNoRevision) { t.Errorf("the bundle's graph with no revision kept answered %v, want it to wait", err) } + // W4.4, ADR 0663: the seat mesh-catalog's verb revision answers, from the indexer's own objects, exactly the parts whose + // entry in the newest trunk revision declares it, the holder refused too; a candidate's by its group and heads; none + // kept, everyone waits. + verbObjects := newMemObjects(0) + keep := func(rev meshcatalog.CatalogRevision, declares map[string]bool) { + entries := map[string][]byte{} + for _, name := range []string{"avahi", Module, "mesh-grants", "node-tools"} { + e := meshcatalog.CatalogEntry{Module: name, Repository: catalogueRepo} + if declares[name] { + e.Invokes = []string{RevisionInvoke} + } + data, _ := json.Marshal(e) + digest := fmt.Sprintf("%s-%d-%s", rev.Kind, rev.Revision, name) + entries[digest] = data + rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: name, Digest: digest}) + } + if err := KeepRevision(context.Background(), verbObjects, rev, entries); err != nil { + t.Fatal(err) + } + } + if _, err := Revision(context.Background(), verbObjects, Module, "n1.node-tools", meshcatalog.RevisionInput{}); !errors.Is(err, ErrNoRevision) { + t.Errorf("revision with none kept answered %v, want it to wait", err) + } + keep(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, map[string]bool{"node-tools": true, Module: true}) + keep(meshcatalog.CatalogRevision{Revision: 4, Kind: "candidate", Group: "feat/x", Heads: "0123456789ab"}, map[string]bool{"node-tools": true, "mesh-grants": true}) + for caller, answered := range map[string]bool{"n1.node-tools": true, "n1.mesh-grants": false, "n1." + Module: false, "person.someone": false, "unproven": false} { + got, err := Revision(context.Background(), verbObjects, Module, caller, meshcatalog.RevisionInput{}) + if (err == nil) != answered || (answered && (got.Revision.Revision != 3 || len(got.Entries) != 4 || got.Next != "")) { + t.Errorf("revision to %s answered %+v (%v), want answered %v with trunk revision 3 whole", caller, got, err, answered) + } + } + if got, err := Revision(context.Background(), verbObjects, Module, "n1.node-tools", meshcatalog.RevisionInput{Group: "feat/x", Heads: "0123456789ab", After: "mesh-grants"}); err != nil || + got.Revision.Kind != "candidate" || len(got.Entries) != 1 || got.Entries[0].Module != "node-tools" { + t.Errorf("a candidate's revision after mesh-grants answered %+v (%v), want its one entry node-tools", got, err) + } + if _, err := Revision(context.Background(), verbObjects, Module, "n1.node-tools", meshcatalog.RevisionInput{Revision: 9}); err == nil { + t.Error("a trunk revision not kept was answered") + } // ADR 0614 §2, §3: the graph names what each module invokes and serves, and leaves out, over the whole graph, the // verb edges of two modules calling each other, from either side; a third calling one of them keeps its edge. invoking := func(name, invokes string) mod { diff --git a/modules/mesh-delivery-indexer/internal/index/views.go b/modules/mesh-delivery-indexer/internal/index/views.go index 88bebe00..a428ba9d 100644 --- a/modules/mesh-delivery-indexer/internal/index/views.go +++ b/modules/mesh-delivery-indexer/internal/index/views.go @@ -11,10 +11,11 @@ import ( const MarksBucket = Module + "_" + views.MarksState // FirstViewTools are the indexer's first-view tools (ADR 0621 §1, ADR 0622 §1, ADR 0631 §1): the tool, the state it answers, and -// the words of what that state holds. Each answers only its callers list (CallersEnv), which the manifest writes. +// the words of what that state holds. Each answers only its callers list (CallersEnv), which the manifest writes. The fourth +// first view, of which modules exist, is the seat mesh-catalog's verb revision (ADR 0663 §1, RevisionTool), granted by +// declaration, not by a list. var FirstViewTools = []ViewedTool{ {Tool: "manifests", State: "manifests", Words: "every manifest the indexer keeps, by its digest."}, - {Tool: "modules", State: "modules", Words: "every module the indexer holds, its fingerprint, inputs and manifest digest."}, {Tool: "verdicts", State: VerdictsState, Words: "the judge's verdict on each manifest: accepted, waiting or refused."}, {Tool: "graph", State: graph.State, Words: "the module graph: each module's build, provision and verb edges, whose verb edges a want waits on."}, } diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 25a07565..c94f7d6d 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -175,7 +175,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. The same bundle // reads the verdicts state (indexer_verdicts) and asks the indexer to judge again through the takeins state // (indexer_judge; issue 774, clause 3). 1966/AC11: indexer_old_sections reads the count of the old sections (state oldsections). - // ADR 0621 §1, ADR 0622 §1, ADR 0631 §1: the first-view tools manifests, modules, verdicts and graph, and design 25 § 12.2's read + // ADR 0621 §1, ADR 0622 §1, ADR 0631 §1, ADR 0663 §1: the first-view tools manifests, verdicts and graph (modules is the seat + // verb revision's, served by the claim), and design 25 § 12.2's read // tools, all of the tools bundle, which runs in a runtime of its own on the indexer's account (runs-as). want := []string{"indexer_take_in", "indexer_old_sections", "indexer_verdicts", "indexer_judge"} for _, v := range append(append([]index.ViewedTool{}, index.FirstViewTools...), index.ReadTools...) { @@ -189,12 +190,13 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build index.SeatDefinitionTool, index.SeatDefinitionsAgainTool, index.OutOfStepTool, // DLV-I19 index.HoldsTool, index.EngineReadsTool, // ADR 0632 §2: the mesh-wide holds and the engines' reads they weigh index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TryCandidatesTool, index.DropTryCandidateTool, index.RecoverTriesTool, index.TakeInMergeTool, index.FingerprintsAgainTool) // issue 1966; 1585/T12, T13 (ADR 0633 §5) - if !reflect.DeepEqual(m.Tools, want) || len(want) != 40 { + if !reflect.DeepEqual(m.Tools, want) || len(want) != 40 || slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) } // Every read tool of the indexer a reader of this repository invokes is one it serves (the checker's seventh first // view, indexer_read_seat_definitions, was invoked and not served). readers, _ := filepath.Glob("../*/module.json") + var declaring []string for _, p := range readers { raw, err := os.ReadFile(p) if err != nil { @@ -207,24 +209,36 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build t.Fatalf("%s: %v", p, err) } for _, inv := range r.Invokes { + // The grants part reads modules until its own subtask (W8) moves it to revision; the join removes this line. + if p == "../mesh-grants/module.json" && inv == "mesh-delivery-indexer.modules" { + continue + } if tool, ok := strings.CutPrefix(inv, "mesh-delivery-indexer."); ok && !slices.Contains(m.Tools, tool) { t.Errorf("%s invokes %s, which the indexer does not serve", p, inv) } + if inv == index.RevisionInvoke { + declaring = append(declaring, filepath.Base(filepath.Dir(p))) + } } } + // ADR 0663 §2: the assigner reads which modules exist by declaring the seat's verb revision (the grants part too, from W8). + if !slices.Contains(declaring, "mesh-delivery-assigner") { + t.Errorf("the parts of this repository declaring %s are %v, want the assigner", index.RevisionInvoke, declaring) + } if m.RunsAs != m.Module || m.SecretsOwner != m.RunsAs { t.Errorf("runs-as = %q, secrets-owner %q: the tools bundle runs on the indexer's own account (ADR 0621 §1)", m.RunsAs, m.SecretsOwner) } // ADR 0621 §2, ADR 0622 §1, ADR 0631 §1: each first-view tool answers exactly its readers; the indexer is on none, and // graph answers the assigner alone. - callers := map[string]string{"manifests": "mesh-grants,node-tools,mesh-delivery-assigner", "modules": "mesh-grants,node-tools,mesh-delivery-assigner", "verdicts": "mesh-grants", + // ADR 0663 §2: revision has no callers list; it answers the parts whose entry declares it. + callers := map[string]string{"manifests": "mesh-grants,node-tools,mesh-delivery-assigner", "verdicts": "mesh-grants", "graph": "mesh-delivery-assigner"} for _, a := range m.Build.Artifacts { if a.Name != "tools" { continue } if len(a.Env) != len(callers) { - t.Errorf("tools env = %v, want only the four callers lists", a.Env) + t.Errorf("tools env = %v, want only the three callers lists", a.Env) } for tool, list := range callers { if got := a.Env[views.CallersEnv(tool)]; got != list { @@ -254,10 +268,10 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build } } // ADR 0604 §2, ADR 0636 §4: it holds the seat mesh-catalog, serving its four verbs and seat-definitions from its tools - // bundle (which stays without a request subject). + // bundle (which stays without a request subject); ADR 0663 §1: and revision, in the same runtime of its own. if len(m.Claims) != 1 || m.Claims[0].Name != index.CatalogSeat || m.Claims[0].Scope != catalogue.ScopeMesh || - !slices.Equal(m.Claims[0].Serves, []string{"modules", "module", "provides", "dependents", index.SeatDefinitionsVerb}) { - t.Errorf("claims = %+v, want mesh-catalog with its five verbs", m.Claims) + !slices.Equal(m.Claims[0].Serves, []string{"modules", "module", "provides", "dependents", index.SeatDefinitionsVerb, "revision"}) { + t.Errorf("claims = %+v, want mesh-catalog with its six verbs", m.Claims) } for _, e := range []string{index.SeatDefinitionChangedEvent, index.SeatDefinitionRemovedEvent} { if slices.Contains(m.Emits, e) { diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index ef26c524..b928b5e3 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -12,7 +12,8 @@ "module", "provides", "dependents", - "seat-definitions" + "seat-definitions", + "revision" ] } ], @@ -22,7 +23,6 @@ "indexer_verdicts", "indexer_judge", "manifests", - "modules", "verdicts", "graph", "indexer_read_manifests", @@ -195,7 +195,6 @@ ], "env": { "MESH_VIEW_CALLERS_MANIFESTS": "mesh-grants,node-tools,mesh-delivery-assigner", - "MESH_VIEW_CALLERS_MODULES": "mesh-grants,node-tools,mesh-delivery-assigner", "MESH_VIEW_CALLERS_VERDICTS": "mesh-grants", "MESH_VIEW_CALLERS_GRAPH": "mesh-delivery-assigner" } From b2b449c913dc21b3083287f0b719411af57344d4 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 9 Nov 2026 21:21:10 +0100 Subject: [PATCH 13/15] The indexer says filling, full and room of its catalog graph revisions and holds a merge while they are full (milestone 47/G W4.6) Design 47, Two graphs: a keep that would pass the store's cap refuses the next revision and says full, naming the source revision held pending with the reason objects full (Pending.Held, which indexer_pending shows); a keep crossing 80 % says filling; once something removes revisions, RoomAgain says room below the filling mark and evaluates pending again (no timer). Each event is said once by its duplicate id. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-indexer/main.go | 1 + .../cmd/mesh-delivery-indexer/objects.go | 20 ++++++ .../internal/index/candidates.go | 3 + .../internal/index/catalog_revisions.go | 66 +++++++++++++++++++ .../internal/index/pending.go | 26 +++++++- .../internal/index/pending_test.go | 52 ++++++++++++++- 6 files changed, 166 insertions(+), 2 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 9faf2d03..24d386d0 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -447,6 +447,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro SeatDefinitions: awaitedStore{b: seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats, Message: messageVia(calls.CallKept), Revisions: revisions, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1) SayCatalogRevision: sayCatalogRevision(conn, cred), + SayFill: sayFill(conn, cred), // The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear; // until then every merge is taken in at once, as before. Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.PendingBucket, "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)}, diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go index 025301db..7b250e47 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/objects.go @@ -144,3 +144,23 @@ func sayCatalogRevision(conn *nats.Conn, cred process.Account) func(context.Cont return conn.PublishMsg(msg) } } + +// sayFill publishes the seat mesh-catalog's event filling, full or room of the catalog graph revisions under the duplicate +// id given (milestone 47/G W4.6), so one said twice is heard once. +func sayFill(conn *nats.Conn, cred process.Account) func(context.Context, string, string, meshcatalog.Fill) error { + return func(_ context.Context, event, id string, f meshcatalog.Fill) error { + data, err := json.Marshal(f) + if err != nil { + return err + } + msg := nats.NewMsg(seatEventSubject(event)) + msg.Header.Set("x-event-id", id) + msg.Header.Set("x-source", cred.Module) + msg.Header.Set("x-node", cred.Node) + msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano)) + msg.Header.Set("content-type", "application/json") + msg.Header.Set(nats.MsgIdHdr, id) + msg.Data = data + return conn.PublishMsg(msg) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 5d9888dd..5fcfa42b 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -102,6 +102,9 @@ type Indexer struct { Revisions Objects // SayCatalogRevision says the seat mesh-catalog's event catalog-revision for a revision kept; nil says nothing. SayCatalogRevision func(ctx context.Context, e meshcatalog.CatalogRevisionEvent) error + // SayFill says the seat mesh-catalog's event filling, full or room of the catalog graph revisions, under the duplicate + // id given (W4.6). + SayFill func(ctx context.Context, event, id string, f meshcatalog.Fill) error // Graph is the module graph's state (GraphBucket, graph.go; novox/hq ADR 0603 §2); nil writes no graph. Graph Store // Files asks the forge for every blob under each directory at a commit, by directory then path relative to it, each diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index 408b5951..b044b4bc 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -268,10 +268,22 @@ func (ix *Indexer) keepEntries(ctx context.Context, entries map[string]meshcatal if (any && len(changed) == 0) || (!any && len(entries) == 0) { return nil // nothing moved; an empty first revision is never kept, so the first start's catch-up still runs } + used, capacity, fillErr := ix.Revisions.Fill(ctx) + if fillErr == nil && capacity > 0 && used+newBytes(ctx, ix.Revisions, rev, objects) > capacity { + // W4.6 (design 47, Two graphs): a full store refuses the next revision, said once per source revision, and the + // take-in that made it holds its pending merge with the reason objects full until room is said. + refused := fmt.Sprintf("the next revision; source revision %d held pending with the reason %s", src, HeldObjectsFull) + ix.logf("catalog graph: revision %d of source revision %d is not kept: %s is full (%d of %d bytes)", rev.Revision, src, CatalogRevisionsObjects, used, capacity) + ix.sayFill(ctx, meshcatalog.EventFull, fmt.Sprintf("%d", src), meshcatalog.Fill{Bytes: used, Cap: capacity, Refused: refused}) + return fmt.Errorf("revision %d of source revision %d: %w", rev.Revision, src, ErrObjectsFull) + } if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err) return err } + if after, capacity, err := ix.Revisions.Fill(ctx); fillErr == nil && err == nil && capacity > 0 && used*5 < capacity*4 && after*5 >= capacity*4 { + ix.sayFill(ctx, meshcatalog.EventFilling, fmt.Sprintf("%d", rev.Revision), meshcatalog.Fill{Bytes: after, Cap: capacity}) + } ix.logf("catalog graph: revision %d of source revision %d kept, %d entries, %d changed", rev.Revision, src, len(rev.Entries), len(changed)) if ix.SayCatalogRevision != nil { if err := ix.SayCatalogRevision(ctx, meshcatalog.CatalogRevisionEvent{V: CatalogEntryLayout, Revision: rev.Revision, Kind: rev.Kind, @@ -282,6 +294,60 @@ func (ix *Indexer) keepEntries(ctx context.Context, entries map[string]meshcatal return nil } +// HeldObjectsFull is the reason a pending merge is held while the catalog graph revisions are full (Pending.Held). +const HeldObjectsFull = "objects full" + +// ErrObjectsFull is a revision refused because the catalog graph revisions are at their cap. +var ErrObjectsFull = errors.New("the catalog graph revisions are full") + +// newBytes is the size of what keeping rev adds: its list and each entry the bucket does not hold yet. +func newBytes(ctx context.Context, o Objects, rev meshcatalog.CatalogRevision, entries map[string][]byte) int64 { + held, _ := o.Names(ctx) + have := make(map[string]bool, len(held)) + for _, n := range held { + have[n] = true + } + list, _ := json.Marshal(rev) + size := int64(len(list)) + for _, ref := range rev.Entries { + if !have[EntryObject(ref.Digest)] { + size += int64(len(entries[ref.Digest])) + } + } + return size +} + +// sayFill says the seat's event filling, full or room of the catalog graph revisions, under the duplicate id of the +// event and what it is about, so the same one said twice is heard once. +func (ix *Indexer) sayFill(ctx context.Context, event, about string, f meshcatalog.Fill) { + if ix.SayFill == nil { + return + } + at := ix.now().UTC() + f.At, f.Store = &at, CatalogRevisionsObjects + if err := ix.SayFill(ctx, event, event+"-"+about, f); err != nil { + ix.logf("catalog graph: %s is not said: %v", event, err) + } +} + +// RoomAgain says room and evaluates pending again when a merge held with the reason objects full finds the store +// below its filling mark (80 % of its cap): called after anything removes catalog graph revisions (no timer). It answers whether room was said. +func (ix *Indexer) RoomAgain(ctx context.Context) (bool, error) { + if ix.Revisions == nil || ix.Pending == nil { + return false, nil + } + entries, err := ReadPending(ctx, ix.Pending, ix.logf) + if err != nil || !slices.ContainsFunc(entries, func(e PendingEntry) bool { return e.Value.Held == HeldObjectsFull }) { + return false, err + } + used, capacity, err := ix.Revisions.Fill(ctx) + if err != nil || (capacity > 0 && used*5 >= capacity*4) { + return false, err // room is said below the filling mark, 80 %, so one refusal is not followed by another at once + } + ix.sayFill(ctx, meshcatalog.EventRoom, fmt.Sprintf("%d", used), meshcatalog.Fill{Bytes: used, Cap: capacity}) + return true, ix.PendingAgain(ctx) // room in catalog-revisions: pending evaluated again +} + // TrunkView is the indexer's view of the trunk read from the newest trunk revision the bucket keeps (the lead's ruling // on milestone 47/G finding 1; ADR 0656 §6, one form of the fact): a store in memory holding each module's value as // ModuleValueOf reads it back from its entry, under its modules key, a store in memory of each module's node of the module diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index c732188f..17b32ef0 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -95,6 +95,9 @@ type Pending struct { // UnreadSince is when the message was first found unread: past meshconditionsclient.CallUnansweredAfter, with the forge // heard since, it is the finding call-unanswered (unreadcall.go, ADR 0668 §3). UnreadSince time.Time `json:"unread-since,omitzero"` + // Held is why a ready merge is not taken in although its revision names this build: objects full (HeldObjectsFull), + // until room is said (RoomAgain). + Held string `json:"held,omitempty"` } // PendingEntry is one key of the pending state as read: its key, revision, value and the merged message decoded. For a @@ -782,11 +785,32 @@ func (ix *Indexer) evaluate(ctx context.Context, own string, start bool) (Applie if errors.Is(err, errVerdictsNotYet) { return a, "the verdicts state is not on the bus yet: the revision waits for it, taken in when it binds", nil } + if errors.Is(err, ErrObjectsFull) { + ix.holdFull(ctx, ready) + return a, ix.waitingLine(w) + "; held: " + HeldObjectsFull, nil + } return a, ix.waitingLine(w), err } return Applied{}, ix.waitingLine(w), ix.holdFor(ctx, ready, names, own, dry) } +// holdFull marks the ready merges held with the reason objects full (Pending.Held), read again at their revisions now +// (the take-in's claim moved them), so indexer_pending shows why they wait. +func (ix *Indexer) holdFull(ctx context.Context, ready []PendingEntry) { + entries, err := ReadPending(ctx, ix.Pending, ix.logf) + if err != nil { + ix.logf("take-in: the merges held by a full %s are not marked: %v", CatalogRevisionsObjects, err) + return + } + var held []PendingEntry + for _, e := range entries { + if slices.ContainsFunc(ready, func(r PendingEntry) bool { return r.Key == e.Key }) && e.Value.Held != HeldObjectsFull { + held = append(held, e) + } + } + ix.markSaid(ctx, held, nil, func(p *Pending) { p.Held = HeldObjectsFull }) +} + // holdLost holds a revision whose newest merge of a repository cannot be read (the forge bridge's stream no longer holds // it, or the bus refuses the read): said once on take-in.pending with the lost keys (TakeIn.Lost), and on each lost entry // (Pending.Lost, the reason, which indexer_pending shows), until a newer merge of its repository covers it or @@ -886,7 +910,7 @@ func (ix *Indexer) dry() *dryRun { Requirements: nil, Findings: nil, SeatDefinitions: nil, ReadFile: ix.ReadFile, SaySeatDefinition: nil, Holds: nil, Counted: nil, AskEngines: nil, HoldBounds: nil, OwnBuildState: nil, underSeats: true, // A dry run keeps no catalog graph revision (catalog_revisions.go): its revision is the take-in's to keep. - Revisions: nil, SayCatalogRevision: nil} + Revisions: nil, SayCatalogRevision: nil, SayFill: nil} x.Logf = func(format string, args ...any) { ix.logf("take-in, computed and not written: "+format, args...) } x.Store, d.candidates = over(ix.Store) x.Modules, d.modules = over(ix.Modules) diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index 6a82b105..8487b9fd 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -744,6 +744,56 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) } } }) + t.Run("W4.6 a full store holds the merge with the reason objects full, says full once, and room takes it in", func(t *testing.T) { + b := newTakeInBed(t) + b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + named := b.fp(t, Module) + b.ix.ownMu.Lock() + b.ix.ownFP = named + b.ix.ownMu.Unlock() + store := newMemObjects(100) // 90 bytes held by another revision: full for this one + _ = store.Put(ctx, "trunk.0", make([]byte, 90)) + b.ix.Revisions = store + var said []string + b.ix.SayFill = func(_ context.Context, event, id string, f meshcatalog.Fill) error { + said = append(said, event+" "+f.Refused) + return nil + } + if _, _, err := b.ix.TakeInAtStart(ctx); err != nil { + t.Fatal(err) + } + var p Pending + _ = json.Unmarshal(b.pending.vals[b.keys()[0]], &p) + if len(b.keys()) != 1 || p.Held != HeldObjectsFull || len(said) != 1 || !strings.HasPrefix(said[0], "full ") || !strings.Contains(said[0], HeldObjectsFull) { + t.Fatalf("pending %v held %q, said %q: want the merge held with %q and full said", b.keys(), p.Held, said, HeldObjectsFull) + } + if roomed, err := b.ix.RoomAgain(ctx); err != nil || roomed { + t.Fatalf("room while still full: %v %v", roomed, err) + } + _ = store.Delete(ctx, "trunk.0") // the other revision removed: room + store.capacity = 0 + if roomed, err := b.ix.RoomAgain(ctx); err != nil || !roomed || len(b.keys()) != 0 || said[len(said)-1] != "room " { + t.Errorf("room said %v (%v), said %q, pending %v: want room said and the merge taken in", roomed, err, said, b.keys()) + } + // The same take-in into a store it fills to 85 % of its cap says filling once, and is taken in. + kept := store.used() + b2 := newTakeInBed(t) + b2.arrive(t, 2, b2.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + b2.ix.ownMu.Lock() + b2.ix.ownFP = b2.fp(t, Module) + b2.ix.ownMu.Unlock() + b2.ix.Revisions = newMemObjects(kept * 100 / 85) + var filling []meshcatalog.Fill + b2.ix.SayFill = func(_ context.Context, event, _ string, f meshcatalog.Fill) error { + if event == meshcatalog.EventFilling { + filling = append(filling, f) + } + return nil + } + if _, _, err := b2.ix.TakeInAtStart(ctx); err != nil || len(b2.keys()) != 0 || len(filling) != 1 || filling[0].Store != CatalogRevisionsObjects { + t.Errorf("a take-in to 85 %% said filling %+v (%v), pending %v: want it said once and the merge taken in", filling, err, b2.keys()) + } + }) t.Run("T7 a toolchain report evaluates pending again: the revision then names the running build and is taken in", func(t *testing.T) { b := newTakeInBed(t) b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) @@ -970,7 +1020,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", "Unjudged", "JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk", "Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", - "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "Holds", "Counted", "AskEngines", "HoldBounds", "OpenTries", "TryRecovery", "OwnBuildState", "Revisions", "SayCatalogRevision"} { + "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "Holds", "Counted", "AskEngines", "HoldBounds", "OpenTries", "TryRecovery", "OwnBuildState", "Revisions", "SayCatalogRevision", "SayFill"} { named[f] = true } ty := reflect.TypeOf(Indexer{}) From a217a44f054f3434d24f25d74838b5803dee4dff Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 9 Nov 2026 21:27:42 +0100 Subject: [PATCH 14/15] The indexer's catalog graph revisions are read and steered by its tools: one revision, every revision with the store's fill, the candidates' halves, a recompute compared with the kept one, and a drop by hand that says room (milestone 47/G W4.12) Pins and the consumer of the gate's events come with W4.5, which makes them. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-indexer-tools/main.go | 4 + .../cmd/mesh-delivery-indexer/takein.go | 10 +- .../internal/index/pending_test.go | 86 +++++ .../internal/index/revision_tools.go | 326 ++++++++++++++++++ .../internal/index/takein.go | 19 +- .../mesh-delivery-indexer/manifest_test.go | 6 +- modules/mesh-delivery-indexer/module.json | 7 +- 7 files changed, 447 insertions(+), 11 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/revision_tools.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 7b2ba739..574441c6 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -289,6 +289,10 @@ func main() { // The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly // the parts whose entry declares it (novox/hq ADR 0663). tools = append(tools, index.RevisionTool(revisions, index.Module)) + tools = append(tools, index.CatalogRevisionTools(revisions, asks, time.Now, func(what, why string) error { + _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.DropCandidateGraphTool, Verb: "drop-candidate-graph"}) + return err + })...) tools = append(tools, viewTools(os.Getenv, trunk.State, time.Now)...) tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, keptRead{stdio.State(index.OwnBuildStateName)}, asks, time.Now)...) tools = append(tools, index.FingerprintsTools(asks, time.Now, func(what, why string) error { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go index 37822709..3f010eba 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go @@ -26,6 +26,8 @@ type takerIn interface { DropTryCandidate(ctx context.Context, id, by, why string) index.TakenIn RecoverTriesByHand(ctx context.Context, by, why string) index.TakenIn FingerprintsAgain(ctx context.Context, module string) index.TakenIn + RecomputeByHand(ctx context.Context, by, why string) index.TakenIn + DropCandidateGraph(ctx context.Context, group, heads, by, why string) index.TakenIn } // takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the @@ -44,7 +46,13 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream return } asked, r := "take-in", index.TakenIn{} - if a.Fingerprints { + if a.Recompute { + asked, r = "recompute of the catalog graph", ix.RecomputeByHand(ctx, a.By, a.Why) + a.Module = r.Module + } else if a.CandidateGroup != "" && a.Drop { + asked, r = "drop of a candidate's catalog half", ix.DropCandidateGraph(ctx, a.CandidateGroup, a.CandidateHeads, a.By, a.Why) + a.Module = r.Module + } else if a.Fingerprints { asked, r = "computing of fingerprints again", ix.FingerprintsAgain(ctx, a.Module) } else if a.RecoverTries { asked, r = "recovery of the tries", ix.RecoverTriesByHand(ctx, a.By, a.Why) diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index 8487b9fd..86cdaabf 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -18,6 +18,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" "git.novox.be/novox/mesh-delivery/sdk/trunk" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/busretry" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -794,6 +795,91 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) t.Errorf("a take-in to 85 %% said filling %+v (%v), pending %v: want it said once and the merge taken in", filling, err, b2.keys()) } }) + t.Run("W4.12 a candidate's half dropped by hand frees its own entries, says room and takes the held merge in; the read tools list it; a recompute that moves nothing is equal", func(t *testing.T) { + bed := func(store *memObjects) *takeInBed { + b := newTakeInBed(t) + b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) + b.ix.ownMu.Lock() + b.ix.ownFP = b.fp(t, Module) + b.ix.ownMu.Unlock() + b.ix.Revisions = store + return b + } + free := newMemObjects(0) + if _, _, err := bed(free).ix.TakeInAtStart(ctx); err != nil { + t.Fatal(err) + } + size := free.used() // what the merge's revision takes + store := newMemObjects(2 * size) + shared := []byte(`{"module":"b"}`) + if err := KeepRevision(ctx, store, meshcatalog.CatalogRevision{Kind: "candidate", Group: "g1", Heads: "h1", Revision: 1, + Entries: []meshcatalog.EntryRef{{Module: "a", Digest: "da"}, {Module: "b", Digest: "db"}}}, + map[string][]byte{"da": make([]byte, size+10), "db": shared}); err != nil { + t.Fatal(err) + } + if err := KeepRevision(ctx, store, meshcatalog.CatalogRevision{Kind: "candidate", Group: "g2", Heads: "h2", Revision: 1, + Entries: []meshcatalog.EntryRef{{Module: "b", Digest: "db"}}}, map[string][]byte{"db": shared}); err != nil { + t.Fatal(err) + } + b := bed(store) + var said []string + b.ix.SayFill = func(_ context.Context, event, _ string, _ meshcatalog.Fill) error { said = append(said, event); return nil } + if _, _, err := b.ix.TakeInAtStart(ctx); err != nil || len(b.keys()) != 1 { + t.Fatalf("pending %v (%v): want the merge held for objects full", b.keys(), err) + } + var recorded []string + asks := 0 + tools := map[string]stdio.Tool{} + for _, tool := range CatalogRevisionTools(store, func(_ string, value []byte) (TakeInAsk, error) { + asks++ + a, err := DecodeTakeIn(value) + if err != nil { + return a, err + } + var r TakenIn + if a.Recompute { + r = b.ix.RecomputeByHand(ctx, a.By, a.Why) + } else { + r = b.ix.DropCandidateGraph(ctx, a.CandidateGroup, a.CandidateHeads, a.By, a.Why) + } + a.Result = &r + return a, nil + }, time.Now, func(what, why string) error { + recorded = append(recorded, what) + if why == "unrecorded" { + return errors.New("mesh-hand-acts did not answer") + } + return nil + }) { + tools[tool.Name] = tool + } + if _, err := tools[DropCandidateGraphTool].Run(map[string]any{"group": "g1", "heads": "h1", "why": "unrecorded"}); err == nil || asks != 0 { + t.Fatalf("a drop whose hand act is not recorded: %v, %d asks: want refused before it is asked", err, asks) + } + got, err := tools[DropCandidateGraphTool].Run(map[string]any{"group": "g1", "heads": "h1", "why": "its gate event never came"}) + r, _ := got.(*TakenIn) + if err != nil || r == nil || !r.Stored || !slices.Equal(r.Dropped, []string{"entry.da"}) || len(recorded) != 2 { + t.Fatalf("drop answered %+v (%v), recorded %q: want candidate.g1.h1 dropped with entry.da alone, after its hand act", r, err, recorded) + } + if _, err := store.Get(ctx, "entry.db"); err != nil || len(b.keys()) != 0 || !slices.Contains(said, meshcatalog.EventRoom) { + t.Errorf("entry.db %v, pending %v, said %q: want the shared entry kept, room said and the merge taken in", err, b.keys(), said) + } + list, err := tools[CatalogRevisionsTool].Run(map[string]any{}) + l, _ := list.(RevisionsAnswer) + if err != nil || len(l.Revisions) != 2 || l.Revisions[0].Name != "candidate.g2.h2" || l.Revisions[1].Name != "trunk.1" || !l.Revisions[1].Current || + l.Fill.Cap != 2*size || l.Fill.Bytes != store.used() || len(l.Unlisted) != 0 { + t.Errorf("indexer_catalog_revisions = %+v (%v): want candidate.g2.h2 and the current trunk.1, the fill against its cap", l, err) + } + if c, err := tools[CandidateGraphsTool].Run(map[string]any{}); err != nil || len(c.(RevisionsAnswer).Revisions) != 1 { + t.Errorf("indexer_candidate_graphs = %+v (%v): want g2's alone", c, err) + } + if one, err := tools[CatalogRevisionTool].Run(map[string]any{}); err != nil || one.(map[string]any)["name"] != "trunk.1" { + t.Errorf("indexer_catalog_revision = %v (%v): want the newest trunk revision whole", one, err) + } + if got, err := tools[RecomputeTool].Run(map[string]any{"why": "a check"}); err != nil || !strings.Contains(got.(*TakenIn).Answer, "equal to the kept revision 1") { + t.Errorf("indexer_recompute = %+v (%v): want equal to the kept revision 1", got, err) + } + }) t.Run("T7 a toolchain report evaluates pending again: the revision then names the running build and is taken in", func(t *testing.T) { b := newTakeInBed(t) b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) diff --git a/modules/mesh-delivery-indexer/internal/index/revision_tools.go b/modules/mesh-delivery-indexer/internal/index/revision_tools.go new file mode 100644 index 00000000..afe1860f --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/revision_tools.go @@ -0,0 +1,326 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "regexp" + "slices" + "strconv" + "strings" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// The indexer's tools of its catalog graph revisions (design 47, the indexer's tools; the instruction "tools are how a +// module is spoken to"): indexer_catalog_revision and indexer_catalog_revisions read the objects catalog-revisions, +// indexer_candidate_graphs every candidate's catalog half; indexer_recompute and indexer_drop_candidate_graph steer the +// indexer by an ask in takeins, as indexer_take_in does, since the indexer's process serves no request subject. +const ( + CatalogRevisionTool = "indexer_catalog_revision" + CatalogRevisionsTool = "indexer_catalog_revisions" + CandidateGraphsTool = "indexer_candidate_graphs" + RecomputeTool = "indexer_recompute" + DropCandidateGraphTool = "indexer_drop_candidate_graph" +) + +// RecomputeKey is the takeins key of a recompute by hand: one at a time. +const RecomputeKey = "recompute" + +var candidateToken = regexp.MustCompile(`^[A-Za-z0-9_=-]+$`) + +// NewRecomputeAsk is the value indexer_recompute writes: its key in takeins and the value. +func NewRecomputeAsk(why, by string, at time.Time) (string, []byte, error) { + data, err := json.Marshal(TakeInAsk{V: 1, Recompute: true, Why: strings.TrimSpace(why), By: by, At: at.UTC()}) + return RecomputeKey, data, err +} + +// NewDropCandidateGraphAsk is the value indexer_drop_candidate_graph writes: its key in takeins, +// candidate-graph.., and the value. +func NewDropCandidateGraphAsk(group, heads, why, by string, at time.Time) (string, []byte, error) { + group, heads = strings.TrimSpace(group), strings.TrimSpace(heads) + switch { + case !candidateToken.MatchString(group) || !candidateToken.MatchString(heads): + return "", nil, errors.New("name the candidate by its group and its heads, as indexer_candidate_graphs names them") + case strings.TrimSpace(why) == "": + return "", nil, errors.New("say why (why): a drop is a hand act, recorded through mesh-hand-acts") + } + data, err := json.Marshal(TakeInAsk{V: 1, CandidateGroup: group, CandidateHeads: heads, Drop: true, Why: strings.TrimSpace(why), By: by, At: at.UTC()}) + return "candidate-graph." + group + "." + heads, data, err +} + +// RevisionSummary is one kept catalog graph revision as indexer_catalog_revisions and indexer_candidate_graphs list it. +type RevisionSummary struct { + Name string `json:"name"` + Kind string `json:"kind"` + Revision int64 `json:"revision"` + Source int64 `json:"source"` + Group string `json:"group,omitempty"` + Heads string `json:"heads,omitempty"` + At *time.Time `json:"at,omitempty"` + Entries int `json:"entries"` + // Current says it is the trunk's newest revision, which is kept whatever pins it. + Current bool `json:"current,omitempty"` +} + +// RevisionsAnswer is what indexer_catalog_revisions answers: every kept revision, the entries stored once by digest, the +// store's fill against its cap, and each object it could not read. +type RevisionsAnswer struct { + Revisions []RevisionSummary `json:"revisions"` + Entries int `json:"entries"` + Unlisted []string `json:"unlisted,omitempty"` + Fill meshcatalog.Fill `json:"fill"` + FillUnread string `json:"fill-unread,omitempty"` + Unread []string `json:"unread,omitempty"` +} + +// ListRevisions reads every revision the objects keep, candidates included (kind "candidate" alone when candidates). +func ListRevisions(ctx context.Context, o Objects, candidates bool) (RevisionsAnswer, error) { + out := RevisionsAnswer{Revisions: []RevisionSummary{}} + names, err := o.Names(ctx) + if err != nil { + return out, fmt.Errorf("the objects %s cannot be listed: %w", CatalogRevisionsObjects, err) + } + listed := map[string]bool{} + var newest int64 + entries := []string{} + for _, n := range names { + if strings.HasPrefix(n, "entry.") { + entries = append(entries, n) + continue + } + data, err := o.Get(ctx, n) + var rev meshcatalog.CatalogRevision + if err == nil { + err = json.Unmarshal(data, &rev) + } + if err != nil { + out.Unread = append(out.Unread, n+": "+err.Error()) + continue + } + for _, ref := range rev.Entries { + listed[EntryObject(ref.Digest)] = true + } + if rev.Kind == "trunk" && rev.Revision > newest { + newest = rev.Revision + } + if candidates && rev.Kind != "candidate" { + continue + } + out.Revisions = append(out.Revisions, RevisionSummary{Name: n, Kind: rev.Kind, Revision: rev.Revision, Source: rev.Source, + Group: rev.Group, Heads: rev.Heads, At: rev.At, Entries: len(rev.Entries)}) + } + for i := range out.Revisions { + out.Revisions[i].Current = out.Revisions[i].Kind == "trunk" && out.Revisions[i].Revision == newest + } + slices.SortFunc(out.Revisions, func(a, b RevisionSummary) int { return strings.Compare(a.Name, b.Name) }) + out.Entries = len(entries) + for _, e := range entries { + if !listed[e] { + out.Unlisted = append(out.Unlisted, e) + } + } + slices.Sort(out.Unlisted) + used, capacity, err := o.Fill(ctx) + out.Fill = meshcatalog.Fill{Bytes: used, Cap: capacity, Store: CatalogRevisionsObjects} + if err != nil { + out.FillUnread = err.Error() + } + return out, nil +} + +// removeUnlisted deletes every entry no kept revision lists any more: an entry is stored once and shared, so it goes +// only with the last revision that lists it. It answers the entries removed. +func removeUnlisted(ctx context.Context, o Objects) ([]string, error) { + list, err := ListRevisions(ctx, o, false) + if err != nil { + return nil, err + } + if len(list.Unread) > 0 { + // A revision that cannot be read may list any entry: none is removed rather than one it still needs. + return nil, fmt.Errorf("no entry is removed while a revision cannot be read: %s", strings.Join(list.Unread, "; ")) + } + var removed []string + for _, e := range list.Unlisted { + if err := o.Delete(ctx, e); err != nil { + return removed, fmt.Errorf("%s: %w", e, err) + } + removed = append(removed, e) + } + return removed, nil +} + +// DropCandidateGraph drops a candidate's catalog half by hand (indexer_drop_candidate_graph, design 47: for a candidate +// whose gate event never came), the entries no other revision lists with it, and says room when that takes the store +// below its filling mark, which takes in a merge held for objects full (RoomAgain). +func (ix *Indexer) DropCandidateGraph(ctx context.Context, group, heads, by, why string) TakenIn { + name := RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: group, Heads: heads}) + r := TakenIn{Module: name} + if ix.Revisions == nil { + r.Answer = "the indexer keeps no catalog graph revisions: nothing dropped" + return r + } + if _, err := ix.Revisions.Get(ctx, name); err != nil { + r.Answer = name + " is not kept (" + err.Error() + "): nothing dropped" + return r + } + ix.keepMu.Lock() + err := ix.Revisions.Delete(ctx, name) + var removed []string + if err == nil { + removed, err = removeUnlisted(ctx, ix.Revisions) + } + ix.keepMu.Unlock() + if err != nil { + r.Answer = name + " could not be dropped whole: " + err.Error() + ix.logf("%s", r.Answer) + return r + } + r.Stored, r.Dropped = true, removed + r.Answer = fmt.Sprintf("%s is dropped by %s (%s), with %d entries no other revision lists", name, by, why, len(removed)) + if roomed, err := ix.RoomAgain(ctx); err != nil { + r.Answer += "; room is not evaluated: " + err.Error() + } else if roomed { + r.Answer += "; room said, what waited for it evaluated again" + } + ix.logf("%s", r.Answer) + return r +} + +// RecomputeByHand computes the catalog graph again now from the view of the trunk, as a toolchain report or a merge +// does, and compares it with the newest revision kept: the modules whose entry moved, and the revision kept for them +// (none when the recompute equals the one kept). +func (ix *Indexer) RecomputeByHand(ctx context.Context, by, why string) TakenIn { + r := TakenIn{Module: "catalog graph"} + if ix.Revisions == nil { + r.Answer = "the indexer keeps no catalog graph revisions: nothing recomputed" + return r + } + before, _, err := newestTrunk(ctx, ix.Revisions) + if err != nil { + r.Answer = "the newest catalog graph revision cannot be read: " + err.Error() + return r + } + if err := ix.Recompute(ctx); err != nil { + r.Answer = "the recompute failed: " + err.Error() + return r + } + after, _, err := newestTrunk(ctx, ix.Revisions) + if err != nil { + r.Answer = "the newest catalog graph revision cannot be read after the recompute: " + err.Error() + return r + } + r.Stored = true + if after.Revision == before.Revision { + r.Answer = fmt.Sprintf("recomputed by %s (%s): equal to the kept revision %d", by, why, before.Revision) + } else { + r.Handed = changedEntries(before.Entries, after.Entries) + r.Answer = fmt.Sprintf("recomputed by %s (%s): revision %d differs from the kept %d in %s; revision %d is kept", by, why, + after.Revision, before.Revision, strings.Join(r.Handed, ", "), after.Revision) + } + ix.logf("%s", r.Answer) + return r +} + +// CatalogRevisionTools are the tools of the catalog graph revisions; ask writes an ask in takeins and waits for the +// indexer's answer, record records a hand act through mesh-hand-acts before a drop is asked. +func CatalogRevisionTools(o Objects, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time, record func(what, why string) error) []stdio.Tool { + str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } + asked := func(key string, value []byte, err error) (any, error) { + if err != nil { + return nil, err + } + a, err := ask(key, value) + if err != nil { + return nil, err + } + return a.Result, nil + } + return []stdio.Tool{ + {Name: CatalogRevisionTool, + Description: "One catalog graph revision the indexer keeps in its objects catalog-revisions, whole: its list (kind, number, " + + "source revision, trunk heads, a candidate's group and heads) and every entry it lists, each module as its manifest " + + "declares it at that source revision (bases, packages, toolchains, provides, requires, invokes, serves, fingerprint, " + + "acting rules, inputs). The trunk's newest with no input; revision a kept trunk revision; group and heads a candidate's. " + + "An entry the store lost is an error naming it. indexer_catalog_revisions lists what is kept.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "revision": map[string]any{"type": "integer", "description": "a kept trunk revision's number"}, + "group": map[string]any{"type": "string", "description": "a candidate's group, with heads"}, + "heads": map[string]any{"type": "string", "description": "a candidate's heads, with group"}}}, + Run: func(args map[string]any) (any, error) { + ctx := context.Background() + n, _ := args["revision"].(float64) + name := "" + switch { + case str(args, "group") != "" || str(args, "heads") != "": + name = RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: str(args, "group"), Heads: str(args, "heads")}) + case n > 0: + name = "trunk." + strconv.FormatInt(int64(n), 10) + default: + newest, ok, err := newestTrunk(ctx, o) + if err != nil { + return nil, fmt.Errorf("the catalog graph revisions cannot be read: %w", err) + } + if !ok { + return nil, ErrNoRevision + } + name = RevisionObject(newest) + } + rev, entries, err := ReadRevision(ctx, o, name) + if err != nil { + return nil, err + } + return map[string]any{"name": name, "revision": rev, "entries": entries}, nil + }}, + {Name: CatalogRevisionsTool, + Description: "Every catalog graph revision the indexer keeps in its objects catalog-revisions, trunk and candidate: each " + + "one's name, kind, number, source revision, when, how many entries it lists and whether it is the trunk's newest " + + "(current); how many entries are stored once by digest, those no revision lists (unlisted), and the store's fill " + + "in bytes against its cap (filling is said at 80 %, full refuses the next revision). An object it cannot read is " + + "named in unread. indexer_catalog_revision reads one whole.", + Input: map[string]any{"type": "object", "properties": map[string]any{}}, + Run: func(map[string]any) (any, error) { + return ListRevisions(context.Background(), o, false) + }}, + {Name: CandidateGraphsTool, + Description: "Every candidate's catalog half the indexer keeps (a delivery group's or a pull request's heads merged onto " + + "a trunk source revision): its group, its heads, its revision, the source revision it is merged onto, since when, " + + "and how many entries it lists; and the store's fill. indexer_drop_candidate_graph drops one whose gate event never came.", + Input: map[string]any{"type": "object", "properties": map[string]any{}}, + Run: func(map[string]any) (any, error) { + return ListRevisions(context.Background(), o, true) + }}, + {Name: RecomputeTool, + Description: "Compute the catalog graph again now, as the indexer does on a toolchain report or a merge, from the view of " + + "the trunk (each module's manifest, its bases, the agreed toolchains and the published packages), and compare it with " + + "the newest revision kept: answers equal, or the modules whose entry differs and the revision kept for them.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "why": map[string]any{"type": "string", "description": "why it is recomputed now, in plain words"}}}, + Run: func(args map[string]any) (any, error) { + key, value, err := NewRecomputeAsk(str(args, "why"), stdio.CallerOf(args), now()) + return asked(key, value, err) + }}, + {Name: DropCandidateGraphTool, + Description: "Drop one candidate's catalog half by hand, for a candidate whose gate event never came: its revision and the " + + "entries no other revision lists. A hand act, recorded through mesh-hand-acts before it is asked. When the drop takes " + + "the store below its filling mark, room is said and a merge held for objects full is taken in. indexer_candidate_graphs " + + "names the candidates.", + Input: map[string]any{"type": "object", "required": []string{"group", "heads", "why"}, "properties": map[string]any{ + "group": map[string]any{"type": "string", "description": "the candidate's group"}, + "heads": map[string]any{"type": "string", "description": "the candidate's heads"}, + "why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}}}, + Run: func(args map[string]any) (any, error) { + key, value, err := NewDropCandidateGraphAsk(str(args, "group"), str(args, "heads"), str(args, "why"), stdio.CallerOf(args), now()) + if err != nil { + return nil, err + } + if err := record("dropped the catalog half of the candidate "+str(args, "group")+" at "+str(args, "heads")+" by hand", str(args, "why")); err != nil { + return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was dropped: %w", err) + } + return asked(key, value, nil) + }}, + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/takein.go b/modules/mesh-delivery-indexer/internal/index/takein.go index d6a9d524..31e4ef4e 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein.go +++ b/modules/mesh-delivery-indexer/internal/index/takein.go @@ -66,12 +66,17 @@ type TakeInAsk struct { // packages as they are read (indexer_fingerprints_again, fingerprints.go). Fingerprints bool `json:"fingerprints,omitempty"` // RecoverTries asks the open tries' candidates to be recovered now (indexer_recover_tries). - RecoverTries bool `json:"recover-tries,omitempty"` - Drop bool `json:"drop,omitempty"` - By string `json:"by,omitempty"` - At time.Time `json:"at"` - Result *TakenIn `json:"result,omitempty"` - Answered time.Time `json:"answered,omitzero"` + RecoverTries bool `json:"recover-tries,omitempty"` + // Recompute asks the catalog graph to be computed again now and compared with the newest kept revision (indexer_recompute). + Recompute bool `json:"recompute,omitempty"` + // CandidateGroup and CandidateHeads, with Drop, ask that candidate's catalog half to be dropped (indexer_drop_candidate_graph). + CandidateGroup string `json:"candidate-group,omitempty"` + CandidateHeads string `json:"candidate-heads,omitempty"` + Drop bool `json:"drop,omitempty"` + By string `json:"by,omitempty"` + At time.Time `json:"at"` + Result *TakenIn `json:"result,omitempty"` + Answered time.Time `json:"answered,omitzero"` } // Done says the ask is answered. @@ -86,7 +91,7 @@ func DecodeTakeIn(data []byte) (TakeInAsk, error) { if a.V != 1 { return a, fmt.Errorf("a takeins value of version %d, not 1", a.V) } - if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && a.TryCandidate == "" && !a.RecoverTries && !a.Fingerprints && !a.CatchUp { + if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && a.TryCandidate == "" && !a.RecoverTries && !a.Fingerprints && !a.CatchUp && !a.Recompute && a.CandidateGroup == "" { return a, errors.New("a takeins value naming no module, repository, pending wait, merge or try") } return a, nil diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index c94f7d6d..8707e686 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -189,8 +189,10 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build want = append(want, "indexer_first_views", "indexer_first_view_again", index.ConfigReadersTool, index.BusStateTool, index.RequirementsTool, index.PackageUsersTool, index.SayAcceptedTool, // DLV-F3r, DLV-I10, DLV-I11, DLV-I7, DLV-I8, DLV-I9c index.SeatDefinitionTool, index.SeatDefinitionsAgainTool, index.OutOfStepTool, // DLV-I19 index.HoldsTool, index.EngineReadsTool, // ADR 0632 §2: the mesh-wide holds and the engines' reads they weigh - index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TryCandidatesTool, index.DropTryCandidateTool, index.RecoverTriesTool, index.TakeInMergeTool, index.FingerprintsAgainTool) // issue 1966; 1585/T12, T13 (ADR 0633 §5) - if !reflect.DeepEqual(m.Tools, want) || len(want) != 40 || slices.Contains(m.Tools, "modules") { + index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TryCandidatesTool, index.DropTryCandidateTool, index.RecoverTriesTool, index.TakeInMergeTool, index.FingerprintsAgainTool, // issue 1966; 1585/T12, T13 (ADR 0633 §5) + // milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered. + index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool) + if !reflect.DeepEqual(m.Tools, want) || len(want) != 45 || slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) } // Every read tool of the indexer a reader of this repository invokes is one it serves (the checker's seventh first diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index b928b5e3..28251b25 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -56,7 +56,12 @@ "indexer_drop_try_candidate", "indexer_recover_tries", "indexer_take_in_merge", - "indexer_fingerprints_again" + "indexer_fingerprints_again", + "indexer_catalog_revision", + "indexer_catalog_revisions", + "indexer_candidate_graphs", + "indexer_recompute", + "indexer_drop_candidate_graph" ], "consumes": [ "go-package-registry.published", From 07832b1a2c6d1d7d5ed8a93a8350d530070ab5e2 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 9 Nov 2026 21:29:18 +0100 Subject: [PATCH 15/15] The indexer's whole-module test proves a merge kept as one catalog graph revision, said on the seat, and the recompute and the drop answered by the running indexer (milestone 47/G W4.13) Rows in TestIndexerWritesModulesAtAMerge; the pin rows come with W4.5. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-delivery-indexer/integration_test.go | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index e1d58357..a0860700 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -686,8 +686,14 @@ func (l buildLog) end(t *testing.T, repo, module, path string) { // message on FORGE makes modules. within 20 s, the fingerprint of the merge's tree and the recorded inputs. func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the modules are the merged message's, no build record names them (issue 1851) // The bucket holds only build-agent's record, which no fingerprint reads since R4g (ADR 0530 §1). + var asks jetstream.KeyValue js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds buildLog) { builds.end(t, "novox/mesh-controller", "build-agent", "modules/build-agent") + }, func(ctx context.Context, js jetstream.JetStream) { + var err error + if asks, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.TakeInsBucket, History: 1}); err != nil { + t.Fatal(err) + } }) const merge = "89abcdef0123456789abcdef0123456789abcdef" data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, @@ -710,6 +716,60 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the m if m.Repository != "novox/mesh-delivery" || m.Commit != merge || m.Path != "modules/a" || m.Manifest == "" || m.Fp != "" || m.None != "" { // 1606/AC2 t.Errorf("module = %+v within 20 s, want path modules/a, its manifest, and no value of today's form", m) } + // Milestone 47/G W4.13 (design 47, the indexer whole): the source revision taken in is one catalog graph revision kept + // as objects, said as catalog-revision on the seat's subject, every entry it lists stored once by digest. + objects := newBusObjects(js, testAccount, index.CatalogRevisionsObjects) + list, err := index.ListRevisions(ctx, objects, false) + if err != nil || len(list.Revisions) != 1 || list.Revisions[0].Name != "trunk.1" || !list.Revisions[0].Current || + list.Entries != list.Revisions[0].Entries || len(list.Unlisted) != 0 || list.Fill.Bytes == 0 { + t.Fatalf("revisions kept = %+v (%v): want trunk.1 alone, current, each entry it lists stored once", list, err) + } + events, err := js.Stream(ctx, "EVENTS") + if err != nil { + t.Fatal(err) + } + if msg, err := events.GetLastMsgForSubject(ctx, "mesh.seat.mesh-catalog.event."+meshcatalog.EventCatalogRevision); err != nil || + !strings.Contains(string(msg.Data), `"revision":1`) { + t.Errorf("catalog-revision said: %v (%v): want revision 1", msg, err) + } + // W4.12 through the process: a recompute by hand moves nothing and answers equal; a candidate's half dropped by hand + // leaves the trunk's revision and its entries. + if err := index.KeepRevision(ctx, objects, meshcatalog.CatalogRevision{Kind: "candidate", Group: "g1", Heads: "h1", Revision: 1, + Entries: []meshcatalog.EntryRef{{Module: "x", Digest: "dx"}}}, map[string][]byte{"dx": []byte(`{"module":"x"}`)}); err != nil { + t.Fatal(err) + } + answer := func(key string, value []byte) index.TakenIn { + t.Helper() + w, err := asks.Watch(ctx, key, jetstream.UpdatesOnly()) + if err != nil { + t.Fatal(err) + } + defer w.Stop() + if _, err := asks.Put(ctx, key, value); err != nil { + t.Fatal(err) + } + for e := range w.Updates() { + if e == nil { + continue + } + if a, err := index.DecodeTakeIn(e.Value()); err == nil && a.Done() { + return *a.Result + } + } + t.Fatalf("%s is not answered", key) + return index.TakenIn{} + } + key, value, _ := index.NewRecomputeAsk("a check", "ctl", time.Now()) + if r := answer(key, value); !r.Stored || !strings.Contains(r.Answer, "equal to the kept revision 1") { + t.Errorf("indexer_recompute answered %+v: want equal to the kept revision 1", r) + } + key, value, _ = index.NewDropCandidateGraphAsk("g1", "h1", "its gate event never came", "ctl", time.Now()) + if r := answer(key, value); !r.Stored || !slices.Equal(r.Dropped, []string{"entry.dx"}) { + t.Errorf("indexer_drop_candidate_graph answered %+v: want candidate.g1.h1 dropped with entry.dx", r) + } + if after, err := index.ListRevisions(ctx, objects, false); err != nil || len(after.Revisions) != 1 || after.Entries != list.Entries { + t.Errorf("revisions after the drop = %+v (%v): want trunk.1 and its entries alone", after, err) + } } // TestIndexerKeepsManifestsAtAMerge: a merged message of mesh-catalog carrying the real manifest of the module avahi and the