Files
mesh-delivery/modules/mesh-delivery-indexer/internal/index/revision_tools.go
T
jschoubben 71a35a6376 MV1: move contracts and sdk under modules/, importers follow
The delivery repository's contracts and SDK live under modules/ as
mesh-delivery-contracts and mesh-delivery-sdk, their Go module paths
renamed to match the folders (the TypeScript names already did), and
every importer in mesh-delivery, mesh-catalog, mesh-host, mesh-tools
and mesh-sdk follows in the same group: a design as it should be, one
path per module, checked by the group's check building every importer.

Builds: design 47 G §427, §88

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
2026-11-12 05:09:56 +01:00

393 lines
17 KiB
Go

package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"regexp"
"slices"
"strconv"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// The indexer's tools of its catalog graph revisions (design 47, the indexer's tools; the instruction "tools are how a
// module is spoken to"): indexer_catalog_revision and indexer_catalog_revisions read the objects catalog-revisions,
// indexer_candidate_graphs every candidate's catalog half; indexer_recompute and indexer_drop_candidate_graph steer the
// indexer by an ask in takeins, as indexer_take_in does, since the indexer's process serves no request subject.
const (
CatalogRevisionTool = "indexer_catalog_revision"
CatalogRevisionsTool = "indexer_catalog_revisions"
CandidateGraphsTool = "indexer_candidate_graphs"
RecomputeTool = "indexer_recompute"
DropCandidateGraphTool = "indexer_drop_candidate_graph"
)
// RecomputeKey is the takeins key of a recompute by hand: one at a time.
const RecomputeKey = "recompute"
var candidateToken = regexp.MustCompile(`^[A-Za-z0-9_=-]+$`)
// NewRecomputeAsk is the value indexer_recompute writes: its key in takeins and the value.
func NewRecomputeAsk(why, by string, at time.Time) (string, []byte, error) {
data, err := json.Marshal(TakeInAsk{V: 1, Recompute: true, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
return RecomputeKey, data, err
}
// NewDropCandidateGraphAsk is the value indexer_drop_candidate_graph writes: its key in takeins,
// candidate-graph.<group>.<heads>, and the value.
func NewDropCandidateGraphAsk(group, heads, why, by string, at time.Time) (string, []byte, error) {
group, heads = strings.TrimSpace(group), strings.TrimSpace(heads)
switch {
case !candidateToken.MatchString(group) || !candidateToken.MatchString(heads):
return "", nil, errors.New("name the candidate by its group and its heads, as indexer_candidate_graphs names them")
case strings.TrimSpace(why) == "":
return "", nil, errors.New("say why (why): a drop is a hand act, recorded through mesh-hand-acts")
}
data, err := json.Marshal(TakeInAsk{V: 1, CandidateGroup: group, CandidateHeads: heads, Drop: true, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
return "candidate-graph." + group + "." + heads, data, err
}
// RevisionSummary is one kept catalog graph revision as indexer_catalog_revisions and indexer_candidate_graphs list it.
type RevisionSummary struct {
Name string `json:"name"`
Kind string `json:"kind"`
Revision int64 `json:"revision"`
Source int64 `json:"source"`
Group string `json:"group,omitempty"`
Heads string `json:"heads,omitempty"`
At *time.Time `json:"at,omitempty"`
Entries int `json:"entries"`
// Current says it is the trunk's newest revision, which is kept whatever pins it.
Current bool `json:"current,omitempty"`
}
// RevisionsAnswer is what indexer_catalog_revisions answers: every kept revision, the entries stored once by digest, the
// store's fill against its cap, and each object it could not read.
type RevisionsAnswer struct {
Revisions []RevisionSummary `json:"revisions"`
Entries int `json:"entries"`
Unlisted []string `json:"unlisted,omitempty"`
Fill meshcatalog.Fill `json:"fill"`
FillUnread string `json:"fill-unread,omitempty"`
Unread []string `json:"unread,omitempty"`
}
// ListRevisions reads every revision the objects keep, candidates included (kind "candidate" alone when candidates).
func ListRevisions(ctx context.Context, o Objects, candidates bool) (RevisionsAnswer, error) {
out := RevisionsAnswer{Revisions: []RevisionSummary{}}
names, err := o.Names(ctx)
if err != nil {
return out, fmt.Errorf("the objects %s cannot be listed: %w", CatalogRevisionsObjects, err)
}
listed := map[string]bool{}
var newest int64
entries := []string{}
for _, n := range names {
if strings.HasPrefix(n, "entry.") {
entries = append(entries, n)
continue
}
data, err := o.Get(ctx, n)
var rev meshcatalog.CatalogRevision
if err == nil {
err = json.Unmarshal(data, &rev)
}
if err != nil {
out.Unread = append(out.Unread, n+": "+err.Error())
continue
}
for _, ref := range rev.Entries {
listed[EntryObject(ref.Digest)] = true
}
if rev.Kind == "trunk" && rev.Revision > newest {
newest = rev.Revision
}
if candidates && rev.Kind != "candidate" {
continue
}
out.Revisions = append(out.Revisions, RevisionSummary{Name: n, Kind: rev.Kind, Revision: rev.Revision, Source: rev.Source,
Group: rev.Group, Heads: rev.Heads, At: rev.At, Entries: len(rev.Entries)})
}
for i := range out.Revisions {
out.Revisions[i].Current = out.Revisions[i].Kind == "trunk" && out.Revisions[i].Revision == newest
}
slices.SortFunc(out.Revisions, func(a, b RevisionSummary) int { return strings.Compare(a.Name, b.Name) })
out.Entries = len(entries)
for _, e := range entries {
if !listed[e] {
out.Unlisted = append(out.Unlisted, e)
}
}
slices.Sort(out.Unlisted)
used, capacity, err := o.Fill(ctx)
out.Fill = meshcatalog.Fill{Bytes: used, Cap: capacity, Store: CatalogRevisionsObjects}
if err != nil {
out.FillUnread = err.Error()
}
return out, nil
}
// removeUnlisted deletes every entry no kept revision lists any more and no want or running build names (design 47 G §675): an
// entry is stored once and shared, so it goes only with the last revision that lists it, and its entry-index key with it. It
// removes none while the sets of the names are not ready (a restart reads named whole before it removes anything), and keeps
// an entry whose set it cannot read. It answers the entries removed. The caller holds keepMu.
func (ix *Indexer) removeUnlisted(ctx context.Context) ([]string, error) {
if ix.NamedSets != nil && !ix.NamedSets.Ready() {
return nil, nil
}
o := ix.Revisions
list, err := ListRevisions(ctx, o, false)
if err != nil {
return nil, err
}
if len(list.Unread) > 0 {
// A revision that cannot be read may list any entry: none is removed rather than one it still needs.
return nil, fmt.Errorf("no entry is removed while a revision cannot be read: %s", strings.Join(list.Unread, "; "))
}
var removed []string
for _, e := range list.Unlisted {
fp, digest := ix.entryFingerprint(ctx, e)
if ix.NamedSets != nil && fp != "" {
if named, err := ix.NamedSets.IsNamed(ctx, fp); named {
if err != nil {
ix.logf("catalog graph: %s is kept, the state %s cannot be read: %v", e, NamedState, err)
}
continue
}
}
if err := o.Delete(ctx, e); err != nil {
return removed, fmt.Errorf("%s: %w", e, err)
}
removed = append(removed, e)
ix.forgetEntryIndex(ctx, fp, digest)
}
return removed, nil
}
// entryFingerprint is the build-input fingerprint of an entry object and its digest; none when the object does not read.
func (ix *Indexer) entryFingerprint(ctx context.Context, object string) (fp, digest string) {
digest = strings.TrimPrefix(object, "entry.")
data, err := ix.Revisions.Get(ctx, object)
if err != nil {
return "", digest
}
var e meshcatalog.CatalogEntry
if json.Unmarshal(data, &e) != nil {
return "", digest
}
return e.Fp, digest
}
// forgetEntryIndex deletes the entry-index key of a fingerprint when it still holds the digest of the object just removed (§675).
func (ix *Indexer) forgetEntryIndex(ctx context.Context, fp, digest string) {
if ix.EntryIndex == nil || fp == "" {
return
}
err := write(ctx, ix.EntryIndex, EntryIndexKey(fp), func(old []byte, found bool) ([]byte, bool, error) {
var v EntryIndexValue
if !found || json.Unmarshal(old, &v) != nil || v.Digest != digest {
return nil, false, nil
}
return nil, true, nil
})
if err != nil {
ix.logf("catalog graph: the key of %s in %s is not removed: %v", fp, EntryIndexState, err)
}
}
// RemoveUnlisted removes the entries nothing names now (§675): called when a set of the names lost a member.
func (ix *Indexer) RemoveUnlisted(ctx context.Context) {
if ix.Revisions == nil {
return
}
ix.keepMu.Lock()
removed, err := ix.removeUnlisted(ctx)
ix.keepMu.Unlock()
if err != nil {
ix.logf("catalog graph: the entries nothing names are not removed: %v", err)
}
if len(removed) > 0 {
ix.logf("catalog graph: %d entries removed, no revision lists them and no want or running build names them", len(removed))
if _, err := ix.RoomAgain(ctx); err != nil {
ix.logf("catalog graph: room is not evaluated: %v", err)
}
}
}
// DropCandidateGraph drops a candidate's catalog half by hand (indexer_drop_candidate_graph, design 47: for a candidate
// whose gate event never came), the entries no other revision lists with it, and says room when that takes the store
// below its filling mark, which takes in a merge held for objects full (RoomAgain).
func (ix *Indexer) DropCandidateGraph(ctx context.Context, group, heads, by, why string) TakenIn {
name := RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: group, Heads: heads})
r := TakenIn{Module: name}
if ix.Revisions == nil {
r.Answer = "the indexer keeps no catalog graph revisions: nothing dropped"
return r
}
if _, err := ix.Revisions.Get(ctx, name); err != nil {
r.Answer = name + " is not kept (" + err.Error() + "): nothing dropped"
return r
}
ix.keepMu.Lock()
err := ix.Revisions.Delete(ctx, name)
var removed []string
if err == nil {
removed, err = ix.removeUnlisted(ctx)
}
ix.keepMu.Unlock()
if err != nil {
r.Answer = name + " could not be dropped whole: " + err.Error()
ix.logf("%s", r.Answer)
return r
}
r.Stored, r.Dropped = true, removed
r.Answer = fmt.Sprintf("%s is dropped by %s (%s), with %d entries no other revision lists", name, by, why, len(removed))
if roomed, err := ix.RoomAgain(ctx); err != nil {
r.Answer += "; room is not evaluated: " + err.Error()
} else if roomed {
r.Answer += "; room said, what waited for it evaluated again"
}
ix.logf("%s", r.Answer)
return r
}
// RecomputeByHand computes the catalog graph again now from the view of the trunk, as a toolchain report or a merge
// does, and compares it with the newest revision kept: the modules whose entry moved, and the revision kept for them
// (none when the recompute equals the one kept).
func (ix *Indexer) RecomputeByHand(ctx context.Context, by, why string) TakenIn {
r := TakenIn{Module: "catalog graph"}
if ix.Revisions == nil {
r.Answer = "the indexer keeps no catalog graph revisions: nothing recomputed"
return r
}
before, _, err := newestTrunk(ctx, ix.Revisions)
if err != nil {
r.Answer = "the newest catalog graph revision cannot be read: " + err.Error()
return r
}
if err := ix.Recompute(ctx); err != nil {
r.Answer = "the recompute failed: " + err.Error()
return r
}
after, _, err := newestTrunk(ctx, ix.Revisions)
if err != nil {
r.Answer = "the newest catalog graph revision cannot be read after the recompute: " + err.Error()
return r
}
r.Stored = true
if after.Revision == before.Revision {
r.Answer = fmt.Sprintf("recomputed by %s (%s): equal to the kept revision %d", by, why, before.Revision)
} else {
r.Handed = changedEntries(before.Entries, after.Entries)
r.Answer = fmt.Sprintf("recomputed by %s (%s): revision %d differs from the kept %d in %s; revision %d is kept", by, why,
after.Revision, before.Revision, strings.Join(r.Handed, ", "), after.Revision)
}
ix.logf("%s", r.Answer)
return r
}
// CatalogRevisionTools are the tools of the catalog graph revisions; ask writes an ask in takeins and waits for the
// indexer's answer, record records a hand act through mesh-hand-acts before a drop is asked.
func CatalogRevisionTools(o Objects, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time, record func(what, why string) error) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
asked := func(key string, value []byte, err error) (any, error) {
if err != nil {
return nil, err
}
a, err := ask(key, value)
if err != nil {
return nil, err
}
return a.Result, nil
}
return []stdio.Tool{
{Name: CatalogRevisionTool,
Description: "One catalog graph revision the indexer keeps in its objects catalog-revisions, whole: its list (kind, number, " +
"source revision, trunk heads, a candidate's group and heads) and every entry it lists, each module as its manifest " +
"declares it at that source revision (bases, packages, toolchains, provides, requires, invokes, serves, fingerprint, " +
"acting rules, inputs). The trunk's newest with no input; revision a kept trunk revision; group and heads a candidate's. " +
"An entry the store lost is an error naming it. indexer_catalog_revisions lists what is kept.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"revision": map[string]any{"type": "integer", "description": "a kept trunk revision's number"},
"group": map[string]any{"type": "string", "description": "a candidate's group, with heads"},
"heads": map[string]any{"type": "string", "description": "a candidate's heads, with group"}}},
Run: func(args map[string]any) (any, error) {
ctx := context.Background()
n, _ := args["revision"].(float64)
name := ""
switch {
case str(args, "group") != "" || str(args, "heads") != "":
name = RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: str(args, "group"), Heads: str(args, "heads")})
case n > 0:
name = "trunk." + strconv.FormatInt(int64(n), 10)
default:
newest, ok, err := newestTrunk(ctx, o)
if err != nil {
return nil, fmt.Errorf("the catalog graph revisions cannot be read: %w", err)
}
if !ok {
return nil, ErrNoRevision
}
name = RevisionObject(newest)
}
rev, entries, err := ReadRevision(ctx, o, name)
if err != nil {
return nil, err
}
return map[string]any{"name": name, "revision": rev, "entries": entries}, nil
}},
{Name: CatalogRevisionsTool,
Description: "Every catalog graph revision the indexer keeps in its objects catalog-revisions, trunk and candidate: each " +
"one's name, kind, number, source revision, when, how many entries it lists and whether it is the trunk's newest " +
"(current); how many entries are stored once by digest, those no revision lists (unlisted), and the store's fill " +
"in bytes against its cap (filling is said at 80 %, full refuses the next revision). An object it cannot read is " +
"named in unread. indexer_catalog_revision reads one whole.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
return ListRevisions(context.Background(), o, false)
}},
{Name: CandidateGraphsTool,
Description: "Every candidate's catalog half the indexer keeps (a delivery group's or a pull request's heads merged onto " +
"a trunk source revision): its group, its heads, its revision, the source revision it is merged onto, since when, " +
"and how many entries it lists; and the store's fill. indexer_drop_candidate_graph drops one whose gate event never came.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
return ListRevisions(context.Background(), o, true)
}},
{Name: RecomputeTool,
Description: "Compute the catalog graph again now, as the indexer does on a toolchain report or a merge, from the view of " +
"the trunk (each module's manifest, its bases, the agreed toolchains and the published packages), and compare it with " +
"the newest revision kept: answers equal, or the modules whose entry differs and the revision kept for them.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"why": map[string]any{"type": "string", "description": "why it is recomputed now, in plain words"}}},
Run: func(args map[string]any) (any, error) {
key, value, err := NewRecomputeAsk(str(args, "why"), stdio.CallerOf(args), now())
return asked(key, value, err)
}},
{Name: DropCandidateGraphTool,
Description: "Drop one candidate's catalog half by hand, for a candidate whose gate event never came: its revision and the " +
"entries no other revision lists. A hand act, recorded through mesh-hand-acts before it is asked. When the drop takes " +
"the store below its filling mark, room is said and a merge held for objects full is taken in. indexer_candidate_graphs " +
"names the candidates.",
Input: map[string]any{"type": "object", "required": []string{"group", "heads", "why"}, "properties": map[string]any{
"group": map[string]any{"type": "string", "description": "the candidate's group"},
"heads": map[string]any{"type": "string", "description": "the candidate's heads"},
"why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}}},
Run: func(args map[string]any) (any, error) {
key, value, err := NewDropCandidateGraphAsk(str(args, "group"), str(args, "heads"), str(args, "why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
if err := record("dropped the catalog half of the candidate "+str(args, "group")+" at "+str(args, "heads")+" by hand", str(args, "why")); err != nil {
return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was dropped: %w", err)
}
return asked(key, value, nil)
}},
}
}