The delivery repository's contracts and SDK live under modules/ as mesh-delivery-contracts and mesh-delivery-sdk, their Go module paths renamed to match the folders (the TypeScript names already did), and every importer in mesh-delivery, mesh-catalog, mesh-host, mesh-tools and mesh-sdk follows in the same group: a design as it should be, one path per module, checked by the group's check building every importer. Builds: design 47 G §427, §88 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
393 lines
17 KiB
Go
393 lines
17 KiB
Go
package index
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"regexp"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
|
)
|
|
|
|
// The indexer's tools of its catalog graph revisions (design 47, the indexer's tools; the instruction "tools are how a
|
|
// module is spoken to"): indexer_catalog_revision and indexer_catalog_revisions read the objects catalog-revisions,
|
|
// indexer_candidate_graphs every candidate's catalog half; indexer_recompute and indexer_drop_candidate_graph steer the
|
|
// indexer by an ask in takeins, as indexer_take_in does, since the indexer's process serves no request subject.
|
|
const (
|
|
CatalogRevisionTool = "indexer_catalog_revision"
|
|
CatalogRevisionsTool = "indexer_catalog_revisions"
|
|
CandidateGraphsTool = "indexer_candidate_graphs"
|
|
RecomputeTool = "indexer_recompute"
|
|
DropCandidateGraphTool = "indexer_drop_candidate_graph"
|
|
)
|
|
|
|
// RecomputeKey is the takeins key of a recompute by hand: one at a time.
|
|
const RecomputeKey = "recompute"
|
|
|
|
var candidateToken = regexp.MustCompile(`^[A-Za-z0-9_=-]+$`)
|
|
|
|
// NewRecomputeAsk is the value indexer_recompute writes: its key in takeins and the value.
|
|
func NewRecomputeAsk(why, by string, at time.Time) (string, []byte, error) {
|
|
data, err := json.Marshal(TakeInAsk{V: 1, Recompute: true, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
|
|
return RecomputeKey, data, err
|
|
}
|
|
|
|
// NewDropCandidateGraphAsk is the value indexer_drop_candidate_graph writes: its key in takeins,
|
|
// candidate-graph.<group>.<heads>, and the value.
|
|
func NewDropCandidateGraphAsk(group, heads, why, by string, at time.Time) (string, []byte, error) {
|
|
group, heads = strings.TrimSpace(group), strings.TrimSpace(heads)
|
|
switch {
|
|
case !candidateToken.MatchString(group) || !candidateToken.MatchString(heads):
|
|
return "", nil, errors.New("name the candidate by its group and its heads, as indexer_candidate_graphs names them")
|
|
case strings.TrimSpace(why) == "":
|
|
return "", nil, errors.New("say why (why): a drop is a hand act, recorded through mesh-hand-acts")
|
|
}
|
|
data, err := json.Marshal(TakeInAsk{V: 1, CandidateGroup: group, CandidateHeads: heads, Drop: true, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
|
|
return "candidate-graph." + group + "." + heads, data, err
|
|
}
|
|
|
|
// RevisionSummary is one kept catalog graph revision as indexer_catalog_revisions and indexer_candidate_graphs list it.
|
|
type RevisionSummary struct {
|
|
Name string `json:"name"`
|
|
Kind string `json:"kind"`
|
|
Revision int64 `json:"revision"`
|
|
Source int64 `json:"source"`
|
|
Group string `json:"group,omitempty"`
|
|
Heads string `json:"heads,omitempty"`
|
|
At *time.Time `json:"at,omitempty"`
|
|
Entries int `json:"entries"`
|
|
// Current says it is the trunk's newest revision, which is kept whatever pins it.
|
|
Current bool `json:"current,omitempty"`
|
|
}
|
|
|
|
// RevisionsAnswer is what indexer_catalog_revisions answers: every kept revision, the entries stored once by digest, the
|
|
// store's fill against its cap, and each object it could not read.
|
|
type RevisionsAnswer struct {
|
|
Revisions []RevisionSummary `json:"revisions"`
|
|
Entries int `json:"entries"`
|
|
Unlisted []string `json:"unlisted,omitempty"`
|
|
Fill meshcatalog.Fill `json:"fill"`
|
|
FillUnread string `json:"fill-unread,omitempty"`
|
|
Unread []string `json:"unread,omitempty"`
|
|
}
|
|
|
|
// ListRevisions reads every revision the objects keep, candidates included (kind "candidate" alone when candidates).
|
|
func ListRevisions(ctx context.Context, o Objects, candidates bool) (RevisionsAnswer, error) {
|
|
out := RevisionsAnswer{Revisions: []RevisionSummary{}}
|
|
names, err := o.Names(ctx)
|
|
if err != nil {
|
|
return out, fmt.Errorf("the objects %s cannot be listed: %w", CatalogRevisionsObjects, err)
|
|
}
|
|
listed := map[string]bool{}
|
|
var newest int64
|
|
entries := []string{}
|
|
for _, n := range names {
|
|
if strings.HasPrefix(n, "entry.") {
|
|
entries = append(entries, n)
|
|
continue
|
|
}
|
|
data, err := o.Get(ctx, n)
|
|
var rev meshcatalog.CatalogRevision
|
|
if err == nil {
|
|
err = json.Unmarshal(data, &rev)
|
|
}
|
|
if err != nil {
|
|
out.Unread = append(out.Unread, n+": "+err.Error())
|
|
continue
|
|
}
|
|
for _, ref := range rev.Entries {
|
|
listed[EntryObject(ref.Digest)] = true
|
|
}
|
|
if rev.Kind == "trunk" && rev.Revision > newest {
|
|
newest = rev.Revision
|
|
}
|
|
if candidates && rev.Kind != "candidate" {
|
|
continue
|
|
}
|
|
out.Revisions = append(out.Revisions, RevisionSummary{Name: n, Kind: rev.Kind, Revision: rev.Revision, Source: rev.Source,
|
|
Group: rev.Group, Heads: rev.Heads, At: rev.At, Entries: len(rev.Entries)})
|
|
}
|
|
for i := range out.Revisions {
|
|
out.Revisions[i].Current = out.Revisions[i].Kind == "trunk" && out.Revisions[i].Revision == newest
|
|
}
|
|
slices.SortFunc(out.Revisions, func(a, b RevisionSummary) int { return strings.Compare(a.Name, b.Name) })
|
|
out.Entries = len(entries)
|
|
for _, e := range entries {
|
|
if !listed[e] {
|
|
out.Unlisted = append(out.Unlisted, e)
|
|
}
|
|
}
|
|
slices.Sort(out.Unlisted)
|
|
used, capacity, err := o.Fill(ctx)
|
|
out.Fill = meshcatalog.Fill{Bytes: used, Cap: capacity, Store: CatalogRevisionsObjects}
|
|
if err != nil {
|
|
out.FillUnread = err.Error()
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// removeUnlisted deletes every entry no kept revision lists any more and no want or running build names (design 47 G §675): an
|
|
// entry is stored once and shared, so it goes only with the last revision that lists it, and its entry-index key with it. It
|
|
// removes none while the sets of the names are not ready (a restart reads named whole before it removes anything), and keeps
|
|
// an entry whose set it cannot read. It answers the entries removed. The caller holds keepMu.
|
|
func (ix *Indexer) removeUnlisted(ctx context.Context) ([]string, error) {
|
|
if ix.NamedSets != nil && !ix.NamedSets.Ready() {
|
|
return nil, nil
|
|
}
|
|
o := ix.Revisions
|
|
list, err := ListRevisions(ctx, o, false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(list.Unread) > 0 {
|
|
// A revision that cannot be read may list any entry: none is removed rather than one it still needs.
|
|
return nil, fmt.Errorf("no entry is removed while a revision cannot be read: %s", strings.Join(list.Unread, "; "))
|
|
}
|
|
var removed []string
|
|
for _, e := range list.Unlisted {
|
|
fp, digest := ix.entryFingerprint(ctx, e)
|
|
if ix.NamedSets != nil && fp != "" {
|
|
if named, err := ix.NamedSets.IsNamed(ctx, fp); named {
|
|
if err != nil {
|
|
ix.logf("catalog graph: %s is kept, the state %s cannot be read: %v", e, NamedState, err)
|
|
}
|
|
continue
|
|
}
|
|
}
|
|
if err := o.Delete(ctx, e); err != nil {
|
|
return removed, fmt.Errorf("%s: %w", e, err)
|
|
}
|
|
removed = append(removed, e)
|
|
ix.forgetEntryIndex(ctx, fp, digest)
|
|
}
|
|
return removed, nil
|
|
}
|
|
|
|
// entryFingerprint is the build-input fingerprint of an entry object and its digest; none when the object does not read.
|
|
func (ix *Indexer) entryFingerprint(ctx context.Context, object string) (fp, digest string) {
|
|
digest = strings.TrimPrefix(object, "entry.")
|
|
data, err := ix.Revisions.Get(ctx, object)
|
|
if err != nil {
|
|
return "", digest
|
|
}
|
|
var e meshcatalog.CatalogEntry
|
|
if json.Unmarshal(data, &e) != nil {
|
|
return "", digest
|
|
}
|
|
return e.Fp, digest
|
|
}
|
|
|
|
// forgetEntryIndex deletes the entry-index key of a fingerprint when it still holds the digest of the object just removed (§675).
|
|
func (ix *Indexer) forgetEntryIndex(ctx context.Context, fp, digest string) {
|
|
if ix.EntryIndex == nil || fp == "" {
|
|
return
|
|
}
|
|
err := write(ctx, ix.EntryIndex, EntryIndexKey(fp), func(old []byte, found bool) ([]byte, bool, error) {
|
|
var v EntryIndexValue
|
|
if !found || json.Unmarshal(old, &v) != nil || v.Digest != digest {
|
|
return nil, false, nil
|
|
}
|
|
return nil, true, nil
|
|
})
|
|
if err != nil {
|
|
ix.logf("catalog graph: the key of %s in %s is not removed: %v", fp, EntryIndexState, err)
|
|
}
|
|
}
|
|
|
|
// RemoveUnlisted removes the entries nothing names now (§675): called when a set of the names lost a member.
|
|
func (ix *Indexer) RemoveUnlisted(ctx context.Context) {
|
|
if ix.Revisions == nil {
|
|
return
|
|
}
|
|
ix.keepMu.Lock()
|
|
removed, err := ix.removeUnlisted(ctx)
|
|
ix.keepMu.Unlock()
|
|
if err != nil {
|
|
ix.logf("catalog graph: the entries nothing names are not removed: %v", err)
|
|
}
|
|
if len(removed) > 0 {
|
|
ix.logf("catalog graph: %d entries removed, no revision lists them and no want or running build names them", len(removed))
|
|
if _, err := ix.RoomAgain(ctx); err != nil {
|
|
ix.logf("catalog graph: room is not evaluated: %v", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// DropCandidateGraph drops a candidate's catalog half by hand (indexer_drop_candidate_graph, design 47: for a candidate
|
|
// whose gate event never came), the entries no other revision lists with it, and says room when that takes the store
|
|
// below its filling mark, which takes in a merge held for objects full (RoomAgain).
|
|
func (ix *Indexer) DropCandidateGraph(ctx context.Context, group, heads, by, why string) TakenIn {
|
|
name := RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: group, Heads: heads})
|
|
r := TakenIn{Module: name}
|
|
if ix.Revisions == nil {
|
|
r.Answer = "the indexer keeps no catalog graph revisions: nothing dropped"
|
|
return r
|
|
}
|
|
if _, err := ix.Revisions.Get(ctx, name); err != nil {
|
|
r.Answer = name + " is not kept (" + err.Error() + "): nothing dropped"
|
|
return r
|
|
}
|
|
ix.keepMu.Lock()
|
|
err := ix.Revisions.Delete(ctx, name)
|
|
var removed []string
|
|
if err == nil {
|
|
removed, err = ix.removeUnlisted(ctx)
|
|
}
|
|
ix.keepMu.Unlock()
|
|
if err != nil {
|
|
r.Answer = name + " could not be dropped whole: " + err.Error()
|
|
ix.logf("%s", r.Answer)
|
|
return r
|
|
}
|
|
r.Stored, r.Dropped = true, removed
|
|
r.Answer = fmt.Sprintf("%s is dropped by %s (%s), with %d entries no other revision lists", name, by, why, len(removed))
|
|
if roomed, err := ix.RoomAgain(ctx); err != nil {
|
|
r.Answer += "; room is not evaluated: " + err.Error()
|
|
} else if roomed {
|
|
r.Answer += "; room said, what waited for it evaluated again"
|
|
}
|
|
ix.logf("%s", r.Answer)
|
|
return r
|
|
}
|
|
|
|
// RecomputeByHand computes the catalog graph again now from the view of the trunk, as a toolchain report or a merge
|
|
// does, and compares it with the newest revision kept: the modules whose entry moved, and the revision kept for them
|
|
// (none when the recompute equals the one kept).
|
|
func (ix *Indexer) RecomputeByHand(ctx context.Context, by, why string) TakenIn {
|
|
r := TakenIn{Module: "catalog graph"}
|
|
if ix.Revisions == nil {
|
|
r.Answer = "the indexer keeps no catalog graph revisions: nothing recomputed"
|
|
return r
|
|
}
|
|
before, _, err := newestTrunk(ctx, ix.Revisions)
|
|
if err != nil {
|
|
r.Answer = "the newest catalog graph revision cannot be read: " + err.Error()
|
|
return r
|
|
}
|
|
if err := ix.Recompute(ctx); err != nil {
|
|
r.Answer = "the recompute failed: " + err.Error()
|
|
return r
|
|
}
|
|
after, _, err := newestTrunk(ctx, ix.Revisions)
|
|
if err != nil {
|
|
r.Answer = "the newest catalog graph revision cannot be read after the recompute: " + err.Error()
|
|
return r
|
|
}
|
|
r.Stored = true
|
|
if after.Revision == before.Revision {
|
|
r.Answer = fmt.Sprintf("recomputed by %s (%s): equal to the kept revision %d", by, why, before.Revision)
|
|
} else {
|
|
r.Handed = changedEntries(before.Entries, after.Entries)
|
|
r.Answer = fmt.Sprintf("recomputed by %s (%s): revision %d differs from the kept %d in %s; revision %d is kept", by, why,
|
|
after.Revision, before.Revision, strings.Join(r.Handed, ", "), after.Revision)
|
|
}
|
|
ix.logf("%s", r.Answer)
|
|
return r
|
|
}
|
|
|
|
// CatalogRevisionTools are the tools of the catalog graph revisions; ask writes an ask in takeins and waits for the
|
|
// indexer's answer, record records a hand act through mesh-hand-acts before a drop is asked.
|
|
func CatalogRevisionTools(o Objects, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time, record func(what, why string) error) []stdio.Tool {
|
|
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
|
asked := func(key string, value []byte, err error) (any, error) {
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
a, err := ask(key, value)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return a.Result, nil
|
|
}
|
|
return []stdio.Tool{
|
|
{Name: CatalogRevisionTool,
|
|
Description: "One catalog graph revision the indexer keeps in its objects catalog-revisions, whole: its list (kind, number, " +
|
|
"source revision, trunk heads, a candidate's group and heads) and every entry it lists, each module as its manifest " +
|
|
"declares it at that source revision (bases, packages, toolchains, provides, requires, invokes, serves, fingerprint, " +
|
|
"acting rules, inputs). The trunk's newest with no input; revision a kept trunk revision; group and heads a candidate's. " +
|
|
"An entry the store lost is an error naming it. indexer_catalog_revisions lists what is kept.",
|
|
Input: map[string]any{"type": "object", "properties": map[string]any{
|
|
"revision": map[string]any{"type": "integer", "description": "a kept trunk revision's number"},
|
|
"group": map[string]any{"type": "string", "description": "a candidate's group, with heads"},
|
|
"heads": map[string]any{"type": "string", "description": "a candidate's heads, with group"}}},
|
|
Run: func(args map[string]any) (any, error) {
|
|
ctx := context.Background()
|
|
n, _ := args["revision"].(float64)
|
|
name := ""
|
|
switch {
|
|
case str(args, "group") != "" || str(args, "heads") != "":
|
|
name = RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: str(args, "group"), Heads: str(args, "heads")})
|
|
case n > 0:
|
|
name = "trunk." + strconv.FormatInt(int64(n), 10)
|
|
default:
|
|
newest, ok, err := newestTrunk(ctx, o)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the catalog graph revisions cannot be read: %w", err)
|
|
}
|
|
if !ok {
|
|
return nil, ErrNoRevision
|
|
}
|
|
name = RevisionObject(newest)
|
|
}
|
|
rev, entries, err := ReadRevision(ctx, o, name)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return map[string]any{"name": name, "revision": rev, "entries": entries}, nil
|
|
}},
|
|
{Name: CatalogRevisionsTool,
|
|
Description: "Every catalog graph revision the indexer keeps in its objects catalog-revisions, trunk and candidate: each " +
|
|
"one's name, kind, number, source revision, when, how many entries it lists and whether it is the trunk's newest " +
|
|
"(current); how many entries are stored once by digest, those no revision lists (unlisted), and the store's fill " +
|
|
"in bytes against its cap (filling is said at 80 %, full refuses the next revision). An object it cannot read is " +
|
|
"named in unread. indexer_catalog_revision reads one whole.",
|
|
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
|
Run: func(map[string]any) (any, error) {
|
|
return ListRevisions(context.Background(), o, false)
|
|
}},
|
|
{Name: CandidateGraphsTool,
|
|
Description: "Every candidate's catalog half the indexer keeps (a delivery group's or a pull request's heads merged onto " +
|
|
"a trunk source revision): its group, its heads, its revision, the source revision it is merged onto, since when, " +
|
|
"and how many entries it lists; and the store's fill. indexer_drop_candidate_graph drops one whose gate event never came.",
|
|
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
|
Run: func(map[string]any) (any, error) {
|
|
return ListRevisions(context.Background(), o, true)
|
|
}},
|
|
{Name: RecomputeTool,
|
|
Description: "Compute the catalog graph again now, as the indexer does on a toolchain report or a merge, from the view of " +
|
|
"the trunk (each module's manifest, its bases, the agreed toolchains and the published packages), and compare it with " +
|
|
"the newest revision kept: answers equal, or the modules whose entry differs and the revision kept for them.",
|
|
Input: map[string]any{"type": "object", "properties": map[string]any{
|
|
"why": map[string]any{"type": "string", "description": "why it is recomputed now, in plain words"}}},
|
|
Run: func(args map[string]any) (any, error) {
|
|
key, value, err := NewRecomputeAsk(str(args, "why"), stdio.CallerOf(args), now())
|
|
return asked(key, value, err)
|
|
}},
|
|
{Name: DropCandidateGraphTool,
|
|
Description: "Drop one candidate's catalog half by hand, for a candidate whose gate event never came: its revision and the " +
|
|
"entries no other revision lists. A hand act, recorded through mesh-hand-acts before it is asked. When the drop takes " +
|
|
"the store below its filling mark, room is said and a merge held for objects full is taken in. indexer_candidate_graphs " +
|
|
"names the candidates.",
|
|
Input: map[string]any{"type": "object", "required": []string{"group", "heads", "why"}, "properties": map[string]any{
|
|
"group": map[string]any{"type": "string", "description": "the candidate's group"},
|
|
"heads": map[string]any{"type": "string", "description": "the candidate's heads"},
|
|
"why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}}},
|
|
Run: func(args map[string]any) (any, error) {
|
|
key, value, err := NewDropCandidateGraphAsk(str(args, "group"), str(args, "heads"), str(args, "why"), stdio.CallerOf(args), now())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := record("dropped the catalog half of the candidate "+str(args, "group")+" at "+str(args, "heads")+" by hand", str(args, "why")); err != nil {
|
|
return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was dropped: %w", err)
|
|
}
|
|
return asked(key, value, nil)
|
|
}},
|
|
}
|
|
}
|