673 lines
27 KiB
Go
673 lines
27 KiB
Go
package index
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
"git.novox.be/novox/mesh-sdk/go/keys"
|
|
"git.novox.be/novox/mesh-sdk/go/process"
|
|
"git.novox.be/novox/mesh-sdk/go/views"
|
|
)
|
|
|
|
// The indexer's object bucket catalog-revisions (design 47, Two graphs; novox/hq ADR 0619 §2: declared in the old
|
|
// `objects` section, its cap there): a revision is its list of entries' digests under trunk.<revision> or
|
|
// candidate.<group>.<heads>, and each entry one object entry.<digest>, written once and shared by every revision that
|
|
// lists it.
|
|
|
|
// CatalogRevisionsObjects is the bucket's local name in the indexer's manifest.
|
|
const CatalogRevisionsObjects = "catalog-revisions"
|
|
|
|
// ErrNoObject is an object the bucket does not hold.
|
|
var ErrNoObject = errors.New("no such object")
|
|
|
|
// Objects is an object bucket as the indexer keeps its revisions in it.
|
|
type Objects interface {
|
|
// Put stores data whole under name, replacing one of that name.
|
|
Put(ctx context.Context, name string, data []byte) error
|
|
// Get is the object's bytes; ErrNoObject when it holds none of that name.
|
|
Get(ctx context.Context, name string) ([]byte, error)
|
|
// Delete removes the object; one it does not hold is no error.
|
|
Delete(ctx context.Context, name string) error
|
|
// Names is every object's name; none is an empty list.
|
|
Names(ctx context.Context) ([]string, error)
|
|
// Fill is the bucket's bytes used and its cap in bytes.
|
|
Fill(ctx context.Context) (used, capacity int64, err error)
|
|
}
|
|
|
|
// EntryObject is the object name of the entry of that digest.
|
|
func EntryObject(digest string) string { return "entry." + digest }
|
|
|
|
// TrunkBranch is the first watched branch, whose revisions are kept as trunk.<revision> (design 47 G §291, A-85).
|
|
const TrunkBranch = "main"
|
|
|
|
// isTrunkBranch says whether a branch's revisions are the trunk's: the first watched branch, or none named.
|
|
func isTrunkBranch(branch string) bool { return branch == "" || branch == TrunkBranch }
|
|
|
|
// BranchObject is the object name of a revision of a watched branch other than the first: branch.<branch>.<revision>,
|
|
// the branch escaped as a name (A-85), so a dot in it never reads as a part.
|
|
func BranchObject(branch string, revision int64) string {
|
|
return "branch." + keys.EscapeName(branch) + "." + strconv.FormatInt(revision, 10)
|
|
}
|
|
|
|
// RevisionObject is the object name of a revision: trunk.<revision> for the first watched branch,
|
|
// branch.<branch>.<revision> for another watched branch (A-85), or candidate.<group>.<heads> for a candidate,
|
|
// which keeps its newest revision only.
|
|
func RevisionObject(rev meshcatalog.CatalogRevision) string {
|
|
if rev.Kind == "candidate" {
|
|
return "candidate." + rev.Group + "." + rev.Heads
|
|
}
|
|
if !isTrunkBranch(rev.Branch) {
|
|
return BranchObject(rev.Branch, rev.Revision)
|
|
}
|
|
return "trunk." + strconv.FormatInt(rev.Revision, 10)
|
|
}
|
|
|
|
// BranchRevisionOf is the revision number a branch object's name holds for that branch, and whether it is one.
|
|
func BranchRevisionOf(branch, name string) (int64, bool) {
|
|
n, ok := strings.CutPrefix(name, "branch."+keys.EscapeName(branch)+".")
|
|
if !ok {
|
|
return 0, false
|
|
}
|
|
r, err := strconv.ParseInt(n, 10, 64)
|
|
return r, err == nil
|
|
}
|
|
|
|
// TrunkRevisionOf is the revision number a trunk object's name holds, and whether it is one.
|
|
func TrunkRevisionOf(name string) (int64, bool) {
|
|
n, ok := strings.CutPrefix(name, "trunk.")
|
|
if !ok {
|
|
return 0, false
|
|
}
|
|
r, err := strconv.ParseInt(n, 10, 64)
|
|
return r, err == nil
|
|
}
|
|
|
|
// KeepRevision stores a revision as CatalogRevisionOf made it: each entry object the bucket does not hold yet, then the
|
|
// revision's list, so a list is never kept whose entries are not; an entry already held is not written again.
|
|
func KeepRevision(ctx context.Context, o Objects, rev meshcatalog.CatalogRevision, entries map[string][]byte) error {
|
|
held, err := o.Names(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err)
|
|
}
|
|
have := make(map[string]bool, len(held))
|
|
for _, n := range held {
|
|
have[n] = true
|
|
}
|
|
for _, ref := range rev.Entries {
|
|
name := EntryObject(ref.Digest)
|
|
if have[name] {
|
|
continue
|
|
}
|
|
data, ok := entries[ref.Digest]
|
|
if !ok {
|
|
return fmt.Errorf("revision %s lists the entry of %s (%s), which it does not carry", RevisionObject(rev), ref.Module, ref.Digest)
|
|
}
|
|
if err := o.Put(ctx, name, data); err != nil {
|
|
return fmt.Errorf("the entry of %s in %s: %w", ref.Module, CatalogRevisionsObjects, err)
|
|
}
|
|
have[name] = true
|
|
}
|
|
list, err := json.Marshal(rev)
|
|
if err != nil {
|
|
return fmt.Errorf("revision %s: %w", RevisionObject(rev), err)
|
|
}
|
|
if err := o.Put(ctx, RevisionObject(rev), list); err != nil {
|
|
return fmt.Errorf("revision %s in %s: %w", RevisionObject(rev), CatalogRevisionsObjects, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ReadRevision is the revision kept under name and its entries whole, in the revision's order; an entry it lists that
|
|
// the bucket lost is an error naming it, never an entry left out.
|
|
func ReadRevision(ctx context.Context, o Objects, name string) (meshcatalog.CatalogRevision, []meshcatalog.CatalogEntry, error) {
|
|
var rev meshcatalog.CatalogRevision
|
|
data, err := o.Get(ctx, name)
|
|
if err != nil {
|
|
return rev, nil, fmt.Errorf("revision %s: %w", name, err)
|
|
}
|
|
if err := json.Unmarshal(data, &rev); err != nil {
|
|
return rev, nil, fmt.Errorf("revision %s: %w", name, err)
|
|
}
|
|
entries := make([]meshcatalog.CatalogEntry, 0, len(rev.Entries))
|
|
for _, ref := range rev.Entries {
|
|
data, err := o.Get(ctx, EntryObject(ref.Digest))
|
|
if err != nil {
|
|
return rev, nil, fmt.Errorf("revision %s, the entry of %s (%s): %w", name, ref.Module, ref.Digest, err)
|
|
}
|
|
var e meshcatalog.CatalogEntry
|
|
if err := json.Unmarshal(data, &e); err != nil {
|
|
return rev, nil, fmt.Errorf("revision %s, the entry of %s: %w", name, ref.Module, err)
|
|
}
|
|
entries = append(entries, e)
|
|
}
|
|
return rev, entries, nil
|
|
}
|
|
|
|
// newestTrunk is the newest trunk revision the bucket keeps, its list only; none is a zero revision and false.
|
|
func newestTrunk(ctx context.Context, o Objects) (meshcatalog.CatalogRevision, bool, error) {
|
|
return newestOf(ctx, o, "")
|
|
}
|
|
|
|
// newestOf is the newest revision of a watched branch the bucket keeps (the trunk's for the first watched branch or none
|
|
// named), its list only; none is a zero revision and false.
|
|
func newestOf(ctx context.Context, o Objects, branch string) (meshcatalog.CatalogRevision, bool, error) {
|
|
names, err := o.Names(ctx)
|
|
if err != nil {
|
|
return meshcatalog.CatalogRevision{}, false, err
|
|
}
|
|
number := TrunkRevisionOf
|
|
if !isTrunkBranch(branch) {
|
|
number = func(n string) (int64, bool) { return BranchRevisionOf(branch, n) }
|
|
}
|
|
var newest int64
|
|
for _, n := range names {
|
|
if r, ok := number(n); ok && r > newest {
|
|
newest = r
|
|
}
|
|
}
|
|
if newest == 0 {
|
|
return meshcatalog.CatalogRevision{}, false, nil
|
|
}
|
|
name := RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: newest})
|
|
data, err := o.Get(ctx, name)
|
|
if err != nil {
|
|
return meshcatalog.CatalogRevision{}, false, err
|
|
}
|
|
var rev meshcatalog.CatalogRevision
|
|
if err := json.Unmarshal(data, &rev); err != nil {
|
|
return rev, false, fmt.Errorf("%s: %w", name, err)
|
|
}
|
|
return rev, true, nil
|
|
}
|
|
|
|
// changedEntries are the modules whose entry differs between two revisions' lists, one gone or added included, by name.
|
|
func changedEntries(before, after []meshcatalog.EntryRef) []string {
|
|
was := map[string]string{}
|
|
for _, r := range before {
|
|
was[r.Module] = r.Digest
|
|
}
|
|
var changed []string
|
|
for _, r := range after {
|
|
if d, ok := was[r.Module]; !ok || d != r.Digest {
|
|
changed = append(changed, r.Module)
|
|
}
|
|
delete(was, r.Module)
|
|
}
|
|
for m := range was {
|
|
changed = append(changed, m)
|
|
}
|
|
slices.Sort(changed)
|
|
return changed
|
|
}
|
|
|
|
// keepTrunkRevision keeps the catalog graph revision a merge's or a recompute's module graph makes (design 47, Two
|
|
// graphs: a revision for each source revision taken in, and for a fingerprint moved with no merge), numbered after the
|
|
// newest kept, and says catalog-revision; a graph whose entries are the newest revision's makes none. A revision not
|
|
// kept is said in the log and returned, so the take-in that made it is not said taken.
|
|
func (ix *Indexer) keepTrunkRevision(ctx context.Context, edges map[string]graphEdges, whole graph.Graph) error {
|
|
if ix.Revisions == nil {
|
|
return nil
|
|
}
|
|
// Each entry is the module's value as this recompute left it in modules (its fingerprint, toolchains, package versions
|
|
// and inputs written), so the revision holds what the next one is computed from.
|
|
held, err := ix.heldModules(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Every module the view holds is an entry, one the recompute computed no edges for (a manifest it cannot read) too.
|
|
entries := make(map[string]meshcatalog.CatalogEntry, len(held))
|
|
for name, cur := range held {
|
|
entries[name] = ix.withArtifacts(ctx, CatalogEntryOf(name, cur, edges[name].manifest, ix.manifestBytes(ctx, cur.Manifest), whole[name]), edges[name].manifest)
|
|
}
|
|
for name, e := range edges {
|
|
if _, ok := held[name]; !ok {
|
|
entries[name] = ix.withArtifacts(ctx, CatalogEntryOf(name, e.cur, e.manifest, ix.manifestBytes(ctx, e.cur.Manifest), whole[name]), e.manifest)
|
|
}
|
|
}
|
|
return ix.keepEntries(ctx, entries)
|
|
}
|
|
|
|
// KeepView keeps the view of the trunk as a trunk revision after a write to it outside a recompute (the lead's ruling on
|
|
// milestone 47/G: the judge pass's verdicts, a build's reported imports, the source stamp, the modules the trunk no longer
|
|
// holds, the indexer's own entry a held revision writes): each entry is the module's value in the view with its node of
|
|
// the module graph in memory, so what the view holds is never lost at a restart. A view whose entries are the newest
|
|
// revision's keeps none.
|
|
func (ix *Indexer) KeepView(ctx context.Context) error {
|
|
if ix.Revisions == nil || ix.Modules == nil || ix.catchingUp.Load() {
|
|
return nil
|
|
}
|
|
entries, err := ix.viewEntries(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return ix.keepEntries(ctx, entries)
|
|
}
|
|
|
|
// viewEntries is the entry of every module the indexer's view (its modules and its module graph) holds: the module's value
|
|
// with its node of the graph, as KeepView keeps them.
|
|
func (ix *Indexer) viewEntries(ctx context.Context) (map[string]meshcatalog.CatalogEntry, error) {
|
|
held, err := ix.heldModules(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
nodes := graph.Graph{}
|
|
if ix.Graph != nil {
|
|
keys, err := ix.Graph.Keys(ctx)
|
|
if err != nil && !errors.Is(err, process.ErrNotYet) {
|
|
return nil, fmt.Errorf("the module graph: %w", err)
|
|
}
|
|
for _, k := range keys {
|
|
if value, _, found, err := ix.Graph.Get(ctx, k); err == nil && found {
|
|
if n, err := graph.Decode(value); err == nil {
|
|
nodes[k] = n
|
|
}
|
|
}
|
|
}
|
|
}
|
|
entries := make(map[string]meshcatalog.CatalogEntry, len(held))
|
|
for name, cur := range held {
|
|
var m catalogue.Manifest
|
|
var raw string
|
|
if cur.Manifest != "" {
|
|
if bytes, read, err := ix.manifestRaw(ctx, cur.Manifest); err == nil {
|
|
m, raw = read, bytes
|
|
}
|
|
}
|
|
entries[name] = ix.withArtifacts(ctx, CatalogEntryOf(name, cur, m, raw, nodes[name]), m)
|
|
}
|
|
return entries, nil
|
|
}
|
|
|
|
// keepEntries keeps entries as the trunk revision after the newest, says catalog-revision, and keeps none when no entry
|
|
// moved; one keep at a time, so two never take the same number.
|
|
func (ix *Indexer) keepEntries(ctx context.Context, entries map[string]meshcatalog.CatalogEntry) error {
|
|
if ix.catchingUp.Load() {
|
|
return nil // the catch-up of a first start keeps its one revision at its end (CatchUp)
|
|
}
|
|
ix.mu.Lock()
|
|
src := int64(ix.source.rev)
|
|
ix.mu.Unlock()
|
|
return ix.keepBranchEntries(ctx, TrunkBranch, src, entries)
|
|
}
|
|
|
|
// KeepBranchEntries keeps entries as the revision of a watched branch after that branch's newest (design 47 G §291,
|
|
// A-85): trunk.<revision> for the first watched branch, branch.<branch>.<revision> for another, numbered per branch, and
|
|
// says catalog-revision naming the branch. src is the branch's source revision the entries were taken in from.
|
|
func (ix *Indexer) KeepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry) error {
|
|
return ix.keepBranchEntries(ctx, branch, src, entries)
|
|
}
|
|
|
|
func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry) error {
|
|
if branch == "" {
|
|
branch = TrunkBranch
|
|
}
|
|
ix.keepMu.Lock()
|
|
defer ix.keepMu.Unlock()
|
|
newest, any, err := newestOf(ctx, ix.Revisions, branch)
|
|
if errors.Is(err, process.ErrNotYet) {
|
|
ix.logf("catalog graph: no revision is kept: its objects %s are not on the bus yet", CatalogRevisionsObjects)
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
ix.logf("catalog graph: no revision is kept: the newest is not read: %v", err)
|
|
return fmt.Errorf("the newest catalog graph revision: %w", err)
|
|
}
|
|
entries = WithEdges(ix.withActs(ctx, entries)) // the acting rules and the edges between artifact references are written here, once, for every reader (A-208)
|
|
at := ix.now().UTC()
|
|
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: newest.Revision + 1, Kind: "trunk", Branch: branch, Source: src, At: &at}, entries)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
changed := changedEntries(newest.Entries, rev.Entries)
|
|
if (any && len(changed) == 0) || (!any && len(entries) == 0) {
|
|
return nil // nothing moved; an empty first revision is never kept, so the first start's catch-up still runs
|
|
}
|
|
used, capacity, fillErr := ix.Revisions.Fill(ctx)
|
|
if fillErr == nil && capacity > 0 && used+newBytes(ctx, ix.Revisions, rev, objects) > capacity {
|
|
// W4.6 (design 47, Two graphs): a full store refuses the next revision, said once per source revision, and the
|
|
// take-in that made it holds its pending merge with the reason objects full until room is said.
|
|
refused := fmt.Sprintf("the next revision; source revision %d held pending with the reason %s", src, HeldObjectsFull)
|
|
ix.logf("catalog graph: revision %d of source revision %d is not kept: %s is full (%d of %d bytes)", rev.Revision, src, CatalogRevisionsObjects, used, capacity)
|
|
ix.sayFill(ctx, meshcatalog.EventFull, fmt.Sprintf("%d", src), meshcatalog.Fill{Bytes: used, Cap: capacity, Refused: refused})
|
|
return fmt.Errorf("revision %d of source revision %d: %w", rev.Revision, src, ErrObjectsFull)
|
|
}
|
|
if err := ix.indexEntries(ctx, rev, entries); err != nil {
|
|
ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err)
|
|
return err
|
|
}
|
|
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
|
|
ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err)
|
|
return err
|
|
}
|
|
// The revision before this one in its branch ceases to be current (§579): it goes unless a record pins it.
|
|
if removed, err := ix.judgePinsLocked(ctx); err != nil {
|
|
ix.logf("pins: the revisions no record pins are not removed: %v", err)
|
|
} else if len(removed) > 0 {
|
|
ix.logf("pins: %d revisions removed (%s), no record pins them and none is current", len(removed), strings.Join(removed, ", "))
|
|
}
|
|
if _, err := ix.removeUnlisted(ctx); err != nil {
|
|
ix.logf("catalog graph: the entries nothing lists or names any more are not removed: %v", err) // at the first revision after none does (§675)
|
|
}
|
|
if after, capacity, err := ix.Revisions.Fill(ctx); fillErr == nil && err == nil && capacity > 0 && used*5 < capacity*4 && after*5 >= capacity*4 {
|
|
ix.sayFill(ctx, meshcatalog.EventFilling, fmt.Sprintf("%d", rev.Revision), meshcatalog.Fill{Bytes: after, Cap: capacity})
|
|
}
|
|
ix.logf("catalog graph: revision %d of source revision %d kept, %d entries, %d changed", rev.Revision, src, len(rev.Entries), len(changed))
|
|
if ix.SayCatalogRevision != nil {
|
|
if err := ix.SayCatalogRevision(ctx, meshcatalog.CatalogRevisionEvent{V: CatalogEntryLayout, Revision: rev.Revision, Kind: rev.Kind,
|
|
Branch: keys.EscapeName(branch), Source: src, At: &at, Changed: changed, Entries: int64(len(rev.Entries))}); err != nil {
|
|
ix.logf("catalog graph: revision %d is not said: %v", rev.Revision, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// HeldObjectsFull is the reason a pending merge is held while the catalog graph revisions are full (Pending.Held).
|
|
const HeldObjectsFull = "objects full"
|
|
|
|
// ErrObjectsFull is a revision refused because the catalog graph revisions are at their cap.
|
|
var ErrObjectsFull = errors.New("the catalog graph revisions are full")
|
|
|
|
// newBytes is the size of what keeping rev adds: its list and each entry the bucket does not hold yet.
|
|
func newBytes(ctx context.Context, o Objects, rev meshcatalog.CatalogRevision, entries map[string][]byte) int64 {
|
|
held, _ := o.Names(ctx)
|
|
have := make(map[string]bool, len(held))
|
|
for _, n := range held {
|
|
have[n] = true
|
|
}
|
|
list, _ := json.Marshal(rev)
|
|
size := int64(len(list))
|
|
for _, ref := range rev.Entries {
|
|
if !have[EntryObject(ref.Digest)] {
|
|
size += int64(len(entries[ref.Digest]))
|
|
}
|
|
}
|
|
return size
|
|
}
|
|
|
|
// sayFill says the seat's event filling, full or room of the catalog graph revisions, under the duplicate id of the
|
|
// event and what it is about, so the same one said twice is heard once.
|
|
func (ix *Indexer) sayFill(ctx context.Context, event, about string, f meshcatalog.Fill) {
|
|
if ix.SayFill == nil {
|
|
return
|
|
}
|
|
at := ix.now().UTC()
|
|
f.At, f.Store = &at, CatalogRevisionsObjects
|
|
if err := ix.SayFill(ctx, event, event+"-"+about, f); err != nil {
|
|
ix.logf("catalog graph: %s is not said: %v", event, err)
|
|
}
|
|
}
|
|
|
|
// RoomAgain says room and evaluates pending again when a merge held with the reason objects full finds the store
|
|
// below its filling mark (80 % of its cap): called after anything removes catalog graph revisions (no timer). It answers whether room was said.
|
|
func (ix *Indexer) RoomAgain(ctx context.Context) (bool, error) {
|
|
if ix.Revisions == nil || ix.Pending == nil {
|
|
return false, nil
|
|
}
|
|
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
|
|
if err != nil || !slices.ContainsFunc(entries, func(e PendingEntry) bool { return e.Value.Held == HeldObjectsFull }) {
|
|
return false, err
|
|
}
|
|
used, capacity, err := ix.Revisions.Fill(ctx)
|
|
if err != nil || (capacity > 0 && used*5 >= capacity*4) {
|
|
return false, err // room is said below the filling mark, 80 %, so one refusal is not followed by another at once
|
|
}
|
|
ix.sayFill(ctx, meshcatalog.EventRoom, fmt.Sprintf("%d", used), meshcatalog.Fill{Bytes: used, Cap: capacity})
|
|
return true, ix.PendingAgain(ctx) // room in catalog-revisions: pending evaluated again
|
|
}
|
|
|
|
// TrunkView is the indexer's view of the trunk read from the newest trunk revision the bucket keeps (the lead's ruling
|
|
// on milestone 47/G finding 1; ADR 0656 §6, one form of the fact): a store in memory holding each module's value as
|
|
// ModuleValueOf reads it back from its entry, under its modules key, a store in memory of each module's node of the module
|
|
// graph (NodeOf), and the revision it was read at (0 when the bucket keeps none yet: both start empty and the first start's
|
|
// catch-up fills them). A lost entry is an error naming the module (ReadRevision), never a module left out of the view.
|
|
func TrunkView(ctx context.Context, o Objects) (modules, nodes Store, at int64, err error) {
|
|
return BranchView(ctx, o, TrunkBranch)
|
|
}
|
|
|
|
// BranchView is TrunkView of a watched branch: the view read from the newest revision the bucket keeps of that branch
|
|
// (design 47 G §291, A-93), empty and at 0 when it keeps none.
|
|
func BranchView(ctx context.Context, o Objects, branch string) (modules, nodes Store, at int64, err error) {
|
|
view, held := newOverlay(nil), newOverlay(nil)
|
|
newest, any, err := newestOf(ctx, o, branch)
|
|
if err != nil || !any {
|
|
return view, held, 0, err
|
|
}
|
|
_, entries, err := ReadRevision(ctx, o, RevisionObject(newest))
|
|
if err != nil {
|
|
return nil, nil, 0, err
|
|
}
|
|
for _, e := range entries {
|
|
key, err := meshcatalogclient.ModulesKey(e.Module)
|
|
if err != nil {
|
|
return nil, nil, 0, fmt.Errorf("trunk.%d: the entry of %q: %w", newest.Revision, e.Module, err)
|
|
}
|
|
data, err := meshcatalogclient.EncodeModule(ModuleValueOf(e))
|
|
if err != nil {
|
|
return nil, nil, 0, fmt.Errorf("trunk.%d: the entry of %s: %w", newest.Revision, e.Module, err)
|
|
}
|
|
if err := view.Create(ctx, key, data); err != nil {
|
|
return nil, nil, 0, fmt.Errorf("trunk.%d: the entry of %s: %w", newest.Revision, e.Module, err)
|
|
}
|
|
gk, err := graph.Key(e.Module)
|
|
if err != nil {
|
|
continue // a module the graph cannot name has no node, as writeGraph writes none
|
|
}
|
|
node, err := graph.Encode(graph.NodeOf(e))
|
|
if err != nil {
|
|
return nil, nil, 0, fmt.Errorf("trunk.%d: the node of %s: %w", newest.Revision, e.Module, err)
|
|
}
|
|
if err := held.Create(ctx, gk, node); err != nil {
|
|
return nil, nil, 0, fmt.Errorf("trunk.%d: the node of %s: %w", newest.Revision, e.Module, err)
|
|
}
|
|
}
|
|
return view, held, newest.Revision, nil
|
|
}
|
|
|
|
// MemoryStore is a store held in memory alone: the module graph's nodes, which each revision's entries carry, so the
|
|
// indexer keeps no state of them on the bus (milestone 47/G).
|
|
func MemoryStore() Store { return newOverlay(nil) }
|
|
|
|
// The view of the trunk is the newest catalog graph revision the indexer's objects catalog-revisions keep (milestone
|
|
// 47/G, ADR 0656 §6): the tools bundle answers the modules and the module graph it once read from the states modules and
|
|
// graph from that revision, at its number, and says it waits while the indexer keeps none yet (ADR 0615 §6), never an
|
|
// empty catalog.
|
|
|
|
// ErrNoRevision is a view asked before the indexer kept its first trunk revision.
|
|
var ErrNoRevision = errors.New("view-waits: the indexer keeps no catalog graph revision yet (its first start takes the trunk in); ask again on catalog-revision")
|
|
|
|
// TrunkStates reads the view of the trunk from the newest trunk revision; under answers every other state.
|
|
type TrunkStates struct {
|
|
Objects Objects
|
|
Under func(string) views.Source
|
|
// Kept, when set, holds the view of the newest revision read, so a read asks only which revision is newest and reads
|
|
// the view again only when a newer one is kept: without it every key read the whole revision again, one object per
|
|
// module, and a first view of the modules took about 11 s a page. Nil reads the view at every read.
|
|
Kept *KeptTrunk
|
|
}
|
|
|
|
// KeptTrunk is the view of the trunk as last read, at its revision, shared by every read of a TrunkStates; Say, when set,
|
|
// says each read of a revision and how long it took.
|
|
type KeptTrunk struct {
|
|
Say func(format string, a ...any)
|
|
mu sync.Mutex
|
|
at int64
|
|
mods Store
|
|
held Store
|
|
}
|
|
|
|
// read is the view's modules and nodes, and the revision they were read at; none kept yet is ErrNoRevision.
|
|
func (t TrunkStates) read(nodes bool) (Store, int64, error) {
|
|
mods, held, at, err := t.view()
|
|
if err == nil && at == 0 {
|
|
err = ErrNoRevision
|
|
}
|
|
if nodes {
|
|
return held, at, err
|
|
}
|
|
return mods, at, err
|
|
}
|
|
|
|
// view is the view of the newest trunk revision: the one kept when it is still the newest, else read whole and kept.
|
|
func (t TrunkStates) view() (modules, nodes Store, at int64, err error) {
|
|
ctx := context.Background()
|
|
if t.Kept == nil {
|
|
return TrunkView(ctx, t.Objects)
|
|
}
|
|
k := t.Kept
|
|
newest, any, err := newestOf(ctx, t.Objects, TrunkBranch)
|
|
if err != nil || !any {
|
|
return newOverlay(nil), newOverlay(nil), 0, err
|
|
}
|
|
k.mu.Lock()
|
|
defer k.mu.Unlock()
|
|
if k.mods != nil && k.at == newest.Revision {
|
|
return k.mods, k.held, k.at, nil
|
|
}
|
|
started := time.Now()
|
|
mods, held, at, err := TrunkView(ctx, t.Objects)
|
|
if err != nil {
|
|
if k.Say != nil {
|
|
k.Say("the view of the trunk: revision %d cannot be read after %s: %v", newest.Revision, time.Since(started).Round(time.Millisecond), err)
|
|
}
|
|
return nil, nil, 0, err
|
|
}
|
|
if k.Say != nil {
|
|
n, _ := mods.Keys(ctx)
|
|
k.Say("the view of the trunk: read revision %d (was %d), %d modules, in %s", at, k.at, len(n), time.Since(started).Round(time.Millisecond))
|
|
}
|
|
k.mods, k.held, k.at = mods, held, at
|
|
return mods, held, at, nil
|
|
}
|
|
|
|
// state is the source of a state by its name: modules and graph from the revision, their marks at its number, every
|
|
// other the indexer's own state.
|
|
func (t TrunkStates) State(name string) views.Source {
|
|
switch name {
|
|
case "modules", graph.State:
|
|
return revisionSource{t: t, nodes: name == graph.State}
|
|
case views.MarksState:
|
|
return markSource{t: t, under: t.Under(name)}
|
|
}
|
|
return t.Under(name)
|
|
}
|
|
|
|
// revisionSource is the modules or the module graph of the newest trunk revision, each value at its number.
|
|
type revisionSource struct {
|
|
t TrunkStates
|
|
nodes bool
|
|
}
|
|
|
|
func (r revisionSource) Keys() ([]string, error) {
|
|
s, _, err := r.t.read(r.nodes)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return s.Keys(context.Background())
|
|
}
|
|
|
|
func (r revisionSource) Get(key string) (*views.Entry, bool, error) {
|
|
s, at, err := r.t.read(r.nodes)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
value, _, found, err := s.Get(context.Background(), key)
|
|
if err != nil || !found {
|
|
return nil, false, err
|
|
}
|
|
return &views.Entry{Value: value, Revision: uint64(at)}, true, nil
|
|
}
|
|
|
|
// Page is the values in key order after a key from one read of the revision, as many as fit maxBytes and at least one
|
|
// (views.Pager): a first view's page reads the revision once, never once per key.
|
|
func (r revisionSource) Page(after string, maxBytes int, _ time.Duration) ([]views.KeyedEntry, string, error) {
|
|
s, at, err := r.t.read(r.nodes)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
ctx := context.Background()
|
|
keys, err := s.Keys(ctx)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
slices.Sort(keys)
|
|
out, size := []views.KeyedEntry{}, 0
|
|
for _, k := range keys {
|
|
if after != "" && k <= after {
|
|
continue
|
|
}
|
|
value, _, found, err := s.Get(ctx, k)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
if !found {
|
|
continue
|
|
}
|
|
if len(out) > 0 && size+len(k)+len(value) > maxBytes {
|
|
return out, out[len(out)-1].Key, nil
|
|
}
|
|
out = append(out, views.KeyedEntry{Key: k, Entry: views.Entry{Value: value, Revision: uint64(at)}})
|
|
size += len(k) + len(value)
|
|
}
|
|
return out, "", nil
|
|
}
|
|
|
|
// markSource answers the mark of modules and graph as the newest trunk revision's number, which catalog-revision says
|
|
// the changes after; every other mark is the views family's.
|
|
type markSource struct {
|
|
t TrunkStates
|
|
under views.Source
|
|
}
|
|
|
|
func (m markSource) Keys() ([]string, error) { return m.under.Keys() }
|
|
|
|
func (m markSource) Get(state string) (*views.Entry, bool, error) {
|
|
if state != "modules" && state != graph.State {
|
|
return m.under.Get(state)
|
|
}
|
|
_, at, err := m.t.read(false)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
data, err := json.Marshal(views.Mark{V: views.Version, State: state, Revision: uint64(at), At: time.Now().UTC()})
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
return &views.Entry{Value: data, Revision: uint64(at)}, true, nil
|
|
}
|
|
|
|
// manifestBytes is the manifest the module's value names by its digest, as the forge answered it; empty when it is not kept.
|
|
func (ix *Indexer) manifestBytes(ctx context.Context, digest string) string {
|
|
if digest == "" || ix.Manifests == nil {
|
|
return ""
|
|
}
|
|
raw, _, err := ix.manifestRaw(ctx, digest)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return raw
|
|
}
|
|
|
|
// NewestRevision is the number of the newest catalog graph revision kept for the watched branch, 0 when none is kept: the
|
|
// revision a take-in has just made, said with its take-in.taken (design 47 G §476, A-448).
|
|
func (ix *Indexer) NewestRevision(ctx context.Context, branch string) int64 {
|
|
if ix.Revisions == nil {
|
|
return 0
|
|
}
|
|
newest, any, err := newestOf(ctx, ix.Revisions, branch)
|
|
if err != nil || !any {
|
|
return 0
|
|
}
|
|
return newest.Revision
|
|
}
|