Files
mesh-delivery/modules/mesh-delivery-indexer/internal/index/versions.go
T
jschoubben 1175fef241 Catalog versions: the floor reads every package's holders too, asks again on a growing pause, and a floor entry takes its release
The floor of a module with no version is the highest among its Go tags and the versions each package holder answers through
its seat's verb versions (npm-package-registry, go-package-registry; the sdk's npm 0.1.18 above its Go tags 0.1.16). An
unanswered read is asked again on a growing pause (5 s, 15 s, 30 s, 1, 2, 5 min) until it answers, with call-unanswered five
minutes after the first. A floor entry holds no artifacts, so a package-published heard later at its version is recorded as
its release and says version-released. One whole-indexer integration test on a real bus with a stand-in seat git and holder.

Builds: design 47 G §412, A-490c, A-490d, A-490e, A-490f, A-490g, A-490h
Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
2026-11-19 17:33:06 +01:00

355 lines
14 KiB
Go

package index
import (
"cmp"
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strconv"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The catalog's released versions (design 47 G §349, §372, §374): the indexer keeps, for the seat mesh-catalog, each module's
// released versions in the state versions, one key per module and version (<module>.<version>) holding each artifact's digest,
// the source revision built and its edge entries. It hears every artifact's package-published and records a version as
// released once every artifact the module's entry names is published at it.
const (
VersionsState = "versions"
// VersionsKept is how many versions of a module the state keeps (§349): at each release the module's oldest beyond them
// go, never its highest of any MAJOR.MINOR.
VersionsKept = 100
// PendingPrefix opens the keys of the state heard (Heard) that keep, per module and version not yet released, the
// artifacts published at it so far, each key part escaped as a name (A-80, §73).
PendingPrefix = "pending."
)
// published is what a not yet released version holds: each artifact published at it, by reference.
type published struct {
Artifacts map[string]meshcatalog.VersionArtifact `json:"artifacts"`
Dependencies []meshbuilds.Dependency `json:"dependencies,omitempty"`
Source int64 `json:"source,omitempty"`
}
// versionParts is MAJOR.MINOR.PATCH, read; false for any other form.
func versionParts(v string) ([3]int, bool) {
var out [3]int
parts := strings.Split(v, ".")
if len(parts) != 3 {
return out, false
}
for i, p := range parts {
n, err := strconv.Atoi(p)
if err != nil || n < 0 {
return out, false
}
out[i] = n
}
return out, true
}
// compareVersions orders two versions MAJOR.MINOR.PATCH numerically.
func compareVersions(a, b string) int {
x, _ := versionParts(a)
y, _ := versionParts(b)
for i := range x {
if x[i] != y[i] {
return x[i] - y[i]
}
}
return 0
}
// ModuleVersions reads every released version the state keeps for module, newest first.
func ModuleVersions(ctx context.Context, st Store, module string) ([]meshcatalog.VersionEntry, error) {
all, err := st.Keys(ctx)
if err != nil {
return nil, err
}
var out []meshcatalog.VersionEntry
for _, k := range all {
m, _, ok := meshcatalog.ParseVersionsByModuleVersionKey(k)
if !ok || m != module {
continue
}
raw, _, found, err := st.Get(ctx, k)
if err != nil {
return nil, err
}
if !found {
continue
}
v, err := meshcatalog.DecodeVersionsByModuleVersionValue(raw)
if err != nil {
return nil, fmt.Errorf("the key %s of the state %s: %w", k, VersionsState, err)
}
out = append(out, v)
}
slices.SortFunc(out, func(a, b meshcatalog.VersionEntry) int { return compareVersions(b.Version, a.Version) })
return out, nil
}
// HearPackagePublished takes one artifact's package-published (design 47 G §349): the artifact is kept as published at its
// version, and once every artifact the module's catalog entry names is published at it the version is released: recorded,
// and said as version-released unless a higher one is released already (then it is recorded marked late and not said). A
// build of a watched branch other than the registration's first releases none (§371) and keeps its edges in build-edges (§374, A-83).
func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.PackagePublished) error {
if ix.Versions == nil || ix.Heard == nil {
return errors.New("the state versions is not kept here")
}
if _, ok := versionParts(p.Version); !ok {
return fmt.Errorf("the package-published of %s names the version %q, which is no MAJOR.MINOR.PATCH", p.Artifact, p.Version)
}
ix.logf("versions: heard the package-published of %s at %s (digest %s, branch %q, source %d)", p.Artifact, p.Version, p.Digest, p.Branch, p.Source)
if ix.Registry != nil {
r, found, err := ix.Registry.Registration(ctx, p.Module)
switch {
case err != nil:
ix.logf("versions: the registration of %s cannot be read, so %s at %s is taken as the first watched branch's: %v", p.Module, p.Artifact, p.Version, err)
case !found:
ix.logf("versions: %s has no registration, so %s at %s is taken as the first watched branch's", p.Module, p.Artifact, p.Version)
case len(r.Branches) > 0 && r.Branches[0] != p.Branch:
ix.logf("versions: %s at %s is of the branch %q, not the first watched branch %q: it releases no version, its edges are kept in build-edges", p.Artifact, p.Version, p.Branch, r.Branches[0])
return ix.hearBranchBuild(ctx, p) // releases no version; its edges are kept in build-edges (A-83)
}
}
if raw, _, found, err := ix.Versions.Get(ctx, meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)); err != nil {
return err
} else if found && !isFloor(raw) {
ix.logf("versions: %s %s is released already, so the package-published of %s is heard again and changes nothing", p.Module, p.Version, p.Artifact)
return nil
}
entry, err := ix.entryOf(ctx, p.Module)
if err != nil {
return err
}
var want []string
for _, a := range entry.Artifacts {
want = append(want, a.Ref)
}
key := PendingPrefix + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Version)
var have published
err = write(ctx, ix.Heard, key, func(raw []byte, found bool) ([]byte, bool, error) {
have = published{}
if found {
if err := json.Unmarshal(raw, &have); err != nil {
return nil, false, err
}
}
if have.Artifacts == nil {
have.Artifacts = map[string]meshcatalog.VersionArtifact{}
}
have.Artifacts[p.Artifact] = meshcatalog.VersionArtifact{Artifact: p.Artifact, Digest: p.Digest}
have.Source = p.Source
for _, d := range p.Dependencies {
if !slices.ContainsFunc(have.Dependencies, func(x meshbuilds.Dependency) bool { return x.Kind == d.Kind && x.Part == d.Part }) {
have.Dependencies = append(have.Dependencies, d)
}
}
data, err := json.Marshal(have)
return data, false, err
})
if err != nil {
return err
}
var missing []string
for _, ref := range want {
if _, ok := have.Artifacts[ref]; !ok {
missing = append(missing, ref)
}
}
if len(missing) > 0 {
ix.logf("versions: %s %s waits for the package-published of %s, which the newest trunk entry of it names (fingerprint %s); heard so far: %d", p.Module, p.Version, strings.Join(missing, ", "), entry.Fp, len(have.Artifacts))
return nil
}
return ix.releaseVersion(ctx, p, key, entry, have)
}
// entryOf is the module's entry in the newest trunk revision.
func (ix *Indexer) entryOf(ctx context.Context, module string) (meshcatalog.CatalogEntry, error) {
if ix.Revisions == nil {
return meshcatalog.CatalogEntry{}, errors.New("the catalog graph is not kept here: no entry names the module's artifacts")
}
_, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
if err != nil {
return meshcatalog.CatalogEntry{}, err
}
if !found {
return meshcatalog.CatalogEntry{}, errors.New("the indexer keeps no trunk revision yet")
}
for _, e := range entries {
if e.Module == module {
return e, nil
}
}
return meshcatalog.CatalogEntry{}, fmt.Errorf("the trunk holds no module %s", module)
}
// releaseVersion records the version of p whole, says version-released unless it is late, deletes what exceeds the kept versions,
// and forgets the artifacts kept as published.
func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePublished, pendingKey string, entry meshcatalog.CatalogEntry, have published) error {
key := meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)
kept, err := ModuleVersions(ctx, ix.Versions, p.Module)
if err != nil {
return err
}
late := len(kept) > 0 && compareVersions(kept[0].Version, p.Version) > 0
at := ix.now().UTC()
v := meshcatalog.VersionEntry{V: int64(keys.Version), Module: p.Module, Version: p.Version, Source: have.Source, At: &at, Late: late}
for _, a := range entry.Artifacts {
v.Artifacts = append(v.Artifacts, have.Artifacts[a.Ref])
}
slices.SortFunc(v.Artifacts, func(a, b meshcatalog.VersionArtifact) int { return strings.Compare(a.Artifact, b.Artifact) })
v.Edges = releaseEdges(entry, have.Dependencies)
data, err := json.Marshal(v)
if err != nil {
return err
}
if err := ix.Versions.Create(ctx, key, data); errors.Is(err, ErrConflict) {
// A floor entry (A-490h) holds no artifacts: this publish is that version's release, so it takes the entry's place.
if raw, rev, found, gerr := ix.Versions.Get(ctx, key); gerr == nil && found && isFloor(raw) {
if err := ix.Versions.Update(ctx, key, data, rev); err != nil && !errors.Is(err, ErrConflict) {
return err
}
}
} else if err != nil {
return err
}
if late {
ix.logf("versions: %s %s is released late: %s is released already, so it is recorded and no version-released is said", p.Module, p.Version, kept[0].Version)
} else if ix.SayVersionReleased == nil {
ix.logf("versions: %s %s is released, and no version-released is said here", p.Module, p.Version)
}
if !late && ix.SayVersionReleased != nil {
ev := meshcatalog.VersionReleasedEvent{V: int64(keys.Version), Module: v.Module, Version: v.Version, Artifacts: v.Artifacts, Source: v.Source, At: &at}
if err := ix.SayVersionReleased(ctx, ev, meshcatalog.EventVersionReleased+"-"+key); err != nil {
return fmt.Errorf("version-released of %s %s is not said: %w", p.Module, p.Version, err)
}
ix.logf("versions: %s %s is released and version-released is said (%d artifacts, source %d)", p.Module, p.Version, len(v.Artifacts), v.Source)
}
if err := ix.trimVersions(ctx, p.Module); err != nil {
ix.logf("versions: %s's oldest versions are not deleted: %v", p.Module, err)
}
return ix.forgetPublished(ctx, pendingKey)
}
// trimVersions deletes the module's oldest versions beyond the kept 100, never its highest of any MAJOR.MINOR.
func (ix *Indexer) trimVersions(ctx context.Context, module string) error {
kept, err := ModuleVersions(ctx, ix.Versions, module) // newest first
if err != nil || len(kept) <= VersionsKept {
return err
}
highest := map[string]bool{} // the highest version of each MAJOR.MINOR, as the list is newest first
seen := map[string]bool{}
for _, v := range kept {
p := strings.Split(v.Version, ".")
if mm := p[0] + "." + p[1]; !seen[mm] {
seen[mm] = true
highest[v.Version] = true
}
}
excess := len(kept) - VersionsKept
for i := len(kept) - 1; i >= 0 && excess > 0; i-- {
v := kept[i].Version
if highest[v] {
continue
}
key := meshcatalog.VersionsByModuleVersionKey(module, v)
if _, rev, found, err := ix.Versions.Get(ctx, key); err == nil && found {
if err := ix.Versions.Delete(ctx, key, rev); err != nil {
return err
}
}
excess--
}
return nil
}
// forgetPublished removes the artifacts kept as published at a version released.
func (ix *Indexer) forgetPublished(ctx context.Context, key string) error {
if _, rev, found, err := ix.Heard.Get(ctx, key); err != nil || !found {
return err
} else {
return ix.Heard.Delete(ctx, key, rev)
}
}
// Versions answers the verb versions: a module's released versions (design 47 G §349), newest first.
func Versions(ctx context.Context, st Store, in meshcatalog.VersionsInput) (meshcatalog.VersionsAnswer, error) {
if in.Module == "" {
return meshcatalog.VersionsAnswer{}, errors.New("name the module (module)")
}
v, err := ModuleVersions(ctx, st, in.Module)
if v == nil {
v = []meshcatalog.VersionEntry{}
}
return meshcatalog.VersionsAnswer{Module: in.Module, Versions: v}, err
}
// VersionsTool is the verb versions as the tool mesh-catalog.versions of the indexer's own runtime.
func VersionsTool(st Store) stdio.Tool {
return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbVersions, Run: func(args map[string]any) (any, error) {
var in meshcatalog.VersionsInput
if raw, err := json.Marshal(args); err == nil {
_ = json.Unmarshal(raw, &in)
}
return Versions(context.Background(), st, in)
}}
}
// notYetDelivered is the version of an edge entry of kind 2, 4 or 6 until its release is first delivered (design 47 G §374).
const notYetDelivered = "not yet delivered"
// releaseEdges are the edge entries of a release (design 47 G §374): kinds 1 and 3 from the dependencies the build
// resolved, as package-published carried them (stale: the dependent is made from them, §378); kinds 2, 4 and 6 of the entry's
// catalog graph edges, reading "not yet delivered" until the first delivery writes them. Kind 5 is not written here.
func releaseEdges(entry meshcatalog.CatalogEntry, deps []meshbuilds.Dependency) []meshcatalog.VersionEdge {
word := func(kind int64, part string) string {
for _, e := range entry.Edges {
if e.Kind == kind && (e.Part == part || e.From == part) {
return e.Word
}
}
return ""
}
var out []meshcatalog.VersionEdge
for _, d := range deps {
out = append(out, meshcatalog.VersionEdge{Kind: d.Kind, Part: d.Part, Digest: d.Digest, Version: d.Version, Stale: d.Kind == 1 || d.Kind == 3, Word: word(d.Kind, d.Part)})
}
for _, e := range entry.Edges {
if e.Kind != 2 && e.Kind != 4 && e.Kind != 6 {
continue
}
part := cmp.Or(e.Part, e.From)
if !slices.ContainsFunc(out, func(x meshcatalog.VersionEdge) bool { return x.Kind == e.Kind && x.Part == part && x.Word == e.Word }) {
out = append(out, meshcatalog.VersionEdge{Kind: e.Kind, Part: part, Word: e.Word, Version: notYetDelivered})
}
}
return out
}
// OutdatedTool is the verb outdated as the tool mesh-catalog.outdated of the indexer's own runtime: it reads the versions
// state and the heard toolchain and delivered keys of the own state.
func OutdatedTool(ix *Indexer) stdio.Tool {
return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbOutdated, Run: func(args map[string]any) (any, error) {
var in meshcatalog.OutdatedInput
if raw, err := json.Marshal(args); err == nil {
_ = json.Unmarshal(raw, &in)
}
return ix.Outdated(context.Background(), in)
}}
}
// isFloor says whether a kept version is a floor entry (A-490h): it holds no artifact, as no release does.
func isFloor(raw []byte) bool {
var v meshcatalog.VersionEntry
return json.Unmarshal(raw, &v) == nil && len(v.Artifacts) == 0
}