Files
mesh-delivery/modules/mesh-delivery-indexer/internal/index/versions.go
T
jschoubben 47ea130332 Design 47 H1 (1): a release re-pin keeps the next catalog revision of the same source on every watched branch
A branch's revision number is the indexer's own count (the newest + 1); the gate's source is the revision's field source, which the take-in's dedupe and source-to-take read. A catalog revision carries its call (the seat mesh-catalog's CatalogRevision and catalog-revision gain call, regenerated by seatgen). A re-pin on a module's release runs on every watched branch holding a module that names its package (the first on the modules state, another on its newest revision's picture), and keeps, where a fingerprint moved, the branch's next revision of the same source with the call 'release of <module> <version>' and the moved modules as its movers, pinned for the release's pipeline, and says catalog-revision; no take-in.taken.

Builds: design 47 H S11, S12, S13, S14 (H-pins-spec-and-gaps item 1), H-A-211, H-A-212, H-A-213 (1)-(4: the pin of every revision a pipeline runs on), H-A-147
2026-11-20 23:36:39 +01:00

463 lines
19 KiB
Go

package index
import (
"cmp"
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"strconv"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/keys"
"mesh-delivery-indexer/internal/fingerprint"
)
// The catalog's released versions (design 47 G §349, §372, §374): the indexer keeps, for the seat mesh-catalog, each module's
// released versions in the state versions, one key per module and version (<module>.<version>) holding each artifact's digest,
// the source revision built and its edge entries. It hears every artifact's package-published and records a version as
// released once every artifact the module's entry names is published at it.
const (
VersionsState = "versions"
// VersionsKept is how many versions of a module the state keeps (§349): at each release the module's oldest beyond them
// go, never its highest of any MAJOR.MINOR.
VersionsKept = 100
// PendingPrefix opens the keys of the state heard (Heard) that keep, per module and version not yet released, the
// artifacts published at it so far, each key part escaped as a name (A-80, §73).
PendingPrefix = "pending."
)
// published is what a not yet released version holds: each artifact published at it, by reference.
type published struct {
Artifacts map[string]meshcatalog.VersionArtifact `json:"artifacts"`
Dependencies []meshbuilds.Dependency `json:"dependencies,omitempty"`
Source int64 `json:"source,omitempty"`
}
// versionParts is MAJOR.MINOR.PATCH, read; false for any other form.
func versionParts(v string) ([3]int, bool) {
var out [3]int
parts := strings.Split(v, ".")
if len(parts) != 3 {
return out, false
}
for i, p := range parts {
n, err := strconv.Atoi(p)
if err != nil || n < 0 {
return out, false
}
out[i] = n
}
return out, true
}
// compareVersions orders two versions MAJOR.MINOR.PATCH numerically.
func compareVersions(a, b string) int {
x, _ := versionParts(a)
y, _ := versionParts(b)
for i := range x {
if x[i] != y[i] {
return x[i] - y[i]
}
}
return 0
}
// ModuleVersions reads every released version the state keeps for module, newest first.
func ModuleVersions(ctx context.Context, st Store, module string) ([]meshcatalog.VersionEntry, error) {
all, err := st.Keys(ctx)
if err != nil {
return nil, err
}
var out []meshcatalog.VersionEntry
for _, k := range all {
m, _, ok := meshcatalog.ParseVersionsByModuleVersionKey(k)
if !ok || m != module {
continue
}
raw, _, found, err := st.Get(ctx, k)
if err != nil {
return nil, err
}
if !found {
continue
}
v, err := meshcatalog.DecodeVersionsByModuleVersionValue(raw)
if err != nil {
return nil, fmt.Errorf("the key %s of the state %s: %w", k, VersionsState, err)
}
out = append(out, v)
}
slices.SortFunc(out, func(a, b meshcatalog.VersionEntry) int { return compareVersions(b.Version, a.Version) })
return out, nil
}
// HearPackagePublished takes one artifact's package-published (design 47 G §349): the artifact is kept as published at its
// version, and once every artifact the module's catalog entry names is published at it the version is released: recorded,
// and said as version-released unless a higher one is released already (then it is recorded marked late and not said). A
// build of a watched branch other than the registration's first releases none (§371) and keeps its edges in build-edges (§374, A-83).
func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.PackagePublished) error {
if ix.Versions == nil || ix.Heard == nil {
return errors.New("the state versions is not kept here")
}
if _, ok := versionParts(p.Version); !ok {
return fmt.Errorf("the package-published of %s names the version %q, which is no MAJOR.MINOR.PATCH", p.Artifact, p.Version)
}
ix.logf("versions: heard the package-published of %s at %s (digest %s, branch %q, source %d)", p.Artifact, p.Version, p.Digest, p.Branch, p.Source)
if ix.Registry != nil {
r, found, err := ix.Registry.Registration(ctx, p.Module)
switch {
case err != nil:
ix.logf("versions: the registration of %s cannot be read, so %s at %s is taken as the first watched branch's: %v", p.Module, p.Artifact, p.Version, err)
case !found:
ix.logf("versions: %s has no registration, so %s at %s is taken as the first watched branch's", p.Module, p.Artifact, p.Version)
case len(r.Branches) > 0 && r.Branches[0] != p.Branch:
ix.logf("versions: %s at %s is of the branch %q, not the first watched branch %q: it releases no version, its edges are kept in build-edges", p.Artifact, p.Version, p.Branch, r.Branches[0])
return ix.hearBranchBuild(ctx, p) // releases no version; its edges are kept in build-edges (A-83)
}
}
if raw, _, found, err := ix.Versions.Get(ctx, meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)); err != nil {
return err
} else if found && !isFloor(raw) {
// A package-published delivered again after a re-pin failed (H-A-167): the re-pin is made again, nothing else.
ix.logf("versions: %s %s is released already, so the package-published of %s is heard again and only re-pins", p.Module, p.Version, p.Artifact)
return ix.repin(ctx, p.Module, p.Version)
}
entry, err := ix.entryOf(ctx, p.Module)
if err != nil {
return err
}
var want []string
for _, a := range entry.Artifacts {
want = append(want, a.Ref)
}
key := PendingPrefix + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Version)
var have published
err = write(ctx, ix.Heard, key, func(raw []byte, found bool) ([]byte, bool, error) {
have = published{}
if found {
if err := json.Unmarshal(raw, &have); err != nil {
return nil, false, err
}
}
if have.Artifacts == nil {
have.Artifacts = map[string]meshcatalog.VersionArtifact{}
}
have.Artifacts[p.Artifact] = meshcatalog.VersionArtifact{Artifact: p.Artifact, Digest: p.Digest}
have.Source = p.Source
for _, d := range p.Dependencies {
if !slices.ContainsFunc(have.Dependencies, func(x meshbuilds.Dependency) bool { return x.Kind == d.Kind && x.Part == d.Part }) {
have.Dependencies = append(have.Dependencies, d)
}
}
data, err := json.Marshal(have)
return data, false, err
})
if err != nil {
return err
}
var missing []string
for _, ref := range want {
if _, ok := have.Artifacts[ref]; !ok {
missing = append(missing, ref)
}
}
if len(missing) > 0 {
ix.logf("versions: %s %s waits for the package-published of %s, which the newest trunk entry of it names (fingerprint %s); heard so far: %d", p.Module, p.Version, strings.Join(missing, ", "), entry.Fp, len(have.Artifacts))
return nil
}
return ix.releaseVersion(ctx, p, key, entry, have)
}
// entryOf is the module's entry in the newest trunk revision.
func (ix *Indexer) entryOf(ctx context.Context, module string) (meshcatalog.CatalogEntry, error) {
if ix.Revisions == nil {
return meshcatalog.CatalogEntry{}, errors.New("the catalog graph is not kept here: no entry names the module's artifacts")
}
_, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
if err != nil {
return meshcatalog.CatalogEntry{}, err
}
if !found {
return meshcatalog.CatalogEntry{}, errors.New("the indexer keeps no trunk revision yet")
}
for _, e := range entries {
if e.Module == module {
return e, nil
}
}
return meshcatalog.CatalogEntry{}, fmt.Errorf("the trunk holds no module %s", module)
}
// releaseVersion records the version of p whole, says version-released unless it is late, deletes what exceeds the kept versions,
// and forgets the artifacts kept as published.
func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePublished, pendingKey string, entry meshcatalog.CatalogEntry, have published) error {
key := meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)
kept, err := ModuleVersions(ctx, ix.Versions, p.Module)
if err != nil {
return err
}
late := len(kept) > 0 && compareVersions(kept[0].Version, p.Version) > 0
at := ix.now().UTC()
v := meshcatalog.VersionEntry{V: int64(keys.Version), Module: p.Module, Version: p.Version, Source: have.Source, At: &at, Late: late}
for _, a := range entry.Artifacts {
v.Artifacts = append(v.Artifacts, have.Artifacts[a.Ref])
}
slices.SortFunc(v.Artifacts, func(a, b meshcatalog.VersionArtifact) int { return strings.Compare(a.Artifact, b.Artifact) })
v.Edges = releaseEdges(entry, have.Dependencies)
data, err := json.Marshal(v)
if err != nil {
return err
}
if err := ix.Versions.Create(ctx, key, data); errors.Is(err, ErrConflict) {
// A floor entry (A-490h) holds no artifacts: this publish is that version's release, so it takes the entry's place.
if raw, rev, found, gerr := ix.Versions.Get(ctx, key); gerr == nil && found && isFloor(raw) {
if err := ix.Versions.Update(ctx, key, data, rev); err != nil && !errors.Is(err, ErrConflict) {
return err
}
}
} else if err != nil {
return err
}
if late {
ix.logf("versions: %s %s is released late: %s is released already, so it is recorded and no version-released is said", p.Module, p.Version, kept[0].Version)
} else if ix.SayVersionReleased == nil {
ix.logf("versions: %s %s is released, and no version-released is said here", p.Module, p.Version)
}
if !late && ix.SayVersionReleased != nil {
ev := meshcatalog.VersionReleasedEvent{V: int64(keys.Version), Module: v.Module, Version: v.Version, Artifacts: v.Artifacts, Source: v.Source, At: &at}
if err := ix.SayVersionReleased(ctx, ev, meshcatalog.EventVersionReleased+"-"+key); err != nil {
return fmt.Errorf("version-released of %s %s is not said: %w", p.Module, p.Version, err)
}
ix.logf("versions: %s %s is released and version-released is said (%d artifacts, source %d)", p.Module, p.Version, len(v.Artifacts), v.Source)
}
if !late {
// The release re-pins every module whose packages hold one of it (design 47 H-A-165); one that fails is not
// acknowledged, so the package-published is delivered again (H-A-167).
if err := ix.repin(ctx, p.Module, p.Version); err != nil {
return err
}
}
if err := ix.trimVersions(ctx, p.Module); err != nil {
ix.logf("versions: %s's oldest versions are not deleted: %v", p.Module, err)
}
return ix.forgetPublished(ctx, pendingKey)
}
// trimVersions deletes the module's oldest versions beyond the kept 100, never its highest of any MAJOR.MINOR.
func (ix *Indexer) trimVersions(ctx context.Context, module string) error {
kept, err := ModuleVersions(ctx, ix.Versions, module) // newest first
if err != nil || len(kept) <= VersionsKept {
return err
}
highest := map[string]bool{} // the highest version of each MAJOR.MINOR, as the list is newest first
seen := map[string]bool{}
for _, v := range kept {
p := strings.Split(v.Version, ".")
if mm := p[0] + "." + p[1]; !seen[mm] {
seen[mm] = true
highest[v.Version] = true
}
}
excess := len(kept) - VersionsKept
for i := len(kept) - 1; i >= 0 && excess > 0; i-- {
v := kept[i].Version
if highest[v] {
continue
}
key := meshcatalog.VersionsByModuleVersionKey(module, v)
if _, rev, found, err := ix.Versions.Get(ctx, key); err == nil && found {
if err := ix.Versions.Delete(ctx, key, rev); err != nil {
return err
}
}
excess--
}
return nil
}
// forgetPublished removes the artifacts kept as published at a version released.
func (ix *Indexer) forgetPublished(ctx context.Context, key string) error {
if _, rev, found, err := ix.Heard.Get(ctx, key); err != nil || !found {
return err
} else {
return ix.Heard.Delete(ctx, key, rev)
}
}
// Versions answers the verb versions: a module's released versions (design 47 G §349), newest first.
func Versions(ctx context.Context, st Store, in meshcatalog.VersionsInput) (meshcatalog.VersionsAnswer, error) {
if in.Module == "" {
return meshcatalog.VersionsAnswer{}, errors.New("name the module (module)")
}
v, err := ModuleVersions(ctx, st, in.Module)
if v == nil {
v = []meshcatalog.VersionEntry{}
}
return meshcatalog.VersionsAnswer{Module: in.Module, Versions: v}, err
}
// VersionsTool is the verb versions as the tool mesh-catalog.versions of the indexer's own runtime.
func VersionsTool(st Store) stdio.Tool {
return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbVersions, Run: func(args map[string]any) (any, error) {
var in meshcatalog.VersionsInput
if raw, err := json.Marshal(args); err == nil {
_ = json.Unmarshal(raw, &in)
}
return Versions(context.Background(), st, in)
}}
}
// notYetDelivered is the version of an edge entry of kind 2, 4 or 6 until its release is first delivered (design 47 G §374).
const notYetDelivered = "not yet delivered"
// releaseEdges are the edge entries of a release (design 47 G §374): kinds 1 and 3 from the dependencies the build
// resolved, as package-published carried them (stale: the dependent is made from them, §378); kinds 2, 4 and 6 of the entry's
// catalog graph edges, reading "not yet delivered" until the first delivery writes them. Kind 5 is not written here.
func releaseEdges(entry meshcatalog.CatalogEntry, deps []meshbuilds.Dependency) []meshcatalog.VersionEdge {
word := func(kind int64, part string) string {
for _, e := range entry.Edges {
if e.Kind == kind && (e.Part == part || e.From == part) {
return e.Word
}
}
return ""
}
var out []meshcatalog.VersionEdge
for _, d := range deps {
out = append(out, meshcatalog.VersionEdge{Kind: d.Kind, Part: d.Part, Digest: d.Digest, Version: d.Version, Stale: d.Kind == 1 || d.Kind == 3, Word: word(d.Kind, d.Part)})
}
for _, e := range entry.Edges {
if e.Kind != 2 && e.Kind != 4 && e.Kind != 6 {
continue
}
part := cmp.Or(e.Part, e.From)
if !slices.ContainsFunc(out, func(x meshcatalog.VersionEdge) bool { return x.Kind == e.Kind && x.Part == part && x.Word == e.Word }) {
out = append(out, meshcatalog.VersionEdge{Kind: e.Kind, Part: part, Word: e.Word, Version: notYetDelivered})
}
}
return out
}
// OutdatedTool is the verb outdated as the tool mesh-catalog.outdated of the indexer's own runtime: it reads the versions
// state and the heard toolchain and delivered keys of the own state.
func OutdatedTool(ix *Indexer) stdio.Tool {
return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbOutdated, Run: func(args map[string]any) (any, error) {
var in meshcatalog.OutdatedInput
if raw, err := json.Marshal(args); err == nil {
_ = json.Unmarshal(raw, &in)
}
return ix.Outdated(context.Background(), in)
}}
}
// isFloor says whether a kept version is a floor entry (A-490h): it holds no artifact, as no release does.
func isFloor(raw []byte) bool {
var v meshcatalog.VersionEntry
return json.Unmarshal(raw, &v) == nil && len(v.Artifacts) == 0
}
// releasedPins are a module's pins (its catalog entry's packages, which the builder builds it against) from the versions its
// fingerprint was computed with (design 47 H-A-161, H-A-183, H-A-184, H-A-185, G §370): each package is pinned to the
// newest version the registry published for its path (the seats' published, kept in the package's series by HearVersion)
// within the series the module accepts, the one its manifest m names or the default (Manifest.AcceptedSeries), whether
// or not that series holds contents yet; a publish in another series pins nothing and moves nothing, and a module's
// release number is never a pin. A package with none published in that series is kept with no version, so its kind-1
// edge stands (H §746) and its build waits for the publish (H-A-165). A module of the module's own repository is never
// pinned, whatever a build reported: it is built from the tree (design 47 H-A-198, H-A-201, H-A-202). nil for none.
//
// required are the mesh packages the module's go.mod graph requires (its kept packages, H-A-201): each of another
// repository is named whether or not the form's versions name it, with no version while none is published, never left
// out, so the builder waits for its publish instead of the build module refusing (design 47 H-A-204, H-A-165).
func releasedPins(versions sdkstep.PackageVersions, series map[string]SeriesPackage, m catalogue.Manifest, repository string, required []string) sdkstep.PackageVersions {
names := slices.Collect(maps.Keys(versions))
if repository != "" {
names = append(names, fingerprint.OtherPackages(required, repository)...)
}
if len(names) == 0 {
return nil
}
out := sdkstep.PackageVersions{}
for _, p := range names {
if repo, _, ok := trunk.PackageRepository(p); ok && repository != "" && trunk.SameRepository(repo, repository) {
continue
}
out[p] = ""
for _, s := range series {
if s.GoModule != p || s.Newest == nil || s.Series != m.AcceptedSeries(s.Name) {
continue
}
v := strings.TrimPrefix(s.Newest.Version, "v")
if _, ok := versionParts(v); ok && (out[p] == "" || compareVersions(v, strings.TrimPrefix(out[p], "v")) > 0) {
out[p] = "v" + v
}
}
}
return out
}
// pinnedOf is a form's fingerprint and its pins (releasedPins): the pins are part of what a build is made of, so every
// form's fingerprint takes them (design 47 H-A-167, H-A-172; ADR 0573 §4) and a re-pin moves it, the module a mover
// built again against the release; a fingerprint with no pin is the form's as it is. The held fingerprint of a build that
// stands is kept while its pins are the ones it was built with, else its import form takes the new pins.
func pinnedOf(f form, series map[string]SeriesPackage, m catalogue.Manifest, repository string, required []string) (string, sdkstep.PackageVersions) {
pins := releasedPins(f.versions, series, m, repository, required)
if f.stands {
if maps.Equal(f.built, map[string]string(pins)) {
return f.fp, pins
}
return fingerprint.WithPins(f.raw, pins), pins
}
return fingerprint.WithPins(f.fp, pins), pins
}
// moduleOfKey is the module a modules key names; "" for none.
func moduleOfKey(key string) string {
if key == "" {
return ""
}
if m, err := meshcatalogclient.ParseModulesKey(key); err == nil {
return m
}
return key
}
// PinsOf are the pins a build is asked with from a catalog entry's packages: each package with a version; one kept with
// none (no released version, H-A-161) is no pin. nil for none.
func PinsOf(packages map[string]string) sdkstep.PackageVersions {
var out sdkstep.PackageVersions
for p, v := range packages {
if v == "" {
continue
}
if out == nil {
out = sdkstep.PackageVersions{}
}
out[p] = v
}
return out
}
// repin is Repin of a release of module at version, said in the journal; its failure is the caller's error.
func (ix *Indexer) repin(ctx context.Context, module, version string) error {
n, err := ix.Repin(ctx, module, version)
if err != nil {
return fmt.Errorf("the modules pinned to %s are not re-pinned to %s: %w", module, version, err)
}
if n > 0 {
ix.logf("versions: %d modules re-pinned to %s %s", n, module, version)
}
return nil
}