Files
mesh-delivery/modules/mesh-delivery-indexer/internal/index/versions.go
T
jschoubben d0436bb38c mesh-delivery-indexer: pins are in every form's fingerprint
The import form's fingerprint left the pins out, so an importer whose
trunk build reported its imports never moved on a re-pin and was never
built again against the release. Every form now takes its sorted pins;
a build that stands keeps its fingerprint while its pins are the ones
it was built with, and a fingerprint with no pin stays as it was.

Builds: design 47 H H-A-172, H-A-167
2026-11-20 17:46:53 +01:00

472 lines
19 KiB
Go

package index
import (
"cmp"
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"strconv"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/keys"
"mesh-delivery-indexer/internal/fingerprint"
)
// The catalog's released versions (design 47 G §349, §372, §374): the indexer keeps, for the seat mesh-catalog, each module's
// released versions in the state versions, one key per module and version (<module>.<version>) holding each artifact's digest,
// the source revision built and its edge entries. It hears every artifact's package-published and records a version as
// released once every artifact the module's entry names is published at it.
const (
VersionsState = "versions"
// VersionsKept is how many versions of a module the state keeps (§349): at each release the module's oldest beyond them
// go, never its highest of any MAJOR.MINOR.
VersionsKept = 100
// PendingPrefix opens the keys of the state heard (Heard) that keep, per module and version not yet released, the
// artifacts published at it so far, each key part escaped as a name (A-80, §73).
PendingPrefix = "pending."
)
// published is what a not yet released version holds: each artifact published at it, by reference.
type published struct {
Artifacts map[string]meshcatalog.VersionArtifact `json:"artifacts"`
Dependencies []meshbuilds.Dependency `json:"dependencies,omitempty"`
Source int64 `json:"source,omitempty"`
}
// versionParts is MAJOR.MINOR.PATCH, read; false for any other form.
func versionParts(v string) ([3]int, bool) {
var out [3]int
parts := strings.Split(v, ".")
if len(parts) != 3 {
return out, false
}
for i, p := range parts {
n, err := strconv.Atoi(p)
if err != nil || n < 0 {
return out, false
}
out[i] = n
}
return out, true
}
// compareVersions orders two versions MAJOR.MINOR.PATCH numerically.
func compareVersions(a, b string) int {
x, _ := versionParts(a)
y, _ := versionParts(b)
for i := range x {
if x[i] != y[i] {
return x[i] - y[i]
}
}
return 0
}
// ModuleVersions reads every released version the state keeps for module, newest first.
func ModuleVersions(ctx context.Context, st Store, module string) ([]meshcatalog.VersionEntry, error) {
all, err := st.Keys(ctx)
if err != nil {
return nil, err
}
var out []meshcatalog.VersionEntry
for _, k := range all {
m, _, ok := meshcatalog.ParseVersionsByModuleVersionKey(k)
if !ok || m != module {
continue
}
raw, _, found, err := st.Get(ctx, k)
if err != nil {
return nil, err
}
if !found {
continue
}
v, err := meshcatalog.DecodeVersionsByModuleVersionValue(raw)
if err != nil {
return nil, fmt.Errorf("the key %s of the state %s: %w", k, VersionsState, err)
}
out = append(out, v)
}
slices.SortFunc(out, func(a, b meshcatalog.VersionEntry) int { return compareVersions(b.Version, a.Version) })
return out, nil
}
// HearPackagePublished takes one artifact's package-published (design 47 G §349): the artifact is kept as published at its
// version, and once every artifact the module's catalog entry names is published at it the version is released: recorded,
// and said as version-released unless a higher one is released already (then it is recorded marked late and not said). A
// build of a watched branch other than the registration's first releases none (§371) and keeps its edges in build-edges (§374, A-83).
func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.PackagePublished) error {
if ix.Versions == nil || ix.Heard == nil {
return errors.New("the state versions is not kept here")
}
if _, ok := versionParts(p.Version); !ok {
return fmt.Errorf("the package-published of %s names the version %q, which is no MAJOR.MINOR.PATCH", p.Artifact, p.Version)
}
ix.logf("versions: heard the package-published of %s at %s (digest %s, branch %q, source %d)", p.Artifact, p.Version, p.Digest, p.Branch, p.Source)
if ix.Registry != nil {
r, found, err := ix.Registry.Registration(ctx, p.Module)
switch {
case err != nil:
ix.logf("versions: the registration of %s cannot be read, so %s at %s is taken as the first watched branch's: %v", p.Module, p.Artifact, p.Version, err)
case !found:
ix.logf("versions: %s has no registration, so %s at %s is taken as the first watched branch's", p.Module, p.Artifact, p.Version)
case len(r.Branches) > 0 && r.Branches[0] != p.Branch:
ix.logf("versions: %s at %s is of the branch %q, not the first watched branch %q: it releases no version, its edges are kept in build-edges", p.Artifact, p.Version, p.Branch, r.Branches[0])
return ix.hearBranchBuild(ctx, p) // releases no version; its edges are kept in build-edges (A-83)
}
}
if raw, _, found, err := ix.Versions.Get(ctx, meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)); err != nil {
return err
} else if found && !isFloor(raw) {
// A package-published delivered again after a re-pin failed (H-A-167): the re-pin is made again, nothing else.
ix.logf("versions: %s %s is released already, so the package-published of %s is heard again and only re-pins", p.Module, p.Version, p.Artifact)
return ix.repin(ctx, p.Module, p.Version)
}
entry, err := ix.entryOf(ctx, p.Module)
if err != nil {
return err
}
var want []string
for _, a := range entry.Artifacts {
want = append(want, a.Ref)
}
key := PendingPrefix + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Version)
var have published
err = write(ctx, ix.Heard, key, func(raw []byte, found bool) ([]byte, bool, error) {
have = published{}
if found {
if err := json.Unmarshal(raw, &have); err != nil {
return nil, false, err
}
}
if have.Artifacts == nil {
have.Artifacts = map[string]meshcatalog.VersionArtifact{}
}
have.Artifacts[p.Artifact] = meshcatalog.VersionArtifact{Artifact: p.Artifact, Digest: p.Digest}
have.Source = p.Source
for _, d := range p.Dependencies {
if !slices.ContainsFunc(have.Dependencies, func(x meshbuilds.Dependency) bool { return x.Kind == d.Kind && x.Part == d.Part }) {
have.Dependencies = append(have.Dependencies, d)
}
}
data, err := json.Marshal(have)
return data, false, err
})
if err != nil {
return err
}
var missing []string
for _, ref := range want {
if _, ok := have.Artifacts[ref]; !ok {
missing = append(missing, ref)
}
}
if len(missing) > 0 {
ix.logf("versions: %s %s waits for the package-published of %s, which the newest trunk entry of it names (fingerprint %s); heard so far: %d", p.Module, p.Version, strings.Join(missing, ", "), entry.Fp, len(have.Artifacts))
return nil
}
return ix.releaseVersion(ctx, p, key, entry, have)
}
// entryOf is the module's entry in the newest trunk revision.
func (ix *Indexer) entryOf(ctx context.Context, module string) (meshcatalog.CatalogEntry, error) {
if ix.Revisions == nil {
return meshcatalog.CatalogEntry{}, errors.New("the catalog graph is not kept here: no entry names the module's artifacts")
}
_, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
if err != nil {
return meshcatalog.CatalogEntry{}, err
}
if !found {
return meshcatalog.CatalogEntry{}, errors.New("the indexer keeps no trunk revision yet")
}
for _, e := range entries {
if e.Module == module {
return e, nil
}
}
return meshcatalog.CatalogEntry{}, fmt.Errorf("the trunk holds no module %s", module)
}
// releaseVersion records the version of p whole, says version-released unless it is late, deletes what exceeds the kept versions,
// and forgets the artifacts kept as published.
func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePublished, pendingKey string, entry meshcatalog.CatalogEntry, have published) error {
key := meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)
kept, err := ModuleVersions(ctx, ix.Versions, p.Module)
if err != nil {
return err
}
late := len(kept) > 0 && compareVersions(kept[0].Version, p.Version) > 0
at := ix.now().UTC()
v := meshcatalog.VersionEntry{V: int64(keys.Version), Module: p.Module, Version: p.Version, Source: have.Source, At: &at, Late: late}
for _, a := range entry.Artifacts {
v.Artifacts = append(v.Artifacts, have.Artifacts[a.Ref])
}
slices.SortFunc(v.Artifacts, func(a, b meshcatalog.VersionArtifact) int { return strings.Compare(a.Artifact, b.Artifact) })
v.Edges = releaseEdges(entry, have.Dependencies)
data, err := json.Marshal(v)
if err != nil {
return err
}
if err := ix.Versions.Create(ctx, key, data); errors.Is(err, ErrConflict) {
// A floor entry (A-490h) holds no artifacts: this publish is that version's release, so it takes the entry's place.
if raw, rev, found, gerr := ix.Versions.Get(ctx, key); gerr == nil && found && isFloor(raw) {
if err := ix.Versions.Update(ctx, key, data, rev); err != nil && !errors.Is(err, ErrConflict) {
return err
}
}
} else if err != nil {
return err
}
if late {
ix.logf("versions: %s %s is released late: %s is released already, so it is recorded and no version-released is said", p.Module, p.Version, kept[0].Version)
} else if ix.SayVersionReleased == nil {
ix.logf("versions: %s %s is released, and no version-released is said here", p.Module, p.Version)
}
if !late && ix.SayVersionReleased != nil {
ev := meshcatalog.VersionReleasedEvent{V: int64(keys.Version), Module: v.Module, Version: v.Version, Artifacts: v.Artifacts, Source: v.Source, At: &at}
if err := ix.SayVersionReleased(ctx, ev, meshcatalog.EventVersionReleased+"-"+key); err != nil {
return fmt.Errorf("version-released of %s %s is not said: %w", p.Module, p.Version, err)
}
ix.logf("versions: %s %s is released and version-released is said (%d artifacts, source %d)", p.Module, p.Version, len(v.Artifacts), v.Source)
}
if !late {
// The release re-pins every module whose packages hold one of it (design 47 H-A-165); one that fails is not
// acknowledged, so the package-published is delivered again (H-A-167).
if err := ix.repin(ctx, p.Module, p.Version); err != nil {
return err
}
}
if err := ix.trimVersions(ctx, p.Module); err != nil {
ix.logf("versions: %s's oldest versions are not deleted: %v", p.Module, err)
}
return ix.forgetPublished(ctx, pendingKey)
}
// trimVersions deletes the module's oldest versions beyond the kept 100, never its highest of any MAJOR.MINOR.
func (ix *Indexer) trimVersions(ctx context.Context, module string) error {
kept, err := ModuleVersions(ctx, ix.Versions, module) // newest first
if err != nil || len(kept) <= VersionsKept {
return err
}
highest := map[string]bool{} // the highest version of each MAJOR.MINOR, as the list is newest first
seen := map[string]bool{}
for _, v := range kept {
p := strings.Split(v.Version, ".")
if mm := p[0] + "." + p[1]; !seen[mm] {
seen[mm] = true
highest[v.Version] = true
}
}
excess := len(kept) - VersionsKept
for i := len(kept) - 1; i >= 0 && excess > 0; i-- {
v := kept[i].Version
if highest[v] {
continue
}
key := meshcatalog.VersionsByModuleVersionKey(module, v)
if _, rev, found, err := ix.Versions.Get(ctx, key); err == nil && found {
if err := ix.Versions.Delete(ctx, key, rev); err != nil {
return err
}
}
excess--
}
return nil
}
// forgetPublished removes the artifacts kept as published at a version released.
func (ix *Indexer) forgetPublished(ctx context.Context, key string) error {
if _, rev, found, err := ix.Heard.Get(ctx, key); err != nil || !found {
return err
} else {
return ix.Heard.Delete(ctx, key, rev)
}
}
// Versions answers the verb versions: a module's released versions (design 47 G §349), newest first.
func Versions(ctx context.Context, st Store, in meshcatalog.VersionsInput) (meshcatalog.VersionsAnswer, error) {
if in.Module == "" {
return meshcatalog.VersionsAnswer{}, errors.New("name the module (module)")
}
v, err := ModuleVersions(ctx, st, in.Module)
if v == nil {
v = []meshcatalog.VersionEntry{}
}
return meshcatalog.VersionsAnswer{Module: in.Module, Versions: v}, err
}
// VersionsTool is the verb versions as the tool mesh-catalog.versions of the indexer's own runtime.
func VersionsTool(st Store) stdio.Tool {
return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbVersions, Run: func(args map[string]any) (any, error) {
var in meshcatalog.VersionsInput
if raw, err := json.Marshal(args); err == nil {
_ = json.Unmarshal(raw, &in)
}
return Versions(context.Background(), st, in)
}}
}
// notYetDelivered is the version of an edge entry of kind 2, 4 or 6 until its release is first delivered (design 47 G §374).
const notYetDelivered = "not yet delivered"
// releaseEdges are the edge entries of a release (design 47 G §374): kinds 1 and 3 from the dependencies the build
// resolved, as package-published carried them (stale: the dependent is made from them, §378); kinds 2, 4 and 6 of the entry's
// catalog graph edges, reading "not yet delivered" until the first delivery writes them. Kind 5 is not written here.
func releaseEdges(entry meshcatalog.CatalogEntry, deps []meshbuilds.Dependency) []meshcatalog.VersionEdge {
word := func(kind int64, part string) string {
for _, e := range entry.Edges {
if e.Kind == kind && (e.Part == part || e.From == part) {
return e.Word
}
}
return ""
}
var out []meshcatalog.VersionEdge
for _, d := range deps {
out = append(out, meshcatalog.VersionEdge{Kind: d.Kind, Part: d.Part, Digest: d.Digest, Version: d.Version, Stale: d.Kind == 1 || d.Kind == 3, Word: word(d.Kind, d.Part)})
}
for _, e := range entry.Edges {
if e.Kind != 2 && e.Kind != 4 && e.Kind != 6 {
continue
}
part := cmp.Or(e.Part, e.From)
if !slices.ContainsFunc(out, func(x meshcatalog.VersionEdge) bool { return x.Kind == e.Kind && x.Part == part && x.Word == e.Word }) {
out = append(out, meshcatalog.VersionEdge{Kind: e.Kind, Part: part, Word: e.Word, Version: notYetDelivered})
}
}
return out
}
// OutdatedTool is the verb outdated as the tool mesh-catalog.outdated of the indexer's own runtime: it reads the versions
// state and the heard toolchain and delivered keys of the own state.
func OutdatedTool(ix *Indexer) stdio.Tool {
return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbOutdated, Run: func(args map[string]any) (any, error) {
var in meshcatalog.OutdatedInput
if raw, err := json.Marshal(args); err == nil {
_ = json.Unmarshal(raw, &in)
}
return ix.Outdated(context.Background(), in)
}}
}
// isFloor says whether a kept version is a floor entry (A-490h): it holds no artifact, as no release does.
func isFloor(raw []byte) bool {
var v meshcatalog.VersionEntry
return json.Unmarshal(raw, &v) == nil && len(v.Artifacts) == 0
}
// releasedOf is the newest released version of each module the state versions keeps (§349; design 47 H-A-161), by module;
// nil when the indexer keeps no such state or it cannot be listed, said.
func (ix *Indexer) releasedOf(ctx context.Context) map[string]string {
if ix.Versions == nil {
return nil
}
all, err := ix.Versions.Keys(ctx)
if err != nil {
ix.logf("versions: the state cannot be listed, so no package is pinned by this pass: %v", err)
return nil
}
out := map[string]string{}
for _, k := range all {
m, v, ok := meshcatalog.ParseVersionsByModuleVersionKey(k)
if _, parts := versionParts(v); !ok || !parts {
continue
}
if cur, has := out[m]; !has || compareVersions(v, cur) > 0 {
out[m] = v
}
}
return out
}
// releasedPins are a module's pins (its catalog entry's packages, which the builder builds it against) from the versions its
// fingerprint was computed with (design 47 H-A-161, G §370): a package the held module holding it has released is pinned
// to that module's newest released version, as Go names it (v<version>); a version already in that form (an accepted
// series' newest, DLV-I6) stands; any other, a commit, is no pin: its package is kept with no version, so its kind-1 edge
// stands (H §746) and no build is made against a commit. nil for none.
func releasedPins(versions sdkstep.PackageVersions, held map[string]meshcatalog.ModulesByModuleValue, released map[string]string) sdkstep.PackageVersions {
if len(versions) == 0 {
return nil
}
out := sdkstep.PackageVersions{}
for p, v := range versions {
rel := released[moduleOfKey(fingerprint.HolderOf(p, held))]
series := sdkstep.PackageSemver.MatchString(v)
_, plain := versionParts(strings.TrimPrefix(v, "v"))
switch {
case rel != "" && (!series || !plain || compareVersions(rel, strings.TrimPrefix(v, "v")) > 0):
out[p] = "v" + rel // never an older series' version over a newer release (H-A-167)
case series:
out[p] = v
default:
out[p] = ""
}
}
return out
}
// pinnedOf is a form's fingerprint and its pins (releasedPins): the pins are part of what a build is made of, so every
// form's fingerprint takes them (design 47 H-A-167, H-A-172; ADR 0573 §4) and a re-pin moves it, the module a mover
// built again against the release; a fingerprint with no pin is the form's as it is. The held fingerprint of a build that
// stands is kept while its pins are the ones it was built with, else its import form takes the new pins.
func pinnedOf(f form, held map[string]meshcatalog.ModulesByModuleValue, released map[string]string) (string, sdkstep.PackageVersions) {
pins := releasedPins(f.versions, held, released)
if f.stands {
if maps.Equal(f.built, map[string]string(pins)) {
return f.fp, pins
}
return fingerprint.WithPins(f.raw, pins), pins
}
return fingerprint.WithPins(f.fp, pins), pins
}
// moduleOfKey is the module a modules key names; "" for none.
func moduleOfKey(key string) string {
if key == "" {
return ""
}
if m, err := meshcatalogclient.ParseModulesKey(key); err == nil {
return m
}
return key
}
// PinsOf are the pins a build is asked with from a catalog entry's packages: each package with a version; one kept with
// none (no released version, H-A-161) is no pin. nil for none.
func PinsOf(packages map[string]string) sdkstep.PackageVersions {
var out sdkstep.PackageVersions
for p, v := range packages {
if v == "" {
continue
}
if out == nil {
out = sdkstep.PackageVersions{}
}
out[p] = v
}
return out
}
// repin is Repin of a release of module at version, said in the journal; its failure is the caller's error.
func (ix *Indexer) repin(ctx context.Context, module, version string) error {
n, err := ix.Repin(ctx, module)
if err != nil {
return fmt.Errorf("the modules pinned to %s are not re-pinned to %s: %w", module, version, err)
}
if n > 0 {
ix.logf("versions: %d modules re-pinned to %s %s", n, module, version)
}
return nil
}