A package-published of mesh-delivery-pipeline 0.1.6 left the catalog's versions empty and the indexer's journal silent: the hearer said nothing when it heard one, waited for the module's other artifacts, took it as another branch's, found the version released already or released it. Each of those ends is now said, naming the artifact, the version, the branch and source, the artifacts still awaited and the fingerprint of the entry compared. Builds: design 47 G §349, §371, A-420
346 lines
14 KiB
Go
346 lines
14 KiB
Go
package index
|
|
|
|
import (
|
|
"cmp"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
|
"git.novox.be/novox/mesh-sdk/go/keys"
|
|
)
|
|
|
|
// The catalog's released versions (design 47 G §349, §372, §374): the indexer keeps, for the seat mesh-catalog, each module's
|
|
// released versions in the state versions, one key per module and version (<module>.<version>) holding each artifact's digest,
|
|
// the source revision built and its edge entries. It hears every artifact's package-published and records a version as
|
|
// released once every artifact the module's entry names is published at it.
|
|
const (
|
|
VersionsState = "versions"
|
|
// VersionsKept is how many versions of a module the state keeps (§349): at each release the module's oldest beyond them
|
|
// go, never its highest of any MAJOR.MINOR.
|
|
VersionsKept = 100
|
|
// PendingPrefix opens the keys of the state heard (Heard) that keep, per module and version not yet released, the
|
|
// artifacts published at it so far, each key part escaped as a name (A-80, §73).
|
|
|
|
PendingPrefix = "pending."
|
|
)
|
|
|
|
// published is what a not yet released version holds: each artifact published at it, by reference.
|
|
type published struct {
|
|
Artifacts map[string]meshcatalog.VersionArtifact `json:"artifacts"`
|
|
Dependencies []meshbuilds.Dependency `json:"dependencies,omitempty"`
|
|
Source int64 `json:"source,omitempty"`
|
|
}
|
|
|
|
// versionParts is MAJOR.MINOR.PATCH, read; false for any other form.
|
|
func versionParts(v string) ([3]int, bool) {
|
|
var out [3]int
|
|
parts := strings.Split(v, ".")
|
|
if len(parts) != 3 {
|
|
return out, false
|
|
}
|
|
for i, p := range parts {
|
|
n, err := strconv.Atoi(p)
|
|
if err != nil || n < 0 {
|
|
return out, false
|
|
}
|
|
out[i] = n
|
|
}
|
|
return out, true
|
|
}
|
|
|
|
// compareVersions orders two versions MAJOR.MINOR.PATCH numerically.
|
|
func compareVersions(a, b string) int {
|
|
x, _ := versionParts(a)
|
|
y, _ := versionParts(b)
|
|
for i := range x {
|
|
if x[i] != y[i] {
|
|
return x[i] - y[i]
|
|
}
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// ModuleVersions reads every released version the state keeps for module, newest first.
|
|
func ModuleVersions(ctx context.Context, st Store, module string) ([]meshcatalog.VersionEntry, error) {
|
|
all, err := st.Keys(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var out []meshcatalog.VersionEntry
|
|
for _, k := range all {
|
|
m, _, ok := meshcatalog.ParseVersionsByModuleVersionKey(k)
|
|
if !ok || m != module {
|
|
continue
|
|
}
|
|
raw, _, found, err := st.Get(ctx, k)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !found {
|
|
continue
|
|
}
|
|
v, err := meshcatalog.DecodeVersionsByModuleVersionValue(raw)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the key %s of the state %s: %w", k, VersionsState, err)
|
|
}
|
|
out = append(out, v)
|
|
}
|
|
slices.SortFunc(out, func(a, b meshcatalog.VersionEntry) int { return compareVersions(b.Version, a.Version) })
|
|
return out, nil
|
|
}
|
|
|
|
// HearPackagePublished takes one artifact's package-published (design 47 G §349): the artifact is kept as published at its
|
|
// version, and once every artifact the module's catalog entry names is published at it the version is released: recorded,
|
|
// and said as version-released unless a higher one is released already (then it is recorded marked late and not said). A
|
|
// build of a watched branch other than the registration's first releases none (§371) and keeps its edges in build-edges (§374, A-83).
|
|
func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.PackagePublished) error {
|
|
if ix.Versions == nil || ix.Heard == nil {
|
|
return errors.New("the state versions is not kept here")
|
|
}
|
|
if _, ok := versionParts(p.Version); !ok {
|
|
return fmt.Errorf("the package-published of %s names the version %q, which is no MAJOR.MINOR.PATCH", p.Artifact, p.Version)
|
|
}
|
|
ix.logf("versions: heard the package-published of %s at %s (digest %s, branch %q, source %d)", p.Artifact, p.Version, p.Digest, p.Branch, p.Source)
|
|
if ix.Registry != nil {
|
|
r, found, err := ix.Registry.Registration(ctx, p.Module)
|
|
switch {
|
|
case err != nil:
|
|
ix.logf("versions: the registration of %s cannot be read, so %s at %s is taken as the first watched branch's: %v", p.Module, p.Artifact, p.Version, err)
|
|
case !found:
|
|
ix.logf("versions: %s has no registration, so %s at %s is taken as the first watched branch's", p.Module, p.Artifact, p.Version)
|
|
case len(r.Branches) > 0 && r.Branches[0] != p.Branch:
|
|
ix.logf("versions: %s at %s is of the branch %q, not the first watched branch %q: it releases no version, its edges are kept in build-edges", p.Artifact, p.Version, p.Branch, r.Branches[0])
|
|
return ix.hearBranchBuild(ctx, p) // releases no version; its edges are kept in build-edges (A-83)
|
|
}
|
|
}
|
|
if _, _, found, err := ix.Versions.Get(ctx, meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)); err != nil {
|
|
return err
|
|
} else if found {
|
|
ix.logf("versions: %s %s is released already, so the package-published of %s is heard again and changes nothing", p.Module, p.Version, p.Artifact)
|
|
return nil
|
|
}
|
|
entry, err := ix.entryOf(ctx, p.Module)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var want []string
|
|
for _, a := range entry.Artifacts {
|
|
want = append(want, a.Ref)
|
|
}
|
|
key := PendingPrefix + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Version)
|
|
var have published
|
|
err = write(ctx, ix.Heard, key, func(raw []byte, found bool) ([]byte, bool, error) {
|
|
have = published{}
|
|
if found {
|
|
if err := json.Unmarshal(raw, &have); err != nil {
|
|
return nil, false, err
|
|
}
|
|
}
|
|
if have.Artifacts == nil {
|
|
have.Artifacts = map[string]meshcatalog.VersionArtifact{}
|
|
}
|
|
have.Artifacts[p.Artifact] = meshcatalog.VersionArtifact{Artifact: p.Artifact, Digest: p.Digest}
|
|
have.Source = p.Source
|
|
for _, d := range p.Dependencies {
|
|
if !slices.ContainsFunc(have.Dependencies, func(x meshbuilds.Dependency) bool { return x.Kind == d.Kind && x.Part == d.Part }) {
|
|
have.Dependencies = append(have.Dependencies, d)
|
|
}
|
|
}
|
|
data, err := json.Marshal(have)
|
|
return data, false, err
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var missing []string
|
|
for _, ref := range want {
|
|
if _, ok := have.Artifacts[ref]; !ok {
|
|
missing = append(missing, ref)
|
|
}
|
|
}
|
|
if len(missing) > 0 {
|
|
ix.logf("versions: %s %s waits for the package-published of %s, which the newest trunk entry of it names (fingerprint %s); heard so far: %d", p.Module, p.Version, strings.Join(missing, ", "), entry.Fp, len(have.Artifacts))
|
|
return nil
|
|
}
|
|
return ix.releaseVersion(ctx, p, key, entry, have)
|
|
}
|
|
|
|
// entryOf is the module's entry in the newest trunk revision.
|
|
func (ix *Indexer) entryOf(ctx context.Context, module string) (meshcatalog.CatalogEntry, error) {
|
|
if ix.Revisions == nil {
|
|
return meshcatalog.CatalogEntry{}, errors.New("the catalog graph is not kept here: no entry names the module's artifacts")
|
|
}
|
|
rev, found, err := newestTrunk(ctx, ix.Revisions)
|
|
if err != nil {
|
|
return meshcatalog.CatalogEntry{}, err
|
|
}
|
|
if !found {
|
|
return meshcatalog.CatalogEntry{}, errors.New("the indexer keeps no trunk revision yet")
|
|
}
|
|
_, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision))
|
|
if err != nil {
|
|
return meshcatalog.CatalogEntry{}, err
|
|
}
|
|
for _, e := range entries {
|
|
if e.Module == module {
|
|
return e, nil
|
|
}
|
|
}
|
|
return meshcatalog.CatalogEntry{}, fmt.Errorf("the trunk holds no module %s", module)
|
|
}
|
|
|
|
// releaseVersion records the version of p whole, says version-released unless it is late, deletes what exceeds the kept versions,
|
|
// and forgets the artifacts kept as published.
|
|
func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePublished, pendingKey string, entry meshcatalog.CatalogEntry, have published) error {
|
|
key := meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)
|
|
kept, err := ModuleVersions(ctx, ix.Versions, p.Module)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
late := len(kept) > 0 && compareVersions(kept[0].Version, p.Version) > 0
|
|
at := ix.now().UTC()
|
|
v := meshcatalog.VersionEntry{V: int64(keys.Version), Module: p.Module, Version: p.Version, Source: have.Source, At: &at, Late: late}
|
|
for _, a := range entry.Artifacts {
|
|
v.Artifacts = append(v.Artifacts, have.Artifacts[a.Ref])
|
|
}
|
|
slices.SortFunc(v.Artifacts, func(a, b meshcatalog.VersionArtifact) int { return strings.Compare(a.Artifact, b.Artifact) })
|
|
v.Edges = releaseEdges(entry, have.Dependencies)
|
|
data, err := json.Marshal(v)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := ix.Versions.Create(ctx, key, data); err != nil && !errors.Is(err, ErrConflict) {
|
|
return err
|
|
}
|
|
if late {
|
|
ix.logf("versions: %s %s is released late: %s is released already, so it is recorded and no version-released is said", p.Module, p.Version, kept[0].Version)
|
|
} else if ix.SayVersionReleased == nil {
|
|
ix.logf("versions: %s %s is released, and no version-released is said here", p.Module, p.Version)
|
|
}
|
|
if !late && ix.SayVersionReleased != nil {
|
|
ev := meshcatalog.VersionReleasedEvent{V: int64(keys.Version), Module: v.Module, Version: v.Version, Artifacts: v.Artifacts, Source: v.Source, At: &at}
|
|
if err := ix.SayVersionReleased(ctx, ev, meshcatalog.EventVersionReleased+"-"+key); err != nil {
|
|
return fmt.Errorf("version-released of %s %s is not said: %w", p.Module, p.Version, err)
|
|
}
|
|
ix.logf("versions: %s %s is released and version-released is said (%d artifacts, source %d)", p.Module, p.Version, len(v.Artifacts), v.Source)
|
|
}
|
|
if err := ix.trimVersions(ctx, p.Module); err != nil {
|
|
ix.logf("versions: %s's oldest versions are not deleted: %v", p.Module, err)
|
|
}
|
|
return ix.forgetPublished(ctx, pendingKey)
|
|
}
|
|
|
|
// trimVersions deletes the module's oldest versions beyond the kept 100, never its highest of any MAJOR.MINOR.
|
|
func (ix *Indexer) trimVersions(ctx context.Context, module string) error {
|
|
kept, err := ModuleVersions(ctx, ix.Versions, module) // newest first
|
|
if err != nil || len(kept) <= VersionsKept {
|
|
return err
|
|
}
|
|
highest := map[string]bool{} // the highest version of each MAJOR.MINOR, as the list is newest first
|
|
seen := map[string]bool{}
|
|
for _, v := range kept {
|
|
p := strings.Split(v.Version, ".")
|
|
if mm := p[0] + "." + p[1]; !seen[mm] {
|
|
seen[mm] = true
|
|
highest[v.Version] = true
|
|
}
|
|
}
|
|
excess := len(kept) - VersionsKept
|
|
for i := len(kept) - 1; i >= 0 && excess > 0; i-- {
|
|
v := kept[i].Version
|
|
if highest[v] {
|
|
continue
|
|
}
|
|
key := meshcatalog.VersionsByModuleVersionKey(module, v)
|
|
if _, rev, found, err := ix.Versions.Get(ctx, key); err == nil && found {
|
|
if err := ix.Versions.Delete(ctx, key, rev); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
excess--
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// forgetPublished removes the artifacts kept as published at a version released.
|
|
func (ix *Indexer) forgetPublished(ctx context.Context, key string) error {
|
|
if _, rev, found, err := ix.Heard.Get(ctx, key); err != nil || !found {
|
|
return err
|
|
} else {
|
|
return ix.Heard.Delete(ctx, key, rev)
|
|
}
|
|
}
|
|
|
|
// Versions answers the verb versions: a module's released versions (design 47 G §349), newest first.
|
|
func Versions(ctx context.Context, st Store, in meshcatalog.VersionsInput) (meshcatalog.VersionsAnswer, error) {
|
|
if in.Module == "" {
|
|
return meshcatalog.VersionsAnswer{}, errors.New("name the module (module)")
|
|
}
|
|
v, err := ModuleVersions(ctx, st, in.Module)
|
|
if v == nil {
|
|
v = []meshcatalog.VersionEntry{}
|
|
}
|
|
return meshcatalog.VersionsAnswer{Module: in.Module, Versions: v}, err
|
|
}
|
|
|
|
// VersionsTool is the verb versions as the tool mesh-catalog.versions of the indexer's own runtime.
|
|
func VersionsTool(st Store) stdio.Tool {
|
|
return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbVersions, Run: func(args map[string]any) (any, error) {
|
|
var in meshcatalog.VersionsInput
|
|
if raw, err := json.Marshal(args); err == nil {
|
|
_ = json.Unmarshal(raw, &in)
|
|
}
|
|
return Versions(context.Background(), st, in)
|
|
}}
|
|
}
|
|
|
|
// notYetDelivered is the version of an edge entry of kind 2, 4 or 6 until its release is first delivered (design 47 G §374).
|
|
const notYetDelivered = "not yet delivered"
|
|
|
|
// releaseEdges are the edge entries of a release (design 47 G §374): kinds 1 and 3 from the dependencies the build
|
|
// resolved, as package-published carried them (stale: the dependent is made from them, §378); kinds 2, 4 and 6 of the entry's
|
|
// catalog graph edges, reading "not yet delivered" until the first delivery writes them. Kind 5 is not written here.
|
|
func releaseEdges(entry meshcatalog.CatalogEntry, deps []meshbuilds.Dependency) []meshcatalog.VersionEdge {
|
|
word := func(kind int64, part string) string {
|
|
for _, e := range entry.Edges {
|
|
if e.Kind == kind && (e.Part == part || e.From == part) {
|
|
return e.Word
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
var out []meshcatalog.VersionEdge
|
|
for _, d := range deps {
|
|
out = append(out, meshcatalog.VersionEdge{Kind: d.Kind, Part: d.Part, Digest: d.Digest, Version: d.Version, Stale: d.Kind == 1 || d.Kind == 3, Word: word(d.Kind, d.Part)})
|
|
}
|
|
for _, e := range entry.Edges {
|
|
if e.Kind != 2 && e.Kind != 4 && e.Kind != 6 {
|
|
continue
|
|
}
|
|
part := cmp.Or(e.Part, e.From)
|
|
if !slices.ContainsFunc(out, func(x meshcatalog.VersionEdge) bool { return x.Kind == e.Kind && x.Part == part && x.Word == e.Word }) {
|
|
out = append(out, meshcatalog.VersionEdge{Kind: e.Kind, Part: part, Word: e.Word, Version: notYetDelivered})
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// OutdatedTool is the verb outdated as the tool mesh-catalog.outdated of the indexer's own runtime: it reads the versions
|
|
// state and the heard toolchain and delivered keys of the own state.
|
|
func OutdatedTool(ix *Indexer) stdio.Tool {
|
|
return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbOutdated, Run: func(args map[string]any) (any, error) {
|
|
var in meshcatalog.OutdatedInput
|
|
if raw, err := json.Marshal(args); err == nil {
|
|
_ = json.Unmarshal(raw, &in)
|
|
}
|
|
return ix.Outdated(context.Background(), in)
|
|
}}
|
|
}
|