package versions import ( "context" "encoding/json" "errors" "fmt" "sort" "strconv" "strings" "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" ) // **The reservation is one key per module and MAJOR.MINOR** (design 47 G §369, §52): the builder keeps in its old state // section `reserved` the version its build reserved and each artifact's digest once built, written by compare-and-set at // the build's start. A newer build of the module takes the next after the higher of the highest released and the one // reserved, so two builds never share a version, and a superseded build's version is never released (a gap in the // patches). The key is deleted when the catalog says `version-released` for it, or when a version of another MAJOR.MINOR // of the module is released (the manifest moved on). This is no list of released versions: numbering reads those only from // the catalog (§348), handed to Reserve by the caller. const ( // ReservedState is the builder's old state section of the reservations. ReservedState = "reserved" // ReservedBucket is its bucket. ReservedBucket = "mesh-delivery-builder_" + ReservedState // ReservedTool is builder_reserved, ReservedDropTool builder_reserved_drop. ReservedTool = "builder_reserved" ReservedDropTool = "builder_reserved_drop" // UnreleasedBound is the publish's bound: past it a built reservation with an artifact unpublished is the finding // version-unreleased (§369). UnreleasedBound = 10 * time.Minute // VersionUnreleased is that finding. VersionUnreleased = "version-unreleased" ) // ReservedVersion is the value of a reservation key. type ReservedVersion struct { V int `json:"v"` Module string `json:"module"` Series string `json:"series"` // MAJOR.MINOR Version string `json:"version"` // MAJOR.MINOR.PATCH // Artifacts are the module's artifacts at the version: the digest once built, empty before. Artifacts map[string]string `json:"artifacts"` // BuiltAt is when the build ended with every digest known; the publish's bound counts from it. BuiltAt time.Time `json:"built-at,omitzero"` // Published are the artifacts a registry seat said published at this version with the built digest. Published map[string]bool `json:"published,omitempty"` At time.Time `json:"at"` // Repository and Commit are what the build was made from, and Tag the git tag its release is written as (a Go // package's, GoTagOf): empty for a module with no Go package (design 47 G §370). Repository string `json:"repository,omitempty"` Commit string `json:"commit,omitempty"` Tag string `json:"tag,omitempty"` // Fingerprint is the fingerprint of the catalog entry the reservation was made from, so a publish again reads that // entry (design 47 G A-76). Fingerprint string `json:"fingerprint,omitempty"` // Unstored are the artifacts a holder refused as not stored, with its words: said once, and not published again on // events (design 47 G §588); builder_build_again builds the key again, which reserves anew. Unstored map[string]string `json:"unstored,omitempty"` // Branch and Source are the watched branch and the source revision of the catalog revision whose entry the build came from // (design 47 G §371, A-57): what `package-published` says of the publication. Branch string `json:"branch,omitempty"` Source int64 `json:"source,omitempty"` // Dependencies are, by artifact reference, what the step's result says each artifact resolved (kind 1) and was built with // (kind 3), kept for `package-published` (design 47 G §466, §673, A-57). Dependencies map[string][]meshbuilds.Dependency `json:"dependencies,omitempty"` } // Source is what a build reserving a version is made from (design 47 G §370): the repository and commit the git tag is // written at, and whether the module has a Go package, whose version is tagged. type Source struct { Repository, Commit string // Fingerprint is the fingerprint of the catalog entry the build is made from (design 47 G A-76). Fingerprint string GoPackage bool // Branch is the watched branch of the catalog revision whose entry the build came from, and Revision the source revision of // that catalog revision (design 47 G §371, A-57), kept on the reservation for the publication. Branch string Revision int64 } // ReservedKey is the key of module's reservation of series (MAJOR.MINOR). func ReservedKey(module, series string) string { return Key(module, series) } // ErrNotFilled is what Reserve answers before the catalog says versions-filled: nothing is reserved until then (§369). var ErrNotFilled = errors.New("versions-filled is not heard: nothing is reserved before the catalog's versions are filled") // ReserveStore is the reservations' bucket: compare-and-set, and deletion. type ReserveStore interface { Store Delete(ctx context.Context, key string) error } // Reserver reserves versions. type Reserver struct { Store ReserveStore // ReleasedOf are the patch versions (MAJOR.MINOR.PATCH) the catalog holds released for module, from its verb versions. ReleasedOf func(ctx context.Context, module string) ([]string, error) // Filled says the catalog said versions-filled. Filled func() bool Now func() time.Time } func (r *Reserver) now() time.Time { if r.Now != nil { return r.Now().UTC() } return time.Now().UTC() } // SplitVersion is a MAJOR.MINOR.PATCH version's series and patch. func SplitVersion(version string) (series string, patch int, err error) { i := strings.LastIndex(version, ".") if i < 0 || strings.Count(version, ".") != 2 { return "", 0, fmt.Errorf("%q is not MAJOR.MINOR.PATCH", version) } patch, err = strconv.Atoi(version[i+1:]) if err != nil || patch < 0 { return "", 0, fmt.Errorf("%q is not MAJOR.MINOR.PATCH", version) } return version[:i], patch, nil } // Reserve reserves the next version of module's series for a build that starts, for artifacts (their names), and answers // it. The next patch is the one after the higher of the highest released and the one reserved, 0 when there is neither. A // conflicting write (another build reserved first) is read again and the next taken. func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts []string, src Source) (ReservedVersion, error) { if r.Filled != nil && !r.Filled() { return ReservedVersion{}, ErrNotFilled } released, err := r.ReleasedOf(ctx, module) if err != nil { return ReservedVersion{}, fmt.Errorf("the released versions of %s cannot be read: %w", module, err) } high := -1 for _, v := range released { if s, p, err := SplitVersion(v); err == nil && s == series && p > high { high = p } } key := ReservedKey(module, series) for { cur, rev, found, err := r.get(ctx, key) if err != nil { return ReservedVersion{}, err } next := high if found { if _, p, err := SplitVersion(cur.Version); err == nil && p > next { next = p } } out := ReservedVersion{V: 1, Module: module, Series: series, Version: fmt.Sprintf("%s.%d", series, next+1), Artifacts: map[string]string{}, At: r.now(), Repository: src.Repository, Commit: src.Commit, Fingerprint: src.Fingerprint, Branch: src.Branch, Source: src.Revision} if src.GoPackage { out.Tag = GoTagOf(out.Version) } for _, a := range artifacts { out.Artifacts[a] = "" } raw, _ := json.Marshal(out) if found { _, err = r.Store.Update(ctx, key, raw, rev) } else { _, err = r.Store.Create(ctx, key, raw) } if errors.Is(err, ErrConflict) { continue } if err != nil { return ReservedVersion{}, fmt.Errorf("%s cannot be reserved: %w", key, err) } return out, nil } } func (r *Reserver) get(ctx context.Context, key string) (ReservedVersion, uint64, bool, error) { raw, rev, found, err := r.Store.Get(ctx, key) if err != nil || !found { return ReservedVersion{}, 0, false, err } var v ReservedVersion if err := json.Unmarshal(raw, &v); err != nil { return ReservedVersion{}, 0, false, fmt.Errorf("%s holds what is not a reservation: %w", key, err) } return v, rev, true, nil } // Built records artifact's digest under the reserved version of module's series; once every artifact has one the build is // ended (BuiltAt). A reservation moved on to another version (a newer build took the next) is left alone and answers false. func (r *Reserver) Built(ctx context.Context, module, version, artifact, digest string, deps []meshbuilds.Dependency) (bool, error) { series, _, err := SplitVersion(version) if err != nil { return false, err } key := ReservedKey(module, series) for { cur, rev, found, err := r.get(ctx, key) if err != nil || !found || cur.Version != version { return false, err } cur.Artifacts[artifact] = digest if len(deps) > 0 { if cur.Dependencies == nil { cur.Dependencies = map[string][]meshbuilds.Dependency{} } cur.Dependencies[artifact] = deps } if cur.BuiltAt.IsZero() && allSet(cur.Artifacts) { cur.BuiltAt = r.now() } raw, _ := json.Marshal(cur) if _, err := r.Store.Update(ctx, key, raw, rev); errors.Is(err, ErrConflict) { continue } else if err != nil { return false, fmt.Errorf("%s cannot be written: %w", key, err) } return true, nil } } func allSet(m map[string]string) bool { for _, d := range m { if d == "" { return false } } return true } // Released takes the catalog's version-released in: the key of that version is deleted, and every key of module's other // series (the manifest moved on). A key holding a newer reserved version of the same series stays. It answers the keys // deleted. func (r *Reserver) Released(ctx context.Context, module, version string) ([]string, error) { series, _, err := SplitVersion(version) if err != nil { return nil, err } keys, err := r.Store.KeysOf(ctx, NameFilter(module)) if err != nil { return nil, fmt.Errorf("the reservations of %s cannot be listed: %w", module, err) } var gone []string for _, k := range keys { cur, _, found, err := r.get(ctx, k) if err != nil { return gone, err } if !found || (cur.Series == series && cur.Version != version) { continue } if err := r.Store.Delete(ctx, k); err != nil { return gone, fmt.Errorf("%s cannot be deleted: %w", k, err) } gone = append(gone, k) } sort.Strings(gone) return gone, nil } // Unreleased are the reservations whose build ended more than UnreleasedBound before now with an artifact unpublished: // the finding version-unreleased, naming the module, the version and the artifacts missing (published says whether an // artifact is). A reservation whose build has not ended is not one. type Unreleased struct { Module, Version string Missing []string } // Overdue answers the reservations past the bound with an artifact not published. func (r *Reserver) Overdue(ctx context.Context, published func(module, version, artifact string) bool) ([]Unreleased, error) { keys, err := r.Store.KeysOf(ctx, AllFilter) if err != nil { return nil, fmt.Errorf("the reservations cannot be listed: %w", err) } sort.Strings(keys) var out []Unreleased for _, k := range keys { cur, _, found, err := r.get(ctx, k) if err != nil { return nil, err } if !found || cur.BuiltAt.IsZero() || r.now().Sub(cur.BuiltAt) < UnreleasedBound { continue } var missing []string for _, a := range cur.Missing() { if published == nil || !published(cur.Module, cur.Version, a) { missing = append(missing, a) } } if len(missing) > 0 { out = append(out, Unreleased{Module: cur.Module, Version: cur.Version, Missing: missing}) } } return out, nil } // All are the reservations, sorted by key: builder_reserved. func (r *Reserver) All(ctx context.Context) ([]ReservedVersion, error) { keys, err := r.Store.KeysOf(ctx, AllFilter) if err != nil { return nil, fmt.Errorf("the reservations cannot be listed: %w", err) } sort.Strings(keys) var out []ReservedVersion for _, k := range keys { cur, _, found, err := r.get(ctx, k) if err != nil { return nil, err } if found { out = append(out, cur) } } return out, nil } // Drop deletes module's reservation of series on a person's word (builder_reserved_drop); the caller writes the hand-act // log. It answers whether there was one. func (r *Reserver) Drop(ctx context.Context, module, series string) (bool, error) { key := ReservedKey(module, series) _, _, found, err := r.get(ctx, key) if err != nil || !found { return false, err } return true, r.Store.Delete(ctx, key) } // Marked is an artifact of a reservation a registry seat's published newly marked: the reservation as it stands then. type Marked struct { Reserved ReservedVersion Artifact string } // PackagePublished is the seat mesh-builds' package-published of one artifact the holder said published at the reservation's // version (design 47 G §52, §334, §673, A-57): the artifact reference, the version and the digest published, the watched branch and // the source revision the build was made from, and the dependencies its step result named. func (m Marked) PackagePublished(digest string, at time.Time) meshbuilds.PackagePublished { at = at.UTC() return meshbuilds.PackagePublished{V: 1, Artifact: m.Artifact, At: &at, Branch: m.Reserved.Branch, Source: m.Reserved.Source, Module: m.Reserved.Module, Version: m.Reserved.Version, Digest: digest, Dependencies: m.Reserved.Dependencies[m.Artifact]} } // Published takes a registry seat's published in: the artifacts of a reservation at version (a leading v is the Go form) // whose built digest it names are marked published. It answers each artifact it newly marked, with the reservation as marked. func (r *Reserver) Published(ctx context.Context, version, digest string) ([]Marked, error) { version = strings.TrimPrefix(version, "v") if version == "" || digest == "" { return nil, nil } keys, err := r.Store.KeysOf(ctx, AllFilter) if err != nil { return nil, fmt.Errorf("the reservations cannot be listed: %w", err) } sort.Strings(keys) var marked []Marked for _, k := range keys { again: cur, rev, found, err := r.get(ctx, k) if err != nil { return marked, err } if !found || cur.Version != version { continue } changed := false var arts []string for a, d := range cur.Artifacts { if d == digest && !cur.Published[a] { if cur.Published == nil { cur.Published = map[string]bool{} } cur.Published[a], changed = true, true arts = append(arts, a) } } sort.Strings(arts) if !changed { continue } raw, _ := json.Marshal(cur) if _, err := r.Store.Update(ctx, k, raw, rev); errors.Is(err, ErrConflict) { goto again } else if err != nil { return marked, fmt.Errorf("%s cannot be written: %w", k, err) } for _, a := range arts { marked = append(marked, Marked{Reserved: cur, Artifact: a}) } } return marked, nil } // Of is module's reservation of series, and whether there is one; with an empty series the module's highest. func (r *Reserver) Of(ctx context.Context, module, version string) (ReservedVersion, bool, error) { all, err := r.All(ctx) if err != nil { return ReservedVersion{}, false, err } var out ReservedVersion found := false for _, v := range all { if v.Module != module || (version != "" && v.Version != version) { continue } if !found { out, found = v, true continue } _, p, _ := SplitVersion(v.Version) _, q, _ := SplitVersion(out.Version) if v.Series > out.Series || (v.Series == out.Series && p > q) { out = v } } return out, found, nil } const reservedBoundPrefix = "reserved." // ReservedBoundKey is the bound of a reservation's publish in the state bounds (key-ttl): `reserved.`, armed // when its build ended with every digest known. Its end while the reservation stands is the finding version-unreleased. func ReservedBoundKey(module, series string) string { return ReservedBoundFor(ReservedKey(module, series)) } // ReservedBoundFor is the bound of the reservation under key. func ReservedBoundFor(key string) string { return reservedBoundPrefix + key } // ReservedBoundOf is the reserved key a reservation's bound names, and whether key is one. func ReservedBoundOf(key string) (string, bool) { return strings.CutPrefix(key, reservedBoundPrefix) } // readOnlyReserve is the reservations read by the tools bundle, which writes nothing. type readOnlyReserve struct{ readStore } func (readOnlyReserve) Delete(context.Context, string) error { return errReadOnly } // AnswerReserved is builder_reserved: every reservation, or one module's. func AnswerReserved(r Reader, module string) (map[string]any, error) { rs := &Reserver{Store: readOnlyReserve{readStore{r}}} all, err := rs.All(context.Background()) if err != nil { return nil, err } out := []ReservedVersion{} for _, v := range all { if module == "" || v.Module == module { out = append(out, v) } } return map[string]any{"reserved": out}, nil } // AnswerReservedVerb is the seat mesh-builds' verb reserved: module's reserved version (the given one, else the highest) // with each artifact's digest, an artifact not built yet with none. func AnswerReservedVerb(r Reader, module, version string) (meshbuilds.ReservedAnswer, error) { module = strings.TrimSpace(module) if module == "" { return meshbuilds.ReservedAnswer{}, errors.New("module is required") } rs := &Reserver{Store: readOnlyReserve{readStore{r}}} v, found, err := rs.Of(context.Background(), module, strings.TrimSpace(version)) if err != nil { return meshbuilds.ReservedAnswer{}, err } if !found { return meshbuilds.ReservedAnswer{}, fmt.Errorf("%s has no reserved version%s", module, map[bool]string{true: " " + version, false: ""}[version != ""]) } out := meshbuilds.ReservedAnswer{Module: v.Module, Version: v.Version, Artifacts: []meshbuilds.ReservedAnswerArtifactsItem{}} names := make([]string, 0, len(v.Artifacts)) for a := range v.Artifacts { names = append(names, a) } sort.Strings(names) for _, a := range names { out.Artifacts = append(out.Artifacts, meshbuilds.ReservedAnswerArtifactsItem{Artifact: a, Digest: v.Artifacts[a]}) } return out, nil } // CatalogVersions answers a module's released versions, the seat mesh-catalog's verb versions (design 47 G §582). type CatalogVersions func(ctx context.Context, module string) ([]meshcatalog.VersionEntry, error) // AnswerReservedByName is the verb reserved given a package's name (design 47 G §572, §582, A-70): the module is found from the // name in the catalog half's view (carriers); that module's reserved version answers as AnswerReservedVerb does. A version // given with no reservation of it (a reservation goes once its version is released) is asked of the catalog, and answers // released with the artifacts and digests the release recorded. A name no entry carries, more than one entry carries, or a // version neither reserved nor released is answered error, naming which, set alone. func AnswerReservedByName(ctx context.Context, r Reader, carriers func(context.Context, string) ([]Built, error), released CatalogVersions, name, version string) (meshbuilds.ReservedAnswer, error) { name, version = strings.TrimSpace(name), strings.TrimSpace(version) got, err := carriers(ctx, name) if err != nil { return meshbuilds.ReservedAnswer{}, fmt.Errorf("the entries of %s cannot be read: %w", name, err) } switch { case len(got) == 0: return meshbuilds.ReservedAnswer{Error: "no entry carries the name " + name}, nil case len(got) > 1: return meshbuilds.ReservedAnswer{Error: MoreThanOne(name, got)}, nil } module := got[0].Module rs := &Reserver{Store: readOnlyReserve{readStore{r}}} if _, found, err := rs.Of(ctx, module, version); err != nil { return meshbuilds.ReservedAnswer{}, err } else if found { return AnswerReservedVerb(r, module, version) } if version != "" { entries, err := released(ctx, module) if err != nil { return meshbuilds.ReservedAnswer{}, fmt.Errorf("the released versions of %s cannot be read: %w", module, err) } for _, e := range entries { if e.Version != version { continue } out := meshbuilds.ReservedAnswer{Module: module, Version: version, Released: true, Artifacts: []meshbuilds.ReservedAnswerArtifactsItem{}} for _, a := range e.Artifacts { out.Artifacts = append(out.Artifacts, meshbuilds.ReservedAnswerArtifactsItem{Artifact: a.Artifact, Digest: a.Digest}) } return out, nil } return meshbuilds.ReservedAnswer{Error: fmt.Sprintf("%s (%s) has no version %s reserved or released", name, module, version)}, nil } return meshbuilds.ReservedAnswer{Error: fmt.Sprintf("%s (%s) has no reserved version", name, module)}, nil } // The descriptions of the reservation's tools. const ( ReservedDescription = "The versions the builder reserved (its state reserved, design 47 G section 369): one per module and MAJOR.MINOR, " + "the version its build reserved at its start, each artifact's digest once built, when the build ended and which artifacts a registry " + "seat said published. With module, that module's only. A key goes when the catalog says version-released for it. Read-only." ReservedDropDescription = "Drop a module's reservation of one MAJOR.MINOR by hand (design 47 G section 369), when its version will never be " + "released; the next build takes the next after the highest released. A hand act: it is recorded in the hand-act log." // ReservedVerb is the seat mesh-builds' verb served as a tool of the same name. ReservedVerb = meshbuilds.VerbReserved ) // Missing are the artifacts of v no registry seat said published, sorted. func (v ReservedVersion) Missing() []string { var out []string for a := range v.Artifacts { if !v.Published[a] { out = append(out, a) } } sort.Strings(out) return out } // At is the reservation under key, and whether there is one. func (r *Reserver) At(ctx context.Context, key string) (ReservedVersion, bool, error) { v, _, found, err := r.get(ctx, key) return v, found, err }