Files
mesh-delivery/modules/mesh-delivery-builder/internal/versions/reserve.go
T
jschoubben c8cc249a90 B7: a tag names its commit; the builder reads a module's tags before it reserves
A number whose tag stood at another commit was reserved again, and a Go package outside
mesh-sdk's go directory was tagged go/v<version>, which the Go toolchain never reads. The
builder now reads the Go package's tags once per catalog revision, skips a number tagged at
another commit with the finding version-tagged-elsewhere (cleared at the next release), and
tags <directory>/v<version>, the directory read from the package's module path.

Builds: design 47 H2 §796
2026-11-22 01:54:49 +01:00

1154 lines
47 KiB
Go

package versions
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"sort"
"strconv"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
)
// **The reservation is one key per module and MAJOR.MINOR** (design 47 G §369, §52): the builder keeps in its old state
// section `reserved` the version its build reserved and each artifact's digest once built, written by compare-and-set at
// the build's start. A newer build of the module takes the next after the higher of the highest released and the one
// reserved, so two builds never share a version, and a superseded build's version is never released (a gap in the
// patches). The key is deleted when the catalog says `version-released` for it, or when a version of another MAJOR.MINOR
// of the module is released (the manifest moved on). This is no list of released versions: numbering reads those only from
// the catalog (§348), handed to Reserve by the caller.
const (
// ReservedState is the builder's old state section of the reservations.
ReservedState = "reserved"
// ReservedTool is builder_reserved, ReservedDropTool builder_reserved_drop.
ReservedTool = "builder_reserved"
ReservedDropTool = "builder_reserved_drop"
// ReserveDoneTool is builder_reserve_done (design 47 G A-455).
ReserveDoneTool = "builder_reserve_done"
// UnreleasedBound is the publish's bound: past it a built reservation with an artifact unpublished is the finding
// version-unreleased (§369).
UnreleasedBound = 10 * time.Minute
// VersionUnreleased is that finding.
VersionUnreleased = "version-unreleased"
// VersionHeldElsewhere is the finding of a number a registry seat already holds, skipped at reserving or moved from (design 47
// H H-A-275), scope module.
VersionHeldElsewhere = "version-held-elsewhere"
// VersionTaggedElsewhere is the finding of a number whose git tag stands at another commit, skipped at reserving (design 47
// H2 §796), scope module; cleared at the module's next version-released.
VersionTaggedElsewhere = "version-tagged-elsewhere"
)
// ReservedVersion is the value of a reservation key.
type ReservedVersion struct {
V int `json:"v"`
Module string `json:"module"`
Series string `json:"series"` // MAJOR.MINOR
Version string `json:"version"` // MAJOR.MINOR.PATCH
// Artifacts are the module's artifacts at the version: the digest once built, empty before.
Artifacts map[string]string `json:"artifacts"`
// BuiltAt is when the build ended with every digest known; the publish's bound counts from it.
BuiltAt time.Time `json:"built-at,omitzero"`
// Published are the artifacts a registry seat said published at this version with the built digest.
Published map[string]bool `json:"published,omitempty"`
// PublishedAt is when the builder heard or confirmed each artifact's publish (H-A-331); absent for a mark written before it
// was kept.
PublishedAt map[string]time.Time `json:"published-at,omitempty"`
At time.Time `json:"at"`
// Repository and Commit are what the build was made from, and Tag the git tag its release is written as (a Go
// package's, GoTagOf): empty for a module with no Go package (design 47 G §370), or whose Go package's directory is not known.
Repository string `json:"repository,omitempty"`
Commit string `json:"commit,omitempty"`
Tag string `json:"tag,omitempty"`
// Fingerprint is the fingerprint of the catalog entry the reservation was made from, so a publish again reads that
// entry (design 47 G A-76).
Fingerprint string `json:"fingerprint,omitempty"`
// Unstored are the artifacts a holder refused as not stored, with its words: said once, and not published again on
// events (design 47 G §588); builder_build_again builds the key again, which reserves anew.
Unstored map[string]string `json:"unstored,omitempty"`
// Branch and Source are the watched branch and the source revision of the catalog revision whose entry the build came from
// (design 47 G §371, A-57): what `package-published` says of the publication.
Branch string `json:"branch,omitempty"`
Source int64 `json:"source,omitempty"`
// Revision is the catalog revision number the build's step ask named, the indexer's count (design 47 H H-A-218): what
// `package-published` names beside the source, so the runner finds the publishing pipeline; 0 when no step ask named one.
Revision int64 `json:"revision,omitempty"`
// Dependencies are, by artifact reference, what the step's result says each artifact resolved (kind 1) and was built with
// (kind 3), kept for `package-published` (design 47 G §466, §673, A-57).
Dependencies map[string][]meshbuilds.Dependency `json:"dependencies,omitempty"`
// Kept are the registry seats' published heard while the build had not yet recorded every digest and whose digest marked
// no artifact then: kept as heard, they mark their artifact when the build records its digests (design 47 G A-415).
Kept []KeptPublished `json:"kept-published,omitempty"`
// Withdrawn are the versions of the series a take-over by another fingerprint withdrew, each with the fingerprint whose
// build it was (design 47 H H-A-225): a late publish of one is refused as withdrawn. Kept while the series' key stands.
Withdrawn map[string]string `json:"withdrawn,omitempty"`
}
// markPublished marks artifact a published at at, the moment the builder heard or confirmed its publish (H-A-331).
func (v *ReservedVersion) markPublished(a string, at time.Time) {
if v.Published == nil {
v.Published = map[string]bool{}
}
if v.PublishedAt == nil {
v.PublishedAt = map[string]time.Time{}
}
v.Published[a], v.PublishedAt[a] = true, at.UTC()
}
// KeptPublished is a registry seat's published kept on a reservation, as heard (design 47 G A-415).
type KeptPublished struct {
Name string `json:"name"`
Version string `json:"version"`
Digest string `json:"digest"`
}
// Source is what a build reserving a version is made from (design 47 G §370): the repository and commit the git tag is
// written at, and whether the module has a Go package, whose version is tagged.
type Source struct {
Repository, Commit string
// Fingerprint is the fingerprint of the catalog entry the build is made from (design 47 G A-76).
Fingerprint string
GoPackage bool
// GoArtifact is the artifact that is the module's Go package, whose module path names the directory its tag is written in
// (design 47 H2 §796).
GoArtifact string
// Branch is the watched branch of the catalog revision whose entry the build came from, and Revision the source revision of
// that catalog revision (design 47 G §371, A-57), kept on the reservation for the publication.
Branch string
Revision int64
// CatalogRevision is the catalog revision number the build's step ask named (H-A-218), 0 when none did.
CatalogRevision int64
// Rebuild marks a build again of a done key whose version is not released: the standing reservation of Fingerprint is replaced
// and is no floor, so the number is the next above the catalog's highest alone (design 47 G A-490b).
Rebuild bool
}
// ReservedKey is the key of module's reservation of series (MAJOR.MINOR).
func ReservedKey(module, series string) string { return Key(module, series) }
// ErrNotFilled is what Reserve answers before the catalog says versions-filled: nothing is reserved until then (§369).
var ErrNotFilled = errors.New("versions-filled is not heard: nothing is reserved before the catalog's versions are filled")
// ReserveStore is the reservations' bucket: compare-and-set, and deletion.
type ReserveStore interface {
Store
Delete(ctx context.Context, key string) error
}
// Reserver reserves versions.
type Reserver struct {
Store ReserveStore
// ReleasedOf are the patch versions (MAJOR.MINOR.PATCH) the catalog holds released for module, from its verb versions.
ReleasedOf func(ctx context.Context, module string) ([]string, error)
// Filled says the catalog said versions-filled.
Filled func() bool
// Took is told when a build took over the series' reservation of an older build never released (design 47 G A-455): the
// older reservation, which no want follows any more, is gone from the key, replaced by the newer's. Nil is told none.
Took func(ctx context.Context, older, newer ReservedVersion)
// Reserved is told each reservation written, a number newly reserved (never one kept as it stood), so the holder says
// version-reserved of its packages (design 47 H2 §805). Nil is told none.
Reserved func(ctx context.Context, v ReservedVersion)
// Releases is the builder's own record of the versions it saw released, by module and fingerprint (design 47 G A-456a): the
// reservation is deleted at release, so a rerun of a released build's step reads it here, never from the catalog. Nil keeps none.
Releases ReserveStore
// ReleasedKept is how many releases per module the record keeps, the oldest dropped (the setting released-kept; A-456b); 0 is
// DefaultReleasedKept.
ReleasedKept int
Now func() time.Time
// Logf says an index write that failed (H-A-269 (1)): the reservation stands, and Reindex writes its index at the next
// start. Nil says nothing.
Logf func(string, ...any)
// HeldNumbers are the versions a registry seat's record holds of any of module's artifacts (design 47 H H-A-275), read before a
// number is reserved: a number one holds is skipped. Nil reads none; a read that fails is said and the reserve proceeds.
HeldNumbers func(ctx context.Context, module, fingerprint string, refs []string) ([]string, error)
// HeldElsewhere is told the numbers of module a reserve skipped because a registry holds them (H-A-275); nil is told none.
HeldElsewhere func(ctx context.Context, module string, skipped []string)
// Names answers a Go package's module path, which names the directory its tag is written in (design 47 H2 §796); nil, or a
// name not known, tags the version nowhere, said.
Names PackageNames
// Tags reads repository's tags starting with prefix, each with the commit it names, through the seat git's tags (design 47
// H2 §796); away says the read met a transport fault or the error kind away, which lets the reservation go on, said. Nil
// reads none.
Tags func(ctx context.Context, repository, prefix string) (tags []git.Tag, away bool, err error)
// TaggedElsewhere is told the numbers of module a reserve skipped because their tag stands at another commit (§796); nil
// is told none.
TaggedElsewhere func(ctx context.Context, module string, skipped []string)
tagsMu sync.Mutex
// tagsRead are the tags last read of each module, by series patch to the commit, with the catalog revision they were read
// under: a module's tags are read once per catalog revision it reserves under (§796).
tagsRead map[string]tagsAt
}
// tagsAt are a module's tags of one series read under one catalog revision: patch to the commit its tag names.
type tagsAt struct {
revision, prefix string
commits map[int]string
}
// goDirectory is the directory of module's Go package in its repository (§796), from its module path; ok is false, said, when
// no name or no directory of the repository is known.
func (r *Reserver) goDirectory(ctx context.Context, module string, src Source) (string, bool) {
if r.Names == nil {
return "", false
}
path, known, err := r.Names(ctx, module, src.GoArtifact)
switch {
case err != nil:
r.said(fmt.Errorf("%s: its Go package's name cannot be read, so its version is tagged nowhere: %w", module, err))
return "", false
case !known:
r.said(fmt.Errorf("%s: no name of its Go package %s is known yet, so its version is tagged nowhere", module, src.GoArtifact))
return "", false
}
dir, ok := GoDirectory(path, src.Repository)
if !ok {
r.said(fmt.Errorf("%s: its Go package %s is not a module path of the repository %s, so its version is tagged nowhere", module, path, src.Repository))
}
return dir, ok
}
// taggedAt are the patches of series whose tag `<directory>/v<series>.<patch>` stands, each with its commit, read once per
// catalog revision the module reserves under (§796). A read that meets a transport fault or away answers none, said, and the
// reservation goes on; any other failure refuses the reservation, naming it.
func (r *Reserver) taggedAt(ctx context.Context, module, series, dir string, src Source) (map[int]string, error) {
if r.Tags == nil {
return nil, nil
}
prefix := TagPrefix(dir) + series + "."
revision := ""
if src.CatalogRevision > 0 {
revision = fmt.Sprintf("%s.%d", src.Branch, src.CatalogRevision)
r.tagsMu.Lock()
got, ok := r.tagsRead[module]
r.tagsMu.Unlock()
if ok && got.revision == revision && got.prefix == prefix {
return got.commits, nil
}
}
tags, away, err := r.Tags(ctx, src.Repository, prefix)
if err != nil && away {
r.said(fmt.Errorf("%s: its tags %s* cannot be read (%w), so the reservation goes on without them", module, prefix, err))
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("the tags %s* of %s cannot be read: %w", prefix, module, err)
}
commits := map[int]string{}
for _, t := range tags {
if s, p, err := SplitVersion(strings.TrimPrefix(t.Tag, TagPrefix(dir))); err == nil && s == series && strings.HasPrefix(t.Tag, prefix) {
commits[p] = t.Commit
}
}
if revision != "" {
r.tagsMu.Lock()
if r.tagsRead == nil {
r.tagsRead = map[string]tagsAt{}
}
r.tagsRead[module] = tagsAt{revision: revision, prefix: prefix, commits: commits}
r.tagsMu.Unlock()
}
return commits, nil
}
// heldNumbers are the patches of series a registry holds of module's artifacts (H-A-275); a read that fails is said, and none.
func (r *Reserver) heldNumbers(ctx context.Context, module, series, fingerprint string, refs []string) map[int]bool {
if r.HeldNumbers == nil {
return nil
}
vs, err := r.HeldNumbers(ctx, module, fingerprint, refs)
if err != nil {
r.said(fmt.Errorf("%s: the registries' versions are not read before reserving, so a number they hold is not skipped: %w", module, err))
}
out := map[int]bool{}
for _, v := range vs {
if s, p, err := SplitVersion(v); err == nil && s == series {
out[p] = true
}
}
return out
}
// nextFree is the first patch after after that no registry holds and whose tag stands at no other commit than commit, with the
// held numbers it skipped (H-A-275) and those tagged elsewhere (design 47 H2 §796).
func nextFree(series string, after int, held map[int]bool, tagged map[int]string, commit string) (int, []string, []string) {
var skipped, elsewhere []string
for p := after + 1; ; p++ {
switch c, ok := tagged[p]; {
case held[p]:
skipped = append(skipped, fmt.Sprintf("%s.%d", series, p))
case ok && c != commit:
elsewhere = append(elsewhere, fmt.Sprintf("%s.%d (tagged at %s)", series, p, c))
default:
return p, skipped, elsewhere
}
}
}
// said is told the outcome of a reindex: a failure is said, the reservation's own write stands.
func (r *Reserver) said(err error) {
if err != nil && r.Logf != nil {
r.Logf("%v", err)
}
}
func (r *Reserver) now() time.Time {
if r.Now != nil {
return r.Now().UTC()
}
return time.Now().UTC()
}
// SplitVersion is a MAJOR.MINOR.PATCH version's series and patch.
func SplitVersion(version string) (series string, patch int, err error) {
i := strings.LastIndex(version, ".")
if i < 0 || strings.Count(version, ".") != 2 {
return "", 0, fmt.Errorf("%q is not MAJOR.MINOR.PATCH", version)
}
patch, err = strconv.Atoi(version[i+1:])
if err != nil || patch < 0 {
return "", 0, fmt.Errorf("%q is not MAJOR.MINOR.PATCH", version)
}
return version[:i], patch, nil
}
// Reserve reserves the next version of module's series for a build that starts, for artifacts (their names), and answers
// it. The next patch is the one after the higher of the highest released and the one reserved, 0 when there is neither. A
// conflicting write (another build reserved first) is read again and the next taken. A reservation standing for the same
// fingerprint whose version is not released and is above the catalog's highest is kept and answered as it stands, a build again or a re-run after a restart
// alike (design 47 H H-A-225, which amends G A-490b): the run already building carries that number. A reservation another
// fingerprint takes over is withdrawn: its version is kept in Withdrawn, and Took is told so the older build is ended.
func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts []string, src Source) (ReservedVersion, error) {
if r.Filled != nil && !r.Filled() {
return ReservedVersion{}, ErrNotFilled
}
released, err := r.ReleasedOf(ctx, module)
if err != nil {
return ReservedVersion{}, fmt.Errorf("the released versions of %s cannot be read: %w", module, err)
}
high := -1
for _, v := range released {
if s, p, err := SplitVersion(v); err == nil && s == series && p > high {
high = p
}
}
key := ReservedKey(module, series)
held := r.heldNumbers(ctx, module, series, src.Fingerprint, artifacts)
dir, tagged := "", false
var tags map[int]string
if src.GoPackage {
if dir, tagged = r.goDirectory(ctx, module, src); tagged {
if tags, err = r.taggedAt(ctx, module, series, dir, src); err != nil {
return ReservedVersion{}, err
}
}
}
for {
cur, rev, found, err := r.get(ctx, key)
if err != nil {
return ReservedVersion{}, err
}
withdrawn := found && hasKey(cur.Withdrawn, cur.Version) // a number a registry holds elsewhere (H-A-279)
if found && !withdrawn && cur.Fingerprint == src.Fingerprint && cur.Fingerprint != "" && !slices.Contains(released, cur.Version) {
if _, p, err := SplitVersion(cur.Version); err == nil && p > high {
return cur, nil // kept until released (H-A-225); one at or below the catalog's highest can never be released
}
}
next := high
if found && (withdrawn || !(src.Rebuild && cur.Fingerprint == src.Fingerprint)) { // a withdrawn number is never taken again (H-A-226)
if _, p, err := SplitVersion(cur.Version); err == nil && p > next {
next = p
}
}
patch, skipped, elsewhere := nextFree(series, next, held, tags, src.Commit)
out := ReservedVersion{V: 1, Module: module, Series: series, Version: fmt.Sprintf("%s.%d", series, patch), Artifacts: map[string]string{}, At: r.now(),
Repository: src.Repository, Commit: src.Commit, Fingerprint: src.Fingerprint, Branch: src.Branch, Source: src.Revision, Revision: src.CatalogRevision}
if tagged {
out.Tag = GoTagOf(dir, out.Version)
}
for _, a := range artifacts {
out.Artifacts[a] = ""
}
if found {
out.Withdrawn = maps.Clone(cur.Withdrawn)
if cur.Fingerprint != src.Fingerprint {
if out.Withdrawn == nil {
out.Withdrawn = map[string]string{}
}
out.Withdrawn[cur.Version] = cur.Fingerprint // a dropped number stays a gap (H-A-226)
}
// One at or below the catalog's highest released is refused by the plain rule anyway: the map never grows without end.
maps.DeleteFunc(out.Withdrawn, func(v, _ string) bool {
_, p, err := SplitVersion(v)
return err != nil || p <= high
})
if len(out.Withdrawn) == 0 {
out.Withdrawn = nil
}
}
raw, _ := json.Marshal(out)
if found {
_, err = r.Store.Update(ctx, key, raw, rev)
} else {
_, err = r.Store.Create(ctx, key, raw)
}
if errors.Is(err, ErrConflict) {
continue
}
if err != nil {
return ReservedVersion{}, fmt.Errorf("%s cannot be reserved: %w", key, err)
}
r.said(r.reindex(ctx, key, cur, out)) // cur is zero when there was none
if len(skipped) > 0 && r.HeldElsewhere != nil {
r.HeldElsewhere(ctx, module, skipped)
}
if len(elsewhere) > 0 && r.TaggedElsewhere != nil {
r.TaggedElsewhere(ctx, module, elsewhere)
}
if found && r.Took != nil {
r.Took(ctx, cur, out)
}
if r.Reserved != nil {
r.Reserved(ctx, out)
}
return out, nil
}
}
func (r *Reserver) get(ctx context.Context, key string) (ReservedVersion, uint64, bool, error) {
raw, rev, found, err := r.Store.Get(ctx, key)
if err != nil || !found {
return ReservedVersion{}, 0, false, err
}
var v ReservedVersion
if err := json.Unmarshal(raw, &v); err != nil {
return ReservedVersion{}, 0, false, fmt.Errorf("%s holds what is not a reservation: %w", key, err)
}
return v, rev, true, nil
}
// Withdraw withdraws module's reservation of version for the build of its fingerprint, whose number a registry holds at another
// digest (design 47 H H-A-275, H-A-279): the number is kept in Withdrawn and stays a gap (H-A-226), so the build again of that
// fingerprint reserves the next number no registry holds, with fresh digests. It answers the reservation as withdrawn, and false
// when none stands at version.
func (r *Reserver) Withdraw(ctx context.Context, module, version string) (ReservedVersion, bool, error) {
series, _, err := SplitVersion(version)
if err != nil {
return ReservedVersion{}, false, err
}
key := ReservedKey(module, series)
for {
cur, rev, found, err := r.get(ctx, key)
if err != nil || !found || cur.Version != version {
return cur, false, err
}
if hasKey(cur.Withdrawn, version) {
return cur, true, nil
}
if cur.Withdrawn == nil {
cur.Withdrawn = map[string]string{}
}
cur.Withdrawn[version] = cur.Fingerprint
raw, _ := json.Marshal(cur)
if _, err := r.Store.Update(ctx, key, raw, rev); errors.Is(err, ErrConflict) {
continue
} else if err != nil {
return cur, false, fmt.Errorf("%s cannot be withdrawn: %w", key, err)
}
return cur, true, nil
}
}
// Built records artifact's digest under the reserved version of module's series; once every artifact has one the build is
// ended (BuiltAt). A reservation moved on to another version (a newer build took the next) is left alone and answers false.
func (r *Reserver) Built(ctx context.Context, module, version, artifact, digest string, deps []meshbuilds.Dependency) (bool, error) {
series, _, err := SplitVersion(version)
if err != nil {
return false, err
}
key := ReservedKey(module, series)
for {
cur, rev, found, err := r.get(ctx, key)
if err != nil || !found || cur.Version != version {
return false, err
}
was := cur
was.Artifacts = maps.Clone(cur.Artifacts)
cur.Artifacts[artifact] = digest
if len(deps) > 0 {
if cur.Dependencies == nil {
cur.Dependencies = map[string][]meshbuilds.Dependency{}
}
cur.Dependencies[artifact] = deps
}
if cur.BuiltAt.IsZero() && allSet(cur.Artifacts) {
cur.BuiltAt = r.now()
}
raw, _ := json.Marshal(cur)
if _, err := r.Store.Update(ctx, key, raw, rev); errors.Is(err, ErrConflict) {
continue
} else if err != nil {
return false, fmt.Errorf("%s cannot be written: %w", key, err)
}
r.said(r.reindex(ctx, key, was, cur))
return true, nil
}
}
func allSet(m map[string]string) bool {
for _, d := range m {
if d == "" {
return false
}
}
return true
}
// DefaultReleasedKept is the setting released-kept unless set (design 47 G A-456b).
const DefaultReleasedKept = 16
// ReleasedKey is the key of a module's record of its last releases (design 47 G A-456a, A-456b), in the state acted.
func ReleasedKey(module string) string { return "released." + EscapeToken(module) }
// ReleasedRecord is that key's value: the module's last releases, oldest first.
type ReleasedRecord struct {
Kept []ReleasedBuild `json:"kept"`
}
// ReleasedBuild is one release: the fingerprint of the build and its version.
type ReleasedBuild struct {
Fingerprint string `json:"fingerprint"`
Version string `json:"version"`
// Digests are the published digest of each artifact, by reference (H-A-331); absent for a release kept before.
Digests map[string]string `json:"digests,omitempty"`
}
// ReleasedFor says whether the builder saw a version released for the build of fingerprint among the module's last releases.
func (r *Reserver) ReleasedFor(ctx context.Context, module, fingerprint string) (bool, error) {
if r.Releases == nil {
return false, nil
}
raw, _, found, err := r.Releases.Get(ctx, ReleasedKey(module))
if err != nil || !found {
return false, err
}
var rec ReleasedRecord
if json.Unmarshal(raw, &rec) != nil {
return false, nil
}
for _, k := range rec.Kept {
if k.Fingerprint == fingerprint {
return true, nil
}
}
return false, nil
}
// PublishedOf answers the seat mesh-builds' verb published (design 47 H-A-328): whether artifact of module built at fingerprint
// is published. A reservation of that fingerprint whose artifact a registry seat said published at the built digest (marked
// from the registry's record too, H-A-274), its number not withdrawn, answers its version, digest and, where kept, the
// publish time (H-A-331); a release of that fingerprint among the module's
// last releases answers published at its version, every artifact of a release being published; otherwise not published.
func (r *Reserver) PublishedOf(ctx context.Context, module, artifact, fingerprint string) (meshbuilds.PublishedAnswer, error) {
module, artifact, fingerprint = strings.TrimSpace(module), strings.TrimSpace(artifact), strings.TrimSpace(fingerprint)
if module == "" || artifact == "" || fingerprint == "" {
return meshbuilds.PublishedAnswer{}, errors.New("module, artifact and fingerprint are required")
}
keys, err := r.Store.KeysOf(ctx, NameFilter(module))
if err != nil {
return meshbuilds.PublishedAnswer{}, fmt.Errorf("the reservations of %s cannot be listed: %w", module, err)
}
sort.Strings(keys)
for _, k := range keys {
cur, _, found, err := r.get(ctx, k)
if err != nil {
return meshbuilds.PublishedAnswer{}, err
}
// A number withdrawn by a take-over is never answered published (H-A-331).
if found && cur.Module == module && cur.Fingerprint == fingerprint && cur.Published[artifact] && !hasKey(cur.Withdrawn, cur.Version) {
a := meshbuilds.PublishedAnswer{Published: true, Version: cur.Version, Digest: cur.Artifacts[artifact]}
if at, ok := cur.PublishedAt[artifact]; ok {
a.At = &at
}
return a, nil
}
}
if r.Releases != nil {
raw, _, found, err := r.Releases.Get(ctx, ReleasedKey(module))
if err != nil {
return meshbuilds.PublishedAnswer{}, fmt.Errorf("the releases of %s cannot be read: %w", module, err)
}
var rec ReleasedRecord
if found && json.Unmarshal(raw, &rec) == nil {
for i := len(rec.Kept) - 1; i >= 0; i-- {
if rec.Kept[i].Fingerprint == fingerprint {
return meshbuilds.PublishedAnswer{Published: true, Version: rec.Kept[i].Version, Digest: rec.Kept[i].Digests[artifact]}, nil
}
}
}
}
return meshbuilds.PublishedAnswer{Published: false}, nil
}
// keepRelease adds a release to the module's record, dropping the oldest past ReleasedKept.
func (r *Reserver) keepRelease(ctx context.Context, module, fingerprint, version string, digests map[string]string) error {
n := r.ReleasedKept
if n < 1 {
n = DefaultReleasedKept
}
for {
raw, rev, found, err := r.Releases.Get(ctx, ReleasedKey(module))
if err != nil {
return err
}
var rec ReleasedRecord
if found {
_ = json.Unmarshal(raw, &rec)
}
kept := rec.Kept[:0:0]
for _, k := range rec.Kept {
if k.Fingerprint != fingerprint {
kept = append(kept, k)
}
}
kept = append(kept, ReleasedBuild{Fingerprint: fingerprint, Version: version, Digests: digests})
if len(kept) > n {
kept = kept[len(kept)-n:]
}
out, _ := json.Marshal(ReleasedRecord{Kept: kept})
if found {
_, err = r.Releases.Update(ctx, ReleasedKey(module), out, rev)
} else {
_, err = r.Releases.Create(ctx, ReleasedKey(module), out)
}
if errors.Is(err, ErrConflict) {
continue
}
return err
}
}
// Released takes the catalog's version-released in: the key of that version is deleted, and every key of module's other
// series (the manifest moved on). A key holding a newer reserved version of the same series stays. It answers the keys
// deleted.
func (r *Reserver) Released(ctx context.Context, module, version string) ([]string, error) {
series, _, err := SplitVersion(version)
if err != nil {
return nil, err
}
keys, err := r.Store.KeysOf(ctx, NameFilter(module))
if err != nil {
return nil, fmt.Errorf("the reservations of %s cannot be listed: %w", module, err)
}
var gone []string
for _, k := range keys {
cur, _, found, err := r.get(ctx, k)
if err != nil {
return gone, err
}
if !found || (cur.Series == series && cur.Version != version) {
continue
}
if cur.Version == version && cur.Fingerprint != "" && r.Releases != nil {
if err := r.keepRelease(ctx, module, cur.Fingerprint, version, cur.Artifacts); err != nil {
return gone, fmt.Errorf("the release of %s %s cannot be recorded: %w", module, version, err)
}
}
if err := r.Store.Delete(ctx, k); err != nil {
return gone, fmt.Errorf("%s cannot be deleted: %w", k, err)
}
r.said(r.reindex(ctx, k, cur, ReservedVersion{}))
gone = append(gone, k)
}
sort.Strings(gone)
return gone, nil
}
// Unreleased are the reservations whose build ended more than UnreleasedBound before now with an artifact unpublished:
// the finding version-unreleased, naming the module, the version and the artifacts missing (published says whether an
// artifact is). A reservation whose build has not ended is not one.
type Unreleased struct {
Module, Version string
Missing []string
}
// Overdue answers the reservations past the bound with an artifact not published.
func (r *Reserver) Overdue(ctx context.Context, published func(module, version, artifact string) bool) ([]Unreleased, error) {
keys, err := r.Store.KeysOf(ctx, AllFilter)
if err != nil {
return nil, fmt.Errorf("the reservations cannot be listed: %w", err)
}
sort.Strings(keys)
var out []Unreleased
for _, k := range keys {
cur, _, found, err := r.get(ctx, k)
if err != nil {
return nil, err
}
if !found || cur.BuiltAt.IsZero() || r.now().Sub(cur.BuiltAt) < UnreleasedBound {
continue
}
var missing []string
for _, a := range cur.Missing() {
if published == nil || !published(cur.Module, cur.Version, a) {
missing = append(missing, a)
}
}
if len(missing) > 0 {
out = append(out, Unreleased{Module: cur.Module, Version: cur.Version, Missing: missing})
}
}
return out, nil
}
// All are the reservations, sorted by key: builder_reserved.
func (r *Reserver) All(ctx context.Context) ([]ReservedVersion, error) {
keys, err := r.Store.KeysOf(ctx, AllFilter)
if err != nil {
return nil, fmt.Errorf("the reservations cannot be listed: %w", err)
}
sort.Strings(keys)
var out []ReservedVersion
for _, k := range keys {
cur, _, found, err := r.get(ctx, k)
if err != nil {
return nil, err
}
if found {
out = append(out, cur)
}
}
return out, nil
}
// TakenOver is the reservation of module's series whose withdrawn numbers hold one of the build of fingerprint (design 47 H
// H-A-226), and whether there is one: the series moved to that reservation's fingerprint and version (H-A-244).
func (r *Reserver) TakenOver(ctx context.Context, module, fingerprint string) (ReservedVersion, bool, error) {
all, err := r.All(ctx)
if err != nil {
return ReservedVersion{}, false, err
}
for _, v := range all {
if v.Module != module || v.Fingerprint == fingerprint {
continue
}
for _, fp := range v.Withdrawn {
if fp == fingerprint {
return v, true, nil
}
}
}
return ReservedVersion{}, false, nil
}
// Drop deletes module's reservation of series on a person's word (builder_reserved_drop); the caller writes the hand-act
// log. It answers whether there was one.
func (r *Reserver) Drop(ctx context.Context, module, series string) (bool, error) {
key := ReservedKey(module, series)
cur, _, found, err := r.get(ctx, key)
if err != nil || !found {
return false, err
}
if err := r.Store.Delete(ctx, key); err != nil {
return true, err
}
r.said(r.reindex(ctx, key, cur, ReservedVersion{}))
return true, nil
}
// Marked is an artifact of a reservation a registry seat's published newly marked: the reservation as it stands then.
type Marked struct {
Reserved ReservedVersion
Artifact string
}
// PackagePublished is the seat mesh-builds' package-published of one artifact the holder said published at the reservation's
// version (design 47 G §52, §334, §673, A-57): the artifact reference, the version and the digest published, the watched branch,
// the source revision the build was made from and the catalog revision its step ask named (H-A-218), and the dependencies its
// step result named.
func (m Marked) PackagePublished(digest string, at time.Time) meshbuilds.PackagePublished {
at = at.UTC()
return meshbuilds.PackagePublished{V: 1, Artifact: m.Artifact, At: &at, Branch: m.Reserved.Branch, Source: m.Reserved.Source, Revision: m.Reserved.Revision, Module: m.Reserved.Module,
Version: m.Reserved.Version, Digest: digest, Dependencies: m.Reserved.Dependencies[m.Artifact]}
}
// Published takes a registry seat's published in, as PublishedRead does, answering the artifacts newly marked and the keys it is
// kept on.
func (r *Reserver) Published(ctx context.Context, name, version, digest string) ([]Marked, []string, error) {
out, err := r.PublishedRead(ctx, name, version, digest, nil)
return out.Marked, out.Kept, err
}
// ReadReservation is a reservation a published read, as it stood after the published was taken in, and whether one of its built
// digests is the published's (design 47 H H-A-280).
type ReadReservation struct {
Key string
Reserved ReservedVersion
Matched bool
}
// PublishedOutcome is what a published did: the artifacts it newly marked, the keys it is kept on (A-415), and every reservation it
// read, each once, which the journal line is made from (H-A-280).
type PublishedOutcome struct {
Marked []Marked
Kept []string
Read []ReadReservation
}
// PublishedRead takes a registry seat's published in: the artifacts of a reservation at version (a leading v is the Go form)
// whose built digest it names are marked published. A published that marks no artifact of a reservation at version whose build
// has not recorded every digest is kept on that reservation as heard (name, version, digest), and the keys it is kept on are
// answered (design 47 G A-415): Kept marks it when the build records its digests. The reservation is found through the index
// (H-A-269 (1), H-A-271, H-A-280): the reservations holding the digest in one read, and only when none at version holds it, those
// at the version whose build has not recorded every digest in one more; never a scan of every reservation, and each reservation
// read at most once (again only on a write that met another). slow is told before the second pass, which may read many, so the
// caller can say in progress to the bus; nil is told nothing.
func (r *Reserver) PublishedRead(ctx context.Context, name, version, digest string, slow func()) (PublishedOutcome, error) {
version = strings.TrimPrefix(version, "v")
var out PublishedOutcome
if version == "" || digest == "" {
return out, nil
}
keys, err := r.indexed(ctx, IndexDigestKey(digest))
if err != nil {
return out, fmt.Errorf("the index of digest %s cannot be read: %w", digest, err)
}
read := map[string]readOne{}
matched, err := r.publishedOn(ctx, keys, name, version, digest, false, read, &out)
if err != nil || matched {
return out, err
}
keys, err = r.indexed(ctx, IndexUnbuiltKey(version))
if err != nil {
return out, fmt.Errorf("the index of the unbuilt at version %s cannot be read: %w", version, err)
}
if len(keys) > 0 && slow != nil {
slow()
}
_, err = r.publishedOn(ctx, keys, name, version, digest, true, read, &out)
return out, err
}
// readOne is a reservation as one pass read it, kept so the next pass reads it no more.
type readOne struct {
v ReservedVersion
rev uint64
ok bool
}
// publishedOn takes a published in on the reservations under keys: an artifact whose built digest it names is marked; with keep,
// a reservation at version whose build has not recorded every digest and that it matches nothing of keeps it (A-415). A
// reservation read by an earlier pass (read) is taken as read; each read is added to out.Read once. It answers whether a
// reservation at version holds the digest.
func (r *Reserver) publishedOn(ctx context.Context, keys []string, name, version, digest string, keep bool, read map[string]readOne, out *PublishedOutcome) (bool, error) {
anyMatched := false
for _, k := range keys {
one, seen := read[k]
for {
if !seen {
cur, rev, found, err := r.get(ctx, k)
if err != nil {
return anyMatched, err
}
one = readOne{v: cur, rev: rev, ok: found}
}
seen = false // a write that meets another reads again
cur := one.v
if !one.ok || cur.Version != version {
read[k] = one
break
}
changed, matched, keepIt := false, false, false
var arts []string
for a, d := range cur.Artifacts {
if d != digest {
continue
}
matched = true
if !cur.Published[a] {
cur.Published, cur.PublishedAt = maps.Clone(cur.Published), maps.Clone(cur.PublishedAt)
cur.markPublished(a, r.now())
changed = true
arts = append(arts, a)
}
}
sort.Strings(arts)
if keep && !matched && cur.BuiltAt.IsZero() && !cur.keeps(name, digest) {
cur.Kept = append(slices.Clone(cur.Kept), KeptPublished{Name: name, Version: version, Digest: digest})
changed, keepIt = true, true
}
rev := one.rev
if changed {
raw, _ := json.Marshal(cur)
var err error
if rev, err = r.Store.Update(ctx, k, raw, one.rev); errors.Is(err, ErrConflict) {
continue
} else if err != nil {
return anyMatched, fmt.Errorf("%s cannot be written: %w", k, err)
}
for _, a := range arts {
out.Marked = append(out.Marked, Marked{Reserved: cur, Artifact: a})
}
if keepIt {
out.Kept = append(out.Kept, k)
}
}
anyMatched = anyMatched || matched
if _, was := read[k]; !was {
out.Read = append(out.Read, ReadReservation{Key: k, Reserved: cur, Matched: matched})
}
read[k] = readOne{v: cur, rev: rev, ok: true}
break
}
}
return anyMatched, nil
}
func (v ReservedVersion) keeps(name, digest string) bool {
for _, h := range v.Kept {
if h.Name == name && h.Digest == digest {
return true
}
}
return false
}
// Kept takes the published kept on module's reservation of version in once its build recorded every digest (design 47 G
// A-415): each kept published whose digest is an artifact's marks that artifact, as one heard after would, and every kept one
// leaves the reservation. It answers the artifacts newly marked and the kept published whose digest matched no artifact.
func (r *Reserver) Kept(ctx context.Context, module, version string) ([]Marked, []KeptPublished, error) {
series, _, err := SplitVersion(version)
if err != nil {
return nil, nil, err
}
key := ReservedKey(module, series)
for {
cur, rev, found, err := r.get(ctx, key)
if err != nil || !found || cur.Version != version || cur.BuiltAt.IsZero() || len(cur.Kept) == 0 {
return nil, nil, err
}
var arts []string
var unmatched []KeptPublished
for _, h := range cur.Kept {
matched := false
for a, d := range cur.Artifacts {
if d != h.Digest {
continue
}
matched = true
if !cur.Published[a] {
cur.markPublished(a, r.now())
arts = append(arts, a)
}
}
if !matched {
unmatched = append(unmatched, h)
}
}
cur.Kept = nil
sort.Strings(arts)
raw, _ := json.Marshal(cur)
if _, err := r.Store.Update(ctx, key, raw, rev); errors.Is(err, ErrConflict) {
continue
} else if err != nil {
return nil, nil, fmt.Errorf("%s cannot be written: %w", key, err)
}
var marked []Marked
for _, a := range arts {
marked = append(marked, Marked{Reserved: cur, Artifact: a})
}
return marked, unmatched, nil
}
}
// Of is module's reservation of series, and whether there is one; with an empty series the module's highest.
func (r *Reserver) Of(ctx context.Context, module, version string) (ReservedVersion, bool, error) {
all, err := r.All(ctx)
if err != nil {
return ReservedVersion{}, false, err
}
var out ReservedVersion
found := false
for _, v := range all {
if v.Module != module || (version != "" && v.Version != version) {
continue
}
if !found {
out, found = v, true
continue
}
_, p, _ := SplitVersion(v.Version)
_, q, _ := SplitVersion(out.Version)
if v.Series > out.Series || (v.Series == out.Series && p > q) {
out = v
}
}
return out, found, nil
}
const reservedBoundPrefix = "reserved."
// ReservedBoundKey is the bound of a reservation's publish in the state bounds (key-ttl): `reserved.<reserved key>`, armed
// when its build ended with every digest known. Its end while the reservation stands is the finding version-unreleased.
func ReservedBoundKey(module, series string) string {
return ReservedBoundFor(ReservedKey(module, series))
}
// ReservedBoundFor is the bound of the reservation under key.
func ReservedBoundFor(key string) string { return reservedBoundPrefix + key }
// ReservedBoundOf is the reserved key a reservation's bound names, and whether key is one.
func ReservedBoundOf(key string) (string, bool) { return strings.CutPrefix(key, reservedBoundPrefix) }
// readOnlyReserve is the reservations read by the tools bundle, which writes nothing.
type readOnlyReserve struct{ readStore }
func (readOnlyReserve) Delete(context.Context, string) error { return errReadOnly }
// AnswerReserved is builder_reserved: every reservation, or one module's.
func AnswerReserved(r Reader, module string) (map[string]any, error) {
rs := &Reserver{Store: readOnlyReserve{readStore{r}}}
all, err := rs.All(context.Background())
if err != nil {
return nil, err
}
out := []ReservedVersion{}
for _, v := range all {
if module == "" || v.Module == module {
out = append(out, v)
}
}
return map[string]any{"reserved": out}, nil
}
// AnswerReservedVerb is the seat mesh-builds' verb reserved: module's reserved version (the given one, else the highest)
// with each artifact's digest, an artifact not built yet with none.
func AnswerReservedVerb(r Reader, module, version string) (meshbuilds.ReservedAnswer, error) {
module = strings.TrimSpace(module)
if module == "" {
return meshbuilds.ReservedAnswer{}, errors.New("module is required")
}
rs := &Reserver{Store: readOnlyReserve{readStore{r}}}
v, found, err := rs.Of(context.Background(), module, strings.TrimSpace(version))
if err != nil {
return meshbuilds.ReservedAnswer{}, err
}
if !found {
return meshbuilds.ReservedAnswer{}, fmt.Errorf("%s has no reserved version%s", module, map[bool]string{true: " " + version, false: ""}[version != ""])
}
out := meshbuilds.ReservedAnswer{Module: v.Module, Version: v.Version, Artifacts: []meshbuilds.ReservedAnswerArtifactsItem{}}
names := make([]string, 0, len(v.Artifacts))
for a := range v.Artifacts {
names = append(names, a)
}
sort.Strings(names)
for _, a := range names {
out.Artifacts = append(out.Artifacts, meshbuilds.ReservedAnswerArtifactsItem{Artifact: a, Digest: v.Artifacts[a]})
}
return out, nil
}
// CatalogVersions answers a module's released versions, the seat mesh-catalog's verb versions (design 47 G §582).
type CatalogVersions func(ctx context.Context, module string) ([]meshcatalog.VersionEntry, error)
// AnswerReservedByName is the verb reserved given a package's name (design 47 G §572, §582, A-70): the module is found from the
// name in the catalog half's view (carriers); that module's reserved version answers as AnswerReservedVerb does. A version
// given with no reservation of it (a reservation goes once its version is released) is asked of the catalog, and answers
// released with the artifacts and digests the release recorded. A name no entry carries, more than one entry carries, or a
// version neither reserved nor released is answered error, naming which, set alone.
func AnswerReservedByName(ctx context.Context, r Reader, carriers func(context.Context, string) ([]Built, error), released CatalogVersions, name, version string) (meshbuilds.ReservedAnswer, error) {
name, version = strings.TrimSpace(name), strings.TrimSpace(version)
got, err := carriers(ctx, name)
if err != nil {
return meshbuilds.ReservedAnswer{}, fmt.Errorf("the entries of %s cannot be read: %w", name, err)
}
switch {
case len(got) == 0:
return meshbuilds.ReservedAnswer{Error: NoEntryCarries + name}, nil
case len(got) > 1:
return meshbuilds.ReservedAnswer{Error: MoreThanOne(name, got)}, nil
}
module := got[0].Module
rs := &Reserver{Store: readOnlyReserve{readStore{r}}}
if _, found, err := rs.Of(ctx, module, version); err != nil {
return meshbuilds.ReservedAnswer{}, err
} else if found {
return AnswerReservedVerb(r, module, version)
}
if version != "" {
entries, err := released(ctx, module)
if err != nil {
return meshbuilds.ReservedAnswer{}, fmt.Errorf("the released versions of %s cannot be read: %w", module, err)
}
for _, e := range entries {
if e.Version != version {
continue
}
out := meshbuilds.ReservedAnswer{Module: module, Version: version, Released: true, Artifacts: []meshbuilds.ReservedAnswerArtifactsItem{}}
for _, a := range e.Artifacts {
out.Artifacts = append(out.Artifacts, meshbuilds.ReservedAnswerArtifactsItem{Artifact: a.Artifact, Digest: a.Digest})
}
return out, nil
}
if series, _, err := SplitVersion(version); err == nil {
if cur, _, found, err := rs.get(ctx, ReservedKey(module, series)); err == nil && found && hasKey(cur.Withdrawn, version) {
// A late publish of a build a take-over withdrew (H-A-225): refused as what it is.
return meshbuilds.ReservedAnswer{Error: fmt.Sprintf("%s (%s) version %s was withdrawn: the build of %s it was reserved for was taken over by %s, which reserves %s",
name, module, version, cur.Withdrawn[version], cur.Fingerprint, cur.Version)}, nil
}
}
return meshbuilds.ReservedAnswer{Error: fmt.Sprintf("%s (%s) has no version %s reserved or released", name, module, version)}, nil
}
return meshbuilds.ReservedAnswer{Error: fmt.Sprintf("%s (%s) has no reserved version", name, module)}, nil
}
// hasKey says whether m holds k.
func hasKey(m map[string]string, k string) bool {
_, ok := m[k]
return ok
}
// NoEntryCarries opens the answer of reserved given a name no entry carries.
const NoEntryCarries = "no entry carries the name "
// The descriptions of the reservation's tools.
const (
ReservedDescription = "The versions the builder reserved (its state reserved, design 47 G section 369): one per module and MAJOR.MINOR, " +
"the version its build reserved at its start, each artifact's digest once built, when the build ended and which artifacts a registry " +
"seat said published. With module, that module's only. A key goes when the catalog says version-released for it. Read-only."
ReservedDropDescription = "Drop a module's reservation of one MAJOR.MINOR by hand (design 47 G section 369), when its version will never be " +
"released; the next build takes the next after the highest released. A hand act: it is recorded in the hand-act log."
ReserveDoneDescription = "Reserve a version for a done trunk build that reserved none (design 47 G A-455): given module and fp, the next version " +
"of the series is reserved for that fingerprint with the digests of the build's artifacts, and its publish is queued as builder_publish_again " +
"queues it; builds nothing. For a done key that cannot be built again. Refused when the key is not done, is no first watched branch's, or " +
"holds a reservation already. A hand act: it is recorded in the hand-act log."
// ReservedVerb is the seat mesh-builds' verb served as a tool of the same name.
ReservedVerb = meshbuilds.VerbReserved
)
// Missing are the artifacts of v no registry seat said published, sorted.
func (v ReservedVersion) Missing() []string {
var out []string
for a := range v.Artifacts {
if !v.Published[a] {
out = append(out, a)
}
}
sort.Strings(out)
return out
}
// At is the reservation under key, and whether there is one.
func (r *Reserver) At(ctx context.Context, key string) (ReservedVersion, bool, error) {
v, _, found, err := r.get(ctx, key)
return v, found, err
}