With one consumer per holder an item has no class to move ahead of, so builder_publish_first, its mode and its act go. The verb publish-queue lists the items in stream order, the order the one consumer gives them, and its description says so. A holder is a module name, so one ending in -live or -again (an earlier release's class durable) is refused at the learn, said, instead of having another holder's durable deleted. Builds: design 47 H2 H-A-483 third addendum, §793
947 lines
39 KiB
Go
947 lines
39 KiB
Go
package versions
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"slices"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
|
)
|
|
|
|
// **A reserved version of a done build is published by the builder until its release is written** (design 47 G §588, A-457a): the
|
|
// publish is an item of the builder's re-ask state, asked when the build ends and at its start, tried with a growing pause and
|
|
// ended loudly after its tries; each registry seat's holder, as the seat-holders view names it (A-416, A-416a), has its items on
|
|
// the builder's work queue `publishes` (design 47 H2 §793), subject `publish.<holder>.<class>.<module>.<version>`, pulled by one
|
|
// consumer per holder and class, the live class first. A send is the registry seat's `publish` with the digest and no
|
|
// parts (A-73): the holder takes the bytes from the artifact store by digest, and refuses what no store holds. Nothing waits on a
|
|
// node-engine `running` or `manifest-reads-changed` any more.
|
|
|
|
const (
|
|
// PublishQueueState is the builder's old state of the queues, whose items are put on the work queue once at the first start
|
|
// of this release, then removed by the builder itself, which made it (design 47 H2 §793; H-A-461 (1)).
|
|
PublishQueueState = "publish-queue"
|
|
// PublishesLocal is the builder's work queue of publish items, asked for in its requires (§793, §820; H-A-448).
|
|
PublishesLocal = "publishes"
|
|
// ActedState holds, one key per subject, the last stream sequence the builder acted on; capped at ActedMiB.
|
|
ActedState = "acted"
|
|
ActedMiB = 1
|
|
// PublishQueueTool is builder_publish_queue.
|
|
PublishQueueTool = "builder_publish_queue"
|
|
// QueueFull is what version-unreleased says of a version the work queue's cap refused.
|
|
QueueFull = "queue full"
|
|
// PublishDropTool is builder_publish_drop (§793); builder_publish_first is retired (H-A-483, third addendum).
|
|
PublishDropTool = "builder_publish_drop"
|
|
// NotStoredWords opens what a refused artifact is said as.
|
|
NotStoredWords = "not stored: "
|
|
// VerbPublish is the registry seats' verb a publish again sends, VerbVersions the one whose record says what a seat holds
|
|
// (H-A-272).
|
|
VerbPublish = "publish"
|
|
VerbVersions = "versions"
|
|
// PublishClassLive is the class of a publish whose build ended in this run, which a pipeline's build step waits on;
|
|
// PublishClassStart that of a publish asked again at a start, of a build that ended before it. A holder's items are pulled
|
|
// by its one consumer in the order they are put (design 47 H H-A-290 (1), H2 §793, H-A-483).
|
|
PublishClassLive = "live"
|
|
PublishClassStart = "again"
|
|
// PublishSendsAtOnce bounds the sends of one holder that run at once, its consumers' max ack pending (H-A-290 (3), §793).
|
|
PublishSendsAtOnce = 4
|
|
// PublishQueueListCap caps the queued versions and the publish items builder_publish_queue lists (H-A-290 (5)).
|
|
PublishQueueListCap = 50
|
|
)
|
|
|
|
// RepublishTransfer is a publish again's transfer name, the builder's own (the registry part's publish: the caller names its
|
|
// transfer): safe as a transfer id, which holds none of "/", ".", " ", "*" or ">" (mesh-sdk's parts receiver refuses them), so a
|
|
// version's dots are dashes (design 47 G §588).
|
|
func RepublishTransfer(module, version, artifact string) string {
|
|
return strings.NewReplacer("/", "-", ".", "-", " ", "-", "*", "-", ">", "-", "@", "-").Replace("republish-" + module + "-" + version + "-" + artifact)
|
|
}
|
|
|
|
// RegistrySeats are the four registry seats a version is published to.
|
|
var RegistrySeats = []string{SeatImages, SeatArtifacts, SeatGoPackages, SeatNPM}
|
|
|
|
// SeatOfArtifact is the registry seat of one artifact of the catalog graph entry: from its package's ecosystem when the
|
|
// entry names one (go, npm, image), else from its kind (SeatOf).
|
|
func SeatOfArtifact(a meshcatalog.Artifact) (string, bool) {
|
|
if a.Package != nil {
|
|
switch a.Package.Ecosystem {
|
|
case "go":
|
|
return SeatGoPackages, true
|
|
case "npm":
|
|
return SeatNPM, true
|
|
case "image":
|
|
return SeatImages, true
|
|
}
|
|
}
|
|
seat, _, ok := SeatOf(a.Kind, a.Language)
|
|
return seat, ok
|
|
}
|
|
|
|
// QueuedPublish is one item of the work queue publishes (§793), and the value of a key of the old state publish-queue: the
|
|
// artifacts of the version this holder publishes, by reference with their digests, and when it was queued.
|
|
type QueuedPublish struct {
|
|
V int `json:"v"`
|
|
Holder string `json:"holder"`
|
|
Module string `json:"module"`
|
|
Version string `json:"version"`
|
|
Artifacts map[string]string `json:"artifacts"`
|
|
Queued time.Time `json:"queued"`
|
|
// Class is PublishClassLive or PublishClassStart (H-A-290 (1)); a value written before it had one is of the start.
|
|
Class string `json:"class,omitempty"`
|
|
}
|
|
|
|
// streamOrder sorts the keys of the work queue's items in stream order: each key is its item's stream sequence (the builder's
|
|
// reader, publishitem.go), compared as a number; a key that is no number sorts after them by its words.
|
|
func streamOrder(keys []string) {
|
|
sort.SliceStable(keys, func(i, j int) bool {
|
|
a, aerr := strconv.ParseUint(keys[i], 10, 64)
|
|
b, berr := strconv.ParseUint(keys[j], 10, 64)
|
|
switch {
|
|
case aerr == nil && berr == nil:
|
|
return a < b
|
|
case aerr == nil || berr == nil:
|
|
return aerr == nil
|
|
}
|
|
return keys[i] < keys[j]
|
|
})
|
|
}
|
|
|
|
// QueueKey is the key of holder's queued version of module in the old state publish-queue.
|
|
func QueueKey(holder, module, version string) string {
|
|
return EscapeToken(holder) + "." + EscapeToken(module) + "." + EscapeToken(version)
|
|
}
|
|
|
|
// PublishMsgID is the message id of holder's item of module's version in class (§793): the work queue takes one item of each
|
|
// id within its duplicates window, so an item put twice is put once.
|
|
func PublishMsgID(holder, class, module, version string) string {
|
|
return EscapeToken(holder) + "." + class + "." + EscapeToken(module) + "." + EscapeToken(version)
|
|
}
|
|
|
|
// PublishEvent is the subject of holder's item of module's version in class within the work queue publishes (§793):
|
|
// `publish.<holder>.<class>.<module>.<version>`, each token escaped.
|
|
func PublishEvent(holder, class, module, version string) string {
|
|
return "publish." + PublishMsgID(holder, class, module, version)
|
|
}
|
|
|
|
// ErrQueueFull is a put the work queue refused as full: said as QueueFull, and asked again at the item's next try (§820).
|
|
var ErrQueueFull = errors.New("the work queue " + PublishesLocal + " is full")
|
|
|
|
// Acted is the value of an acted key: the last stream sequence acted on for its subject.
|
|
type Acted struct {
|
|
Sequence uint64 `json:"sequence"`
|
|
}
|
|
|
|
// Publish is one artifact's publish again: what the registry seat's publish takes, without parts.
|
|
type Publish struct {
|
|
Seat, Kind, Name, Version, Digest, Fingerprint, Commit, Transfer string
|
|
}
|
|
|
|
// Republisher publishes reserved versions again (design 47 G §588).
|
|
type Republisher struct {
|
|
Reserved *Reserver
|
|
// Put puts one item on the work queue publishes under its message id (PublishMsgID): duplicate says the queue holds that
|
|
// id within its duplicates window and took nothing; a full queue is ErrQueueFull.
|
|
Put func(ctx context.Context, q QueuedPublish) (duplicate bool, err error)
|
|
// Entry reads the catalog graph entry of a fingerprint (the seat mesh-catalog's verb entry).
|
|
Entry func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error)
|
|
// Send sends one publish to a registry seat; an error is the holder's refusal or the call's fault, in its words.
|
|
Send func(ctx context.Context, p Publish) error
|
|
// Unreleased says version-unreleased about a version with the holder's words.
|
|
Unreleased func(ctx context.Context, v ReservedVersion, why string)
|
|
Now func() time.Time
|
|
Logf func(string, ...any)
|
|
// Holders answers a registry seat's holders: the catalog's SeatHolders, the part's one holder function that its FollowSeat
|
|
// of the assignments takes too, read from the builder's start (design 47 G A-416, A-416a); nil answers none.
|
|
Holders func(seat string) []string
|
|
// Started is when this run of the builder started: a version whose build ended at or after it is of the live class, one
|
|
// that ended before it of the start's (H-A-290 (1)). Zero makes every version live.
|
|
Started time.Time
|
|
// Ended ends the publish item of a version that never publishes, its reservation withdrawn or taken over (H-A-290 (2));
|
|
// nil ends none.
|
|
Ended func(ctx context.Context, module, version string)
|
|
|
|
mu sync.Mutex
|
|
// lastQueued is the newest time a version was queued at: a clock that has not moved between two versions (a coarse clock,
|
|
// or a fixed one) must not make the queue's order the order of its keys, so each is queued after the last.
|
|
lastQueued time.Time
|
|
}
|
|
|
|
// queuedAt is the time a version is queued at: now, or a nanosecond after the last one queued when now is not after it, so the
|
|
// order the versions were queued in (newest reservation first) is the order they are sent in.
|
|
func (r *Republisher) queuedAt() time.Time {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
at := r.now()
|
|
if !at.After(r.lastQueued) {
|
|
at = r.lastQueued.Add(time.Nanosecond)
|
|
}
|
|
r.lastQueued = at
|
|
return at
|
|
}
|
|
|
|
func (r *Republisher) now() time.Time {
|
|
if r.Now != nil {
|
|
return r.Now().UTC()
|
|
}
|
|
return time.Now().UTC()
|
|
}
|
|
|
|
func (r *Republisher) logf(f string, a ...any) {
|
|
if r.Logf != nil {
|
|
r.Logf(f, a...)
|
|
}
|
|
}
|
|
|
|
// classOf is the class of v's publish: live when its build ended in this run (H-A-290 (1)).
|
|
func (r *Republisher) classOf(v ReservedVersion) string {
|
|
if v.BuiltAt.Before(r.Started) {
|
|
return PublishClassStart
|
|
}
|
|
return PublishClassLive
|
|
}
|
|
|
|
// NotStanding says why the reservation of a version, as Of answers it (found), never publishes: it stands no more at the
|
|
// version (released, dropped or taken over by a newer build), or it was withdrawn (design 47 H H-A-225, H-A-279). Empty when it
|
|
// stands (H-A-290 (2)).
|
|
func NotStanding(v ReservedVersion, found bool) string {
|
|
switch {
|
|
case !found:
|
|
return "its reservation stands no more at the version (released, dropped or taken over)"
|
|
case hasKey(v.Withdrawn, v.Version):
|
|
return "its reservation was withdrawn"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// newerVersion says whether MAJOR.MINOR.PATCH a is above b, number by number.
|
|
func newerVersion(a, b string) bool {
|
|
pa, pb := strings.Split(a, "."), strings.Split(b, ".")
|
|
if len(pa) != 3 || len(pb) != 3 {
|
|
return false
|
|
}
|
|
for i := range 3 {
|
|
x, errx := strconv.Atoi(pa[i])
|
|
y, erry := strconv.Atoi(pb[i])
|
|
if errx != nil || erry != nil {
|
|
return false
|
|
}
|
|
if x != y {
|
|
return x > y
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Retired are the done builds without a release among all that a newer reservation or a newer release of the same module and
|
|
// series (MAJOR.MINOR) supersedes, each with why (design 47 H H-A-291 (1), H-A-293; H-A-275: the older digest is dead): keyed
|
|
// `<module> <version>`. A standing newer series never retires an older one's build. The releases are the builder's own record
|
|
// of them (Releases), read once per module.
|
|
func (r *Reserver) Retired(ctx context.Context, all []ReservedVersion) (map[string]string, error) {
|
|
seriesOf := func(v ReservedVersion) string {
|
|
s, _, err := SplitVersion(v.Version)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return v.Module + " " + s
|
|
}
|
|
newest := map[string]string{} // module and series -> its newest reserved version
|
|
for _, v := range all {
|
|
k := seriesOf(v)
|
|
if n, ok := newest[k]; k != "" && (!ok || newerVersion(v.Version, n)) {
|
|
newest[k] = v.Version
|
|
}
|
|
}
|
|
kept := map[string][]string{} // module -> the releases the record keeps
|
|
read := map[string]bool{}
|
|
out := map[string]string{}
|
|
for _, v := range all {
|
|
k := seriesOf(v)
|
|
if v.BuiltAt.IsZero() || k == "" {
|
|
continue
|
|
}
|
|
if n := newest[k]; newerVersion(n, v.Version) {
|
|
out[v.Module+" "+v.Version] = "the newer reservation " + n + " of " + v.Module + " supersedes it"
|
|
continue
|
|
}
|
|
if !read[v.Module] && r.Releases != nil {
|
|
read[v.Module] = true
|
|
raw, _, found, err := r.Releases.Get(ctx, ReleasedKey(v.Module))
|
|
if err != nil {
|
|
return out, fmt.Errorf("the releases of %s cannot be read: %w", v.Module, err)
|
|
}
|
|
var rec ReleasedRecord
|
|
if found && json.Unmarshal(raw, &rec) == nil {
|
|
for _, rb := range rec.Kept {
|
|
kept[v.Module] = append(kept[v.Module], rb.Version)
|
|
}
|
|
}
|
|
}
|
|
series, _, _ := SplitVersion(v.Version)
|
|
rel := ""
|
|
for _, kv := range kept[v.Module] {
|
|
if s, _, err := SplitVersion(kv); err == nil && s == series && newerVersion(kv, v.Version) && (rel == "" || newerVersion(kv, rel)) {
|
|
rel = kv
|
|
}
|
|
}
|
|
if rel != "" {
|
|
out[v.Module+" "+v.Version] = "the newer release " + rel + " of " + v.Module + " supersedes it"
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// drop ends module's version's publish item: it never publishes (H-A-290 (2)), said with why; its items on the work queue are
|
|
// terminated when they are pulled (§793).
|
|
func (r *Republisher) drop(ctx context.Context, module, version, why string) {
|
|
r.end(ctx, module, version, why)
|
|
}
|
|
|
|
// end ends module's version's publish item, said with why.
|
|
func (r *Republisher) end(ctx context.Context, module, version, why string) {
|
|
r.logf("publish again: %s %s is not published: %s; its queued key is dropped and its publish item ended", module, version, why)
|
|
if r.Ended != nil {
|
|
r.Ended(context.WithoutCancel(ctx), module, version)
|
|
}
|
|
}
|
|
|
|
// ActedKey is the acted key of a subject.
|
|
func ActedKey(subject string) string { return EscapeToken(subject) }
|
|
|
|
// holdersFn answers a seat's holders from Holders.
|
|
func (r *Republisher) holdersFn(seat string) []string {
|
|
if r.Holders == nil {
|
|
return nil
|
|
}
|
|
return r.Holders(seat)
|
|
}
|
|
|
|
// view is a reserved version with the artifacts of it a holder would publish.
|
|
type view struct {
|
|
v ReservedVersion
|
|
arts map[string]meshcatalog.Artifact // reference -> its catalog artifact
|
|
}
|
|
|
|
// queue queues vs newest first, at most max of them (0 is no limit), each artifact to the first holder holdersOf names for
|
|
// its seat. It answers the versions queued, `<module> <version>`.
|
|
func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf func(seat string) []string, max int) ([]string, error) {
|
|
sort.SliceStable(vs, func(i, j int) bool { return vs[i].At.After(vs[j].At) })
|
|
var queued []string
|
|
var errs []error
|
|
for _, v := range vs {
|
|
if max > 0 && len(queued) >= max {
|
|
break
|
|
}
|
|
if why := NotStanding(v, true); why != "" {
|
|
r.drop(ctx, v.Module, v.Version, why)
|
|
continue
|
|
}
|
|
if v.Fingerprint == "" || v.BuiltAt.IsZero() {
|
|
r.logf("publish again: %s %s is not queued: it holds no fingerprint or its build has not ended with every digest", v.Module, v.Version)
|
|
continue
|
|
}
|
|
e, err := r.Entry(ctx, v.Fingerprint)
|
|
if err != nil {
|
|
// One version's unreadable entry leaves only that version unqueued; the others go on.
|
|
errs = append(errs, fmt.Errorf("the catalog entry %s of %s %s cannot be read: %w", v.Fingerprint, v.Module, v.Version, err))
|
|
continue
|
|
}
|
|
byHolder := map[string]map[string]string{}
|
|
for _, ref := range v.Missing() {
|
|
if _, no := v.Unstored[ref]; no {
|
|
r.logf("publish again: %s %s: %s is not queued: a holder refused it as not stored", v.Module, v.Version, ref)
|
|
continue
|
|
}
|
|
found := false
|
|
for _, a := range e.Artifacts {
|
|
seat, ok := SeatOfArtifact(a)
|
|
if a.Ref != ref || !ok {
|
|
continue
|
|
}
|
|
found = true
|
|
if hs := holdersOf(seat); len(hs) > 0 {
|
|
if byHolder[hs[0]] == nil {
|
|
byHolder[hs[0]] = map[string]string{}
|
|
}
|
|
byHolder[hs[0]][ref] = v.Artifacts[ref]
|
|
} else {
|
|
r.logf("publish again: %s %s: %s is not queued: the registry seat %s has no holder in the seat-holders view", v.Module, v.Version, ref, seat)
|
|
}
|
|
}
|
|
if !found {
|
|
r.logf("publish again: %s %s: %s is not queued: the catalog entry %s names no registry seat for it", v.Module, v.Version, ref, v.Fingerprint)
|
|
}
|
|
}
|
|
did := false
|
|
for holder, arts := range byHolder {
|
|
ok, err := r.enqueue(ctx, v, holder, arts)
|
|
if err != nil {
|
|
return queued, err
|
|
}
|
|
did = did || ok
|
|
}
|
|
if did {
|
|
queued = append(queued, v.Module+" "+v.Version)
|
|
}
|
|
}
|
|
return queued, errors.Join(errs...)
|
|
}
|
|
|
|
// enqueue puts one holder's artifacts of a version on the work queue under its message id: not again while the queue holds that
|
|
// id (its duplicates window), which stands queued, and not past the queue's cap (said as queue full).
|
|
func (r *Republisher) enqueue(ctx context.Context, v ReservedVersion, holder string, arts map[string]string) (bool, error) {
|
|
q := QueuedPublish{V: 1, Holder: holder, Module: v.Module, Version: v.Version, Artifacts: arts, Queued: r.queuedAt(), Class: r.classOf(v)}
|
|
dup, err := r.Put(ctx, q)
|
|
switch {
|
|
case errors.Is(err, ErrQueueFull):
|
|
r.logf("publish again: %s %s is not queued at %s: %v; its publish item asks it again at its next try", v.Module, v.Version, holder, err)
|
|
if r.Unreleased != nil {
|
|
r.Unreleased(ctx, v, QueueFull)
|
|
}
|
|
return false, nil
|
|
case err != nil:
|
|
return false, fmt.Errorf("%s %s cannot be queued at %s: %w", v.Module, v.Version, holder, err)
|
|
case dup:
|
|
r.logf("publish again: %s %s is queued at %s already (class %s): the work queue holds its message id", v.Module, v.Version, holder, q.Class)
|
|
return true, nil
|
|
}
|
|
r.logf("publish again: %s %s is queued to publish at %s: %d artifacts, class %s", v.Module, v.Version, holder, len(arts), q.Class)
|
|
return true, nil
|
|
}
|
|
|
|
// Republish is builder_publish_again: with module and version, that reserved version's missing artifacts; with module
|
|
// alone, each reserved version of it whose artifacts are not all published; with neither, every version the finding
|
|
// version-unreleased names. It queues as the events do (no limit but the cap), at the holders Holders answers (design 47 G
|
|
// A-416), and answers the versions it queued.
|
|
func (r *Republisher) Republish(ctx context.Context, module, version string) ([]string, error) {
|
|
var vs []ReservedVersion
|
|
if module == "" {
|
|
over, err := r.Reserved.Overdue(ctx, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, o := range over {
|
|
if v, ok, err := r.Reserved.Of(ctx, o.Module, o.Version); err == nil && ok {
|
|
vs = append(vs, v)
|
|
}
|
|
}
|
|
} else {
|
|
all, err := r.Reserved.All(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, v := range all {
|
|
if v.Module == module && (version == "" || v.Version == version) && len(v.Missing()) > 0 {
|
|
vs = append(vs, v)
|
|
}
|
|
}
|
|
if len(vs) == 0 {
|
|
return nil, fmt.Errorf("%s holds no reserved version%s with an artifact unpublished", module, map[bool]string{true: " " + version, false: ""}[version != ""])
|
|
}
|
|
}
|
|
return r.queue(ctx, vs, r.holdersFn, 0)
|
|
}
|
|
|
|
// Queued is builder_publish_queue's answer, small at any size of the queue (design 47 H H-A-290 (5)): the queued versions'
|
|
// count per holder and class with the oldest and newest queue time, the first PublishQueueListCap of them in the order they are
|
|
// sent (each with its class), the acted keys with their sequences, and the publish items, counted and capped alike.
|
|
type Queued struct {
|
|
Total int `json:"total"`
|
|
Counts []QueueCount `json:"counts"`
|
|
// Queue are the first queued versions, in stream order, the order each holder's one consumer gives them (H-A-483);
|
|
// with a module named, only that module's.
|
|
Queue []QueuedPublish `json:"queue"`
|
|
// Listed says how many of Total the answer lists.
|
|
Listed int `json:"listed"`
|
|
Acted map[string]uint64 `json:"acted"`
|
|
// ItemCount is how many publish items stand; Items the first of them by their next try (design 47 G A-457a), each with its
|
|
// tries and its next try; an item that ended says why.
|
|
ItemCount int `json:"item-count"`
|
|
Items []PublishItem `json:"items"`
|
|
// Capped says a state held more than the one page read of it (H-A-293); Read names each such state, the keys read and its
|
|
// size. Every count above is then of the keys read.
|
|
Capped bool `json:"capped,omitempty"`
|
|
Read []StateCapped `json:"read,omitempty"`
|
|
}
|
|
|
|
// capped notes a state read only in part.
|
|
func (q *Queued) capped(c *StateCapped) {
|
|
if c != nil {
|
|
q.Capped = true
|
|
q.Read = append(q.Read, *c)
|
|
}
|
|
}
|
|
|
|
// QueueCount is how many versions one holder's queue holds of one class, and when the oldest and the newest were queued.
|
|
type QueueCount struct {
|
|
Holder string `json:"holder"`
|
|
Class string `json:"class"`
|
|
Count int `json:"count"`
|
|
Oldest time.Time `json:"oldest"`
|
|
Newest time.Time `json:"newest"`
|
|
}
|
|
|
|
// PublishItemKeyPrefix opens the key of a publish item in the builder's re-ask state.
|
|
const PublishItemKeyPrefix = "publish."
|
|
|
|
// PublishItem is the builder's own record of a version's publish (design 47 G A-457a): a value of the re-ask state, its tries and
|
|
// its next try. It is asked when the build ends and at the builder's start, rewritten with one more try and a longer pause after
|
|
// each, and deleted when the version is released; after the setting's number of tries it ends, saying so (Answer), and stays.
|
|
type PublishItem struct {
|
|
Module string `json:"module"`
|
|
Version string `json:"version"`
|
|
Tries int `json:"tries"`
|
|
NextTry time.Time `json:"next-try"`
|
|
Why string `json:"why,omitempty"`
|
|
Answer string `json:"answer,omitempty"`
|
|
Answered time.Time `json:"answered,omitzero"`
|
|
}
|
|
|
|
// PublishItemKey is the key of module's version's publish item.
|
|
func PublishItemKey(module, version string) string {
|
|
return PublishItemKeyPrefix + EscapeToken(module) + "." + EscapeToken(version)
|
|
}
|
|
|
|
// PublishQueueOf is the seat mesh-builds' verb publish-queue (design 47 H2 §793; H-A-454): the builder's read of its own work queue
|
|
// publishes (queue, its items read in bulk, capped past its bound), small at any size of the queue (H-A-290 (5)): the items' count
|
|
// per holder and class with the oldest and newest queue time, and the first PublishQueueListCap in stream order, the order the
|
|
// holder's one consumer gives them (H-A-483, third addendum); module
|
|
// (and version) narrow the list to one module's (one version's), so an item's class can be read.
|
|
func PublishQueueOf(queue Reader, module, version string) (meshbuilds.PublishQueueAnswer, error) {
|
|
out := meshbuilds.PublishQueueAnswer{Counts: []meshbuilds.PublishQueueAnswerCountsItem{}, Queue: []meshbuilds.PublishQueueAnswerQueueItem{}}
|
|
vals, c, err := queue.values(PublishesLocal)
|
|
if c != nil {
|
|
out.Capped, out.ItemsRead, out.QueueItems, out.QueueBytes = true, int64(c.KeysRead), int64(c.StateKeys), int64(c.StateBytes)
|
|
}
|
|
if err != nil {
|
|
return out, fmt.Errorf("the work queue %s cannot be read: %w", PublishesLocal, err)
|
|
}
|
|
counts := map[[2]string]*meshbuilds.PublishQueueAnswerCountsItem{}
|
|
var listed []QueuedPublish
|
|
keys := make([]string, 0, len(vals))
|
|
for k := range vals {
|
|
keys = append(keys, k)
|
|
}
|
|
streamOrder(keys)
|
|
for _, k := range keys {
|
|
raw := vals[k]
|
|
var q QueuedPublish
|
|
if json.Unmarshal(raw, &q) != nil {
|
|
continue
|
|
}
|
|
if q.Class == "" {
|
|
q.Class = PublishClassStart
|
|
}
|
|
out.Total++
|
|
c := counts[[2]string{q.Holder, q.Class}]
|
|
if c == nil {
|
|
c = &meshbuilds.PublishQueueAnswerCountsItem{Holder: q.Holder, Class: q.Class, Oldest: q.Queued, Newest: q.Queued}
|
|
counts[[2]string{q.Holder, q.Class}] = c
|
|
}
|
|
c.Count++
|
|
if q.Queued.Before(c.Oldest) {
|
|
c.Oldest = q.Queued
|
|
}
|
|
if q.Queued.After(c.Newest) {
|
|
c.Newest = q.Queued
|
|
}
|
|
if (module == "" || q.Module == module) && (version == "" || q.Version == version) {
|
|
listed = append(listed, q)
|
|
}
|
|
}
|
|
for _, c := range counts {
|
|
out.Counts = append(out.Counts, *c)
|
|
}
|
|
sort.Slice(out.Counts, func(i, j int) bool {
|
|
if out.Counts[i].Holder != out.Counts[j].Holder {
|
|
return out.Counts[i].Holder < out.Counts[j].Holder
|
|
}
|
|
return out.Counts[i].Class < out.Counts[j].Class
|
|
})
|
|
for _, q := range listed {
|
|
if len(out.Queue) == PublishQueueListCap {
|
|
break
|
|
}
|
|
out.Queue = append(out.Queue, meshbuilds.PublishQueueAnswerQueueItem{Holder: q.Holder, Module: q.Module, Version: q.Version,
|
|
Class: q.Class, Artifacts: q.Artifacts, Queued: q.Queued})
|
|
}
|
|
out.Listed = int64(len(out.Queue))
|
|
return out, nil
|
|
}
|
|
|
|
// AnswerQueue is builder_publish_queue: the seat mesh-builds' publish-queue answer the builder gave (q, §793, H-A-454) and the two
|
|
// states as the tools bundle reads them (H-A-290 (5)), each read in bulk (Reader.All), capped at PublishQueueListCap.
|
|
func AnswerQueue(q meshbuilds.PublishQueueAnswer, acted, reasks Reader) (Queued, error) {
|
|
out := Queued{Total: int(q.Total), Counts: []QueueCount{}, Queue: []QueuedPublish{}, Listed: int(q.Listed), Acted: map[string]uint64{}, Items: []PublishItem{}}
|
|
if q.Capped {
|
|
out.capped(&StateCapped{State: PublishesLocal, KeysRead: int(q.ItemsRead), StateKeys: uint64(q.QueueItems), StateBytes: uint64(q.QueueBytes)})
|
|
}
|
|
for _, c := range q.Counts {
|
|
out.Counts = append(out.Counts, QueueCount{Holder: c.Holder, Class: c.Class, Count: int(c.Count), Oldest: c.Oldest, Newest: c.Newest})
|
|
}
|
|
for _, it := range q.Queue {
|
|
out.Queue = append(out.Queue, QueuedPublish{V: 1, Holder: it.Holder, Module: it.Module, Version: it.Version, Artifacts: it.Artifacts, Queued: it.Queued, Class: it.Class})
|
|
}
|
|
avals, c, err := acted.values(ActedState)
|
|
out.capped(c)
|
|
if err != nil {
|
|
return out, fmt.Errorf("the state %s cannot be read: %w", ActedState, err)
|
|
}
|
|
for k, raw := range avals {
|
|
var a Acted
|
|
if json.Unmarshal(raw, &a) == nil {
|
|
out.Acted[k] = a.Sequence
|
|
}
|
|
}
|
|
rvals, c, err := reasks.values("reasks")
|
|
out.capped(c)
|
|
if err != nil {
|
|
return out, fmt.Errorf("the re-ask state cannot be read: %w", err)
|
|
}
|
|
for k, raw := range rvals {
|
|
if !strings.HasPrefix(k, PublishItemKeyPrefix) {
|
|
continue
|
|
}
|
|
var it PublishItem
|
|
if json.Unmarshal(raw, &it) == nil {
|
|
out.Items = append(out.Items, it)
|
|
}
|
|
}
|
|
out.ItemCount = len(out.Items)
|
|
sort.SliceStable(out.Items, func(i, j int) bool { return out.Items[i].NextTry.Before(out.Items[j].NextTry) })
|
|
if len(out.Items) > PublishQueueListCap {
|
|
out.Items = out.Items[:PublishQueueListCap]
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// PublishQueueDescription is builder_publish_queue's description.
|
|
const PublishQueueDescription = "The versions the builder will publish again (its work queue " + PublishesLocal + ", design 47 G section 588, H2 section 793): " +
|
|
"how many each registry seat holder's items hold per class (live, a build a pipeline's step waits on, or again, asked again at a start) " +
|
|
"with the oldest and newest queue time; the first 50 in stream order, the order each holder's one consumer pulls them, each with its class, its artifacts' digests and when it was " +
|
|
"queued (module, and version, list only that module's, so one item's class can be read); the keys of the state " + ActedState + ", the last stream " +
|
|
"sequence the builder acted on for each subject; and the publish items (design 47 G A-457a), counted, the first 50 by next try, each version's " +
|
|
"tries and next try, and why one ended (its tries spent, or dropped). A read past its pages is answered capped, naming the items read and the " +
|
|
"queue's size (H-A-293). Read-only."
|
|
|
|
// PublishDropDescription is builder_publish_drop's (§793).
|
|
const PublishDropDescription = "Drop one version's publish (design 47 H2 section 793): its items on the work queue " + PublishesLocal + " are " +
|
|
"terminated at every holder, and its publish item ended with who dropped it and why, so it is never sent or put again; builder_publish_again " +
|
|
"asks it anew. A hand act, recorded in the hand-act log first."
|
|
|
|
// PublishAgainDescription is builder_publish_again's description.
|
|
const PublishAgainDescription = "Publish reserved versions again (design 47 G section 588), which builds nothing: with module and version, that reserved " +
|
|
"version's missing artifacts; with module alone, each reserved version of that module whose artifacts are not all published; with neither, every " +
|
|
"version the finding version-unreleased names, newest first. It queues them as the builder's publish items do, and answers at once " +
|
|
"with the versions it queued; each outcome is said as package-published, or left in version-unreleased with the holder's words. A hand act, " +
|
|
"recorded in the hand-act log."
|
|
|
|
// MoveQueued puts the values of the old state publish-queue on the work queue, once (§793): each under its message id, so a
|
|
// move run twice puts each once within the duplicates window; its class as of this run (a build ended before it is of the
|
|
// start's, H-A-290 (1)); a version whose reservation was withdrawn, taken over or superseded is not put, its publish item
|
|
// ended (H-A-290 (2), H-A-291 (1)). It reads the reservations once, and answers how many it put.
|
|
func (r *Republisher) MoveQueued(ctx context.Context, vals []json.RawMessage) (int, error) {
|
|
all, err := r.Reserved.All(ctx)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("the queued publishes are not moved: %w", err)
|
|
}
|
|
reserved := make(map[string]ReservedVersion, len(all))
|
|
for _, v := range all {
|
|
reserved[v.Module+" "+v.Version] = v
|
|
}
|
|
retired, err := r.Reserved.Retired(ctx, all)
|
|
if err != nil {
|
|
r.logf("publish again: the superseded builds are not retired at the move: %v", err)
|
|
}
|
|
put := 0
|
|
for _, raw := range vals {
|
|
var q QueuedPublish
|
|
if json.Unmarshal(raw, &q) != nil || q.Holder == "" {
|
|
continue
|
|
}
|
|
v, ok := reserved[q.Module+" "+q.Version]
|
|
why := NotStanding(v, ok)
|
|
if why == "" {
|
|
why = retired[q.Module+" "+q.Version]
|
|
}
|
|
if why != "" {
|
|
r.end(ctx, q.Module, q.Version, why)
|
|
continue
|
|
}
|
|
q.Class = r.classOf(v)
|
|
dup, err := r.Put(ctx, q)
|
|
if err != nil {
|
|
return put, fmt.Errorf("%s %s is not moved to the work queue %s: %w", q.Module, q.Version, PublishesLocal, err)
|
|
}
|
|
if !dup {
|
|
put++
|
|
}
|
|
}
|
|
return put, nil
|
|
}
|
|
|
|
// SendFate is what SendQueued did with one item (§793): Sent, every artifact sent (its item acknowledged when its publish is
|
|
// heard or confirmed); Ended, why the version never publishes (its item terminated); Failed, the holder's or the call's words
|
|
// (its item handed back with the pause).
|
|
type SendFate struct {
|
|
Sent bool
|
|
Ended, Failed string
|
|
}
|
|
|
|
// SendQueued publishes one item's artifacts and says the outcome. Every call it makes honours the one bound it is given,
|
|
// UnreleasedBound (a context's bound, no timer, ADR 0437 §8), and a send that returns without sending says why (H-A-290 (4)).
|
|
func (r *Republisher) SendQueued(ctx context.Context, q QueuedPublish) SendFate {
|
|
bound, cancel := context.WithTimeout(ctx, UnreleasedBound)
|
|
defer cancel()
|
|
all, err := r.Reserved.All(bound)
|
|
if err != nil {
|
|
r.logf("publish again: %s %s is not sent to %s: its reservation cannot be read: %v; it is handed back", q.Module, q.Version, q.Holder, err)
|
|
return SendFate{Failed: "its reservation cannot be read: " + err.Error()}
|
|
}
|
|
var v ReservedVersion
|
|
found := false
|
|
var mine []ReservedVersion
|
|
for _, w := range all {
|
|
if w.Module == q.Module {
|
|
mine = append(mine, w)
|
|
if w.Version == q.Version {
|
|
v, found = w, true
|
|
}
|
|
}
|
|
}
|
|
why := NotStanding(v, found)
|
|
if why == "" {
|
|
retired, err := r.Reserved.Retired(bound, mine)
|
|
if err != nil {
|
|
r.logf("publish again: %s %s: whether a newer version supersedes it cannot be read, so it is sent: %v", q.Module, q.Version, err)
|
|
}
|
|
why = retired[q.Module+" "+q.Version]
|
|
}
|
|
if why != "" {
|
|
r.end(ctx, q.Module, q.Version, why)
|
|
return SendFate{Ended: why}
|
|
}
|
|
e, err := r.Entry(bound, v.Fingerprint)
|
|
if err != nil {
|
|
words := fmt.Sprintf("the catalog entry %s cannot be read: %v", v.Fingerprint, err)
|
|
r.say(ctx, v, words)
|
|
return SendFate{Failed: words}
|
|
}
|
|
missing := map[string]bool{}
|
|
for _, ref := range v.Missing() {
|
|
missing[ref] = true
|
|
}
|
|
refs := make([]string, 0, len(q.Artifacts))
|
|
for ref := range q.Artifacts {
|
|
if _, no := v.Unstored[ref]; no || !missing[ref] {
|
|
continue // published already, or refused as not stored: said once, never sent again
|
|
}
|
|
refs = append(refs, ref)
|
|
}
|
|
sort.Strings(refs)
|
|
if len(refs) == 0 {
|
|
return SendFate{Ended: "none of its artifacts is left to send: each is published or refused as not stored"}
|
|
}
|
|
var failed []string
|
|
sent := 0
|
|
for _, ref := range refs {
|
|
seat, name, kind, ok := publishTarget(e, q.Module, ref)
|
|
if !ok {
|
|
r.say(ctx, v, ref+": the catalog entry names no registry seat for it")
|
|
return SendFate{Failed: ref + ": the catalog entry names no registry seat for it"}
|
|
}
|
|
err := r.Send(bound, Publish{Seat: seat, Kind: kind, Name: name, Version: v.Version, Digest: q.Artifacts[ref], Fingerprint: v.Fingerprint,
|
|
Commit: v.Commit, Transfer: RepublishTransfer(q.Module, v.Version, strings.TrimPrefix(ref, q.Module+":"))})
|
|
if err == nil {
|
|
r.logf("publish again: %s %s: %s sent to %s as %s %s, digest %s; the package-published is said when the seat says published", q.Module, q.Version, ref, seat, name, v.Version, q.Artifacts[ref])
|
|
sent++
|
|
continue
|
|
}
|
|
r.logf("publish again: %s %s: %s refused by %s: %v", q.Module, q.Version, ref, seat, err)
|
|
words := err.Error()
|
|
if strings.Contains(words, strings.TrimSuffix(NotStoredWords, ": ")) {
|
|
if ok, merr := r.Reserved.MarkUnstored(bound, q.Module, q.Version, ref, words); merr != nil {
|
|
r.logf("%s %s: the unstored %s is not kept: %v", q.Module, q.Version, ref, merr)
|
|
} else if ok {
|
|
r.say(ctx, v, ref+" "+NotStoredWords+words)
|
|
}
|
|
continue
|
|
}
|
|
r.say(ctx, v, ref+": "+words)
|
|
failed = append(failed, ref+": "+words)
|
|
}
|
|
switch {
|
|
case len(failed) > 0:
|
|
return SendFate{Failed: strings.Join(failed, "; ")}
|
|
case sent == 0:
|
|
return SendFate{Ended: "each artifact left was refused as not stored"}
|
|
}
|
|
return SendFate{Sent: true}
|
|
}
|
|
|
|
// publishTarget is the registry seat, the name and the kind module's artifact ref is published as, from its catalog entry; ok is
|
|
// false when the entry names no registry seat for it.
|
|
func publishTarget(e meshcatalog.CatalogEntry, module, ref string) (seat, name, kind string, ok bool) {
|
|
var art *meshcatalog.Artifact
|
|
for i := range e.Artifacts {
|
|
if e.Artifacts[i].Ref == ref {
|
|
art = &e.Artifacts[i]
|
|
}
|
|
}
|
|
if art == nil {
|
|
return "", "", "", false
|
|
}
|
|
if seat, ok = SeatOfArtifact(*art); !ok {
|
|
return "", "", "", false
|
|
}
|
|
name = NameOf(module, strings.TrimPrefix(ref, module+":"))
|
|
if art.Package != nil && art.Package.Name != "" {
|
|
name = art.Package.Name
|
|
}
|
|
return seat, name, art.Kind, true
|
|
}
|
|
|
|
func (r *Republisher) say(ctx context.Context, v ReservedVersion, why string) {
|
|
r.logf("%s %s is not published again: %s", v.Module, v.Version, why)
|
|
if r.Unreleased != nil {
|
|
r.Unreleased(ctx, v, why)
|
|
}
|
|
}
|
|
|
|
// MarkUnstored records that a holder refused artifact of module's reserved version as not stored; it answers whether that
|
|
// was new (said once).
|
|
func (r *Reserver) MarkUnstored(ctx context.Context, module, version, artifact, words string) (bool, error) {
|
|
series, _, err := SplitVersion(version)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
key := ReservedKey(module, series)
|
|
for {
|
|
cur, rev, found, err := r.get(ctx, key)
|
|
if err != nil || !found || cur.Version != version {
|
|
return false, err
|
|
}
|
|
if _, was := cur.Unstored[artifact]; was {
|
|
return false, nil
|
|
}
|
|
if cur.Unstored == nil {
|
|
cur.Unstored = map[string]string{}
|
|
}
|
|
cur.Unstored[artifact] = words
|
|
raw, _ := json.Marshal(cur)
|
|
if _, err := r.Store.Update(ctx, key, raw, rev); errors.Is(err, ErrConflict) {
|
|
continue
|
|
} else if err != nil {
|
|
return false, fmt.Errorf("%s cannot be written: %w", key, err)
|
|
}
|
|
return true, nil
|
|
}
|
|
}
|
|
|
|
// UnheardBound is how many messages the builder's durable on the events stream may hold pending and unacknowledged while a
|
|
// publish is asked again (design 47 H H-A-269 (2), H-A-272): at or above it a published of the version may still be unheard, so
|
|
// the try is skipped. A constant of the builder, no setting.
|
|
const UnheardBound = 100
|
|
|
|
// HeldVersion is one version a registry seat's verb versions answers of a name, with its digest.
|
|
type HeldVersion struct {
|
|
Version, Digest string
|
|
}
|
|
|
|
// HeldArtifact is an artifact of a reservation as a registry seat's record holds it at the reservation's version: its reference,
|
|
// the seat and the name it is published as, the version and the digest the seat holds.
|
|
type HeldArtifact struct {
|
|
Ref, Seat, Name, Version, Digest string
|
|
}
|
|
|
|
// Judged is what the registries' record says of a reserved version's missing artifacts (design 47 H H-A-272, H-A-274, H-A-275):
|
|
// those held at the version with the built digest (Held), those held at the version with another digest (Elsewhere), and, when an
|
|
// artifact is held at neither, the first one lacking (Lacks). All says every missing artifact is held with its built digest.
|
|
type Judged struct {
|
|
Held, Elsewhere []HeldArtifact
|
|
Lacks string
|
|
All bool
|
|
}
|
|
|
|
// Held judges v's missing artifacts against the registries' record (design 47 H H-A-269 (2), H-A-272, H-A-274, H-A-275): each
|
|
// artifact's registry seat and name as a publish again sends them, its record read through the seat's verb versions (versionsOf).
|
|
// A version with no missing artifact is All.
|
|
func (r *Republisher) Held(ctx context.Context, v ReservedVersion, versionsOf func(ctx context.Context, seat, name string) ([]HeldVersion, error)) (Judged, error) {
|
|
missing := v.Missing()
|
|
if len(missing) == 0 {
|
|
return Judged{All: true}, nil
|
|
}
|
|
e, err := r.Entry(ctx, v.Fingerprint)
|
|
if err != nil {
|
|
return Judged{}, fmt.Errorf("the catalog entry %s of %s %s cannot be read: %w", v.Fingerprint, v.Module, v.Version, err)
|
|
}
|
|
var out Judged
|
|
for _, ref := range missing {
|
|
seat, name, _, ok := publishTarget(e, v.Module, ref)
|
|
if !ok {
|
|
if out.Lacks == "" {
|
|
out.Lacks = ref + ": the catalog entry names no registry seat for it"
|
|
}
|
|
continue
|
|
}
|
|
held, err := versionsOf(ctx, seat, name)
|
|
if err != nil {
|
|
return Judged{}, fmt.Errorf("%s's versions of %s cannot be read: %w", seat, name, err)
|
|
}
|
|
at, found := HeldArtifact{}, false
|
|
for _, h := range held {
|
|
if strings.TrimPrefix(h.Version, "v") != v.Version {
|
|
continue
|
|
}
|
|
at, found = HeldArtifact{Ref: ref, Seat: seat, Name: name, Version: h.Version, Digest: h.Digest}, true
|
|
if h.Digest == v.Artifacts[ref] {
|
|
break
|
|
}
|
|
}
|
|
switch {
|
|
case !found:
|
|
if out.Lacks == "" {
|
|
out.Lacks = fmt.Sprintf("%s holds no %s %s", seat, name, v.Version)
|
|
}
|
|
case at.Digest == v.Artifacts[ref]:
|
|
out.Held = append(out.Held, at)
|
|
default:
|
|
out.Elsewhere = append(out.Elsewhere, at)
|
|
}
|
|
}
|
|
out.All = len(out.Held) == len(missing)
|
|
return out, nil
|
|
}
|
|
|
|
// HeldNumbers are the versions (MAJOR.MINOR.PATCH) a registry seat's record holds of any of module's artifacts refs, as the catalog
|
|
// entry of fingerprint names their seats and names (design 47 H H-A-275): a number one holds is never reserved. An artifact the
|
|
// entry names no registry seat for is read nowhere.
|
|
func (r *Republisher) HeldNumbers(ctx context.Context, module, fingerprint string, refs []string, versionsOf func(ctx context.Context, seat, name string) ([]HeldVersion, error)) ([]string, error) {
|
|
e, err := r.Entry(ctx, fingerprint)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the catalog entry %s of %s cannot be read: %w", fingerprint, module, err)
|
|
}
|
|
var out []string
|
|
for _, ref := range refs {
|
|
seat, name, _, ok := publishTarget(e, module, ref)
|
|
if !ok {
|
|
continue
|
|
}
|
|
held, err := versionsOf(ctx, seat, name)
|
|
if err != nil {
|
|
return out, fmt.Errorf("%s's versions of %s cannot be read: %w", seat, name, err)
|
|
}
|
|
for _, h := range held {
|
|
if v := strings.TrimPrefix(h.Version, "v"); v != "" && !slices.Contains(out, v) {
|
|
out = append(out, v)
|
|
}
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|