Live, a published at a version fourteen reservations share cost three seconds: the fallback by version read every reservation there, and the journal line read them all again, so the backlog drained at fifty a minute. The published now hands the reservations it read (each once, a second pass taking what the first read) to its journal line. The fallback reads the new index key of the reservations at a version whose build has not recorded every digest (kept at reserve and where a digest is recorded, written again at the start), the only ones a published can be kept on; the line counts the rest from the index by version without reading them. Before that pass the handler says in progress to the bus. Builds: design 47 H H-A-280, H-A-271
206 lines
6.8 KiB
Go
206 lines
6.8 KiB
Go
package versions
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"slices"
|
|
"sort"
|
|
)
|
|
|
|
// **The reservations' index by digest and by version** (design 47 H H-A-269 (1), H-A-271): beside the reservations, in the same
|
|
// state, the builder keeps which reservation keys hold a built digest and which stand at a version, so a registry seat's published
|
|
// finds its reservation in one read and a published whose digest no reservation holds is judged from its version's keys in one
|
|
// more, never a scan of every reservation. The index keys have three tokens, so the reservations' filter (two tokens) never lists
|
|
// them. It is maintained at reserve, at a digest recorded, at release and at drop; an entry left behind by a write that ended
|
|
// halfway is harmless (every read checks the reservation it names), and a missing one is written again by Reindex at the start.
|
|
|
|
const (
|
|
indexDigestPrefix = "index.digest."
|
|
indexVersionPrefix = "index.version."
|
|
indexUnbuiltPrefix = "index.unbuilt."
|
|
// IndexDigestFilter and IndexVersionFilter are the filters of the index's two halves.
|
|
IndexDigestFilter = indexDigestPrefix + "*"
|
|
IndexVersionFilter = indexVersionPrefix + "*"
|
|
// IndexUnbuiltFilter is the filter of the reservations at a version whose build has not recorded every digest (H-A-280).
|
|
IndexUnbuiltFilter = indexUnbuiltPrefix + "*"
|
|
)
|
|
|
|
// IndexDigestKey is the index key of the reservations holding digest as a built artifact's.
|
|
func IndexDigestKey(digest string) string { return indexDigestPrefix + EscapeToken(digest) }
|
|
|
|
// IndexVersionKey is the index key of the reservations standing at version (MAJOR.MINOR.PATCH).
|
|
func IndexVersionKey(version string) string { return indexVersionPrefix + EscapeToken(version) }
|
|
|
|
// IndexUnbuiltKey is the index key of the reservations at version whose build has not recorded every digest (design 47 H
|
|
// H-A-280): the only ones a published whose digest no reservation holds is kept on (A-415), kept apart from the built.
|
|
func IndexUnbuiltKey(version string) string { return indexUnbuiltPrefix + EscapeToken(version) }
|
|
|
|
// IndexEntry is an index key's value: the reservation keys, sorted.
|
|
type IndexEntry struct {
|
|
Keys []string `json:"keys"`
|
|
}
|
|
|
|
// indexed reads an index key's reservation keys.
|
|
func (r *Reserver) indexed(ctx context.Context, key string) ([]string, error) {
|
|
raw, _, found, err := r.Store.Get(ctx, key)
|
|
if err != nil || !found {
|
|
return nil, err
|
|
}
|
|
var e IndexEntry
|
|
if err := json.Unmarshal(raw, &e); err != nil {
|
|
return nil, fmt.Errorf("%s holds what is no index entry: %w", key, err)
|
|
}
|
|
return e.Keys, nil
|
|
}
|
|
|
|
// index adds (or, with add false, removes) reserved under the index key key, by compare-and-set; an entry left empty is deleted.
|
|
func (r *Reserver) index(ctx context.Context, key, reserved string, add bool) error {
|
|
for {
|
|
raw, rev, found, err := r.Store.Get(ctx, key)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var e IndexEntry
|
|
if found {
|
|
_ = json.Unmarshal(raw, &e)
|
|
}
|
|
has := slices.Contains(e.Keys, reserved)
|
|
if has == add {
|
|
return nil
|
|
}
|
|
if add {
|
|
e.Keys = append(e.Keys, reserved)
|
|
sort.Strings(e.Keys)
|
|
} else {
|
|
e.Keys = slices.DeleteFunc(e.Keys, func(k string) bool { return k == reserved })
|
|
}
|
|
if len(e.Keys) == 0 {
|
|
return r.Store.Delete(ctx, key)
|
|
}
|
|
out, _ := json.Marshal(e)
|
|
if found {
|
|
_, err = r.Store.Update(ctx, key, out, rev)
|
|
} else {
|
|
_, err = r.Store.Create(ctx, key, out)
|
|
}
|
|
if errors.Is(err, ErrConflict) {
|
|
continue
|
|
}
|
|
return err
|
|
}
|
|
}
|
|
|
|
// indexKeys are the index keys a reservation stands under: its version's and each built digest's.
|
|
func indexKeys(v ReservedVersion) []string {
|
|
out := []string{}
|
|
if v.Version != "" {
|
|
out = append(out, IndexVersionKey(v.Version))
|
|
if v.BuiltAt.IsZero() {
|
|
out = append(out, IndexUnbuiltKey(v.Version))
|
|
}
|
|
}
|
|
for _, d := range v.Artifacts {
|
|
if d != "" && !slices.Contains(out, IndexDigestKey(d)) {
|
|
out = append(out, IndexDigestKey(d))
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// reindex moves reservation key from the index keys of was (its value before, zero when there was none) to those of now (zero when
|
|
// it is gone).
|
|
func (r *Reserver) reindex(ctx context.Context, key string, was, now ReservedVersion) error {
|
|
add, drop := indexKeys(now), indexKeys(was)
|
|
var errs []error
|
|
for _, k := range add {
|
|
if err := r.index(ctx, k, key, true); err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
}
|
|
for _, k := range drop {
|
|
if !slices.Contains(add, k) {
|
|
if err := r.index(ctx, k, key, false); err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
}
|
|
}
|
|
if err := errors.Join(errs...); err != nil {
|
|
return fmt.Errorf("the index of %s is not written (Reindex writes it at the start): %w", key, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// IndexedAt are the keys of the reservations standing at version, by the index by version alone (H-A-280): no reservation read.
|
|
func (r *Reserver) IndexedAt(ctx context.Context, version string) ([]string, error) {
|
|
return r.indexed(ctx, IndexVersionKey(version))
|
|
}
|
|
|
|
// AtVersion are the reservations standing at version, read through the index by version (H-A-271): one read of the index and
|
|
// one of each reservation it names.
|
|
func (r *Reserver) AtVersion(ctx context.Context, version string) ([]ReservedVersion, error) {
|
|
keys, err := r.indexed(ctx, IndexVersionKey(version))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var out []ReservedVersion
|
|
for _, k := range keys {
|
|
cur, _, found, err := r.get(ctx, k)
|
|
if err != nil {
|
|
return out, err
|
|
}
|
|
if found && cur.Version == version {
|
|
out = append(out, cur)
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Reindex writes the index again from the reservations, once at the builder's start (H-A-269 (1), H-A-271): each reservation is
|
|
// added under its keys, and a reservation an index key names that does not stand under it (read again) is removed from it. Both
|
|
// are the same compare-and-set as every other index write, so a reservation written meanwhile keeps its entries. It answers how
|
|
// many reservations it read.
|
|
func (r *Reserver) Reindex(ctx context.Context) (int, error) {
|
|
all, err := r.All(ctx)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
var errs []error
|
|
for _, v := range all {
|
|
key := ReservedKey(v.Module, v.Series)
|
|
for _, k := range indexKeys(v) {
|
|
if err := r.index(ctx, k, key, true); err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
}
|
|
}
|
|
for _, filter := range []string{IndexDigestFilter, IndexVersionFilter, IndexUnbuiltFilter} {
|
|
keys, err := r.Store.KeysOf(ctx, filter)
|
|
if err != nil {
|
|
return len(all), fmt.Errorf("the index cannot be listed: %w", err)
|
|
}
|
|
for _, k := range keys {
|
|
named, err := r.indexed(ctx, k)
|
|
if err != nil {
|
|
errs = append(errs, err)
|
|
continue
|
|
}
|
|
for _, rk := range named {
|
|
cur, _, found, err := r.get(ctx, rk)
|
|
if err != nil {
|
|
errs = append(errs, err)
|
|
continue
|
|
}
|
|
if found && slices.Contains(indexKeys(cur), k) {
|
|
continue
|
|
}
|
|
if err := r.index(ctx, k, rk, false); err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return len(all), errors.Join(errs...)
|
|
}
|