Builder: a published reads each reservation once, and only the unbuilt ones on a miss
Live, a published at a version fourteen reservations share cost three seconds: the fallback by version read every reservation there, and the journal line read them all again, so the backlog drained at fifty a minute. The published now hands the reservations it read (each once, a second pass taking what the first read) to its journal line. The fallback reads the new index key of the reservations at a version whose build has not recorded every digest (kept at reserve and where a digest is recorded, written again at the start), the only ones a published can be kept on; the line counts the rest from the index by version without reading them. Before that pass the handler says in progress to the bus. Builds: design 47 H H-A-280, H-A-271
This commit is contained in:
@@ -288,19 +288,21 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// published marks the artifacts of the reservation a registry seat's published names (its version and digest).
|
||||
func (rv *reservations) published(ctx context.Context, o versions.Outcome) {
|
||||
marked, kept, err := rv.r.Published(ctx, o.Name, o.Version, o.Digest)
|
||||
// published marks the artifacts of the reservation a registry seat's published names (its version and digest). inProgress is told
|
||||
// before a pass that may read many reservations, so the handler says in progress to the bus (design 47 H H-A-280); nil is told
|
||||
// nothing.
|
||||
func (rv *reservations) published(ctx context.Context, o versions.Outcome, inProgress func()) {
|
||||
out, err := rv.r.PublishedRead(ctx, o.Name, o.Version, o.Digest, inProgress)
|
||||
if err != nil {
|
||||
rv.logf("a published of %s@%s is not recorded in the reservations: %v", o.Name, o.Version, err)
|
||||
}
|
||||
if len(kept) > 0 {
|
||||
rv.logf("the published of %s@%s (digest %s, holder %s) is kept on %s until the build records its digests", o.Name, o.Version, o.Digest, o.Holder, strings.Join(kept, ", "))
|
||||
} else if len(marked) == 0 {
|
||||
rv.logf("package-published not said: the published of %s@%s (digest %s, holder %s) marks no artifact of a reservation: %s", o.Name, o.Version, o.Digest, o.Holder, rv.noMark(ctx, o))
|
||||
if len(out.Kept) > 0 {
|
||||
rv.logf("the published of %s@%s (digest %s, holder %s) is kept on %s until the build records its digests", o.Name, o.Version, o.Digest, o.Holder, strings.Join(out.Kept, ", "))
|
||||
} else if len(out.Marked) == 0 {
|
||||
rv.logf("package-published not said: the published of %s@%s (digest %s, holder %s) marks no artifact of a reservation: %s", o.Name, o.Version, o.Digest, o.Holder, rv.noMark(ctx, o, out.Read))
|
||||
}
|
||||
// Each artifact the holder's published newly marked is said as the seat mesh-builds' package-published (design 47 G §52, §334).
|
||||
for _, m := range marked {
|
||||
for _, m := range out.Marked {
|
||||
rv.sayPublished(ctx, m, o.Digest)
|
||||
}
|
||||
}
|
||||
@@ -363,25 +365,29 @@ func (rv *reservations) tagged(ctx context.Context, ev meshcatalog.VersionReleas
|
||||
return nil
|
||||
}
|
||||
|
||||
// noMark says why a published marked nothing: which reservation holds that version, and what it holds of digests, read through the
|
||||
// index by version (H-A-269 (1), H-A-271).
|
||||
func (rv *reservations) noMark(ctx context.Context, o versions.Outcome) string {
|
||||
// noMark says why a published marked nothing, from the reservations the published read (design 47 H H-A-280: no second read):
|
||||
// what each holds of digests, and how many stand at that version by the index by version (its one key, no reservation read).
|
||||
func (rv *reservations) noMark(ctx context.Context, o versions.Outcome, read []versions.ReadReservation) string {
|
||||
version := strings.TrimPrefix(o.Version, "v")
|
||||
if version == "" || o.Digest == "" {
|
||||
return "it names no version or no digest (a by-digest publish)"
|
||||
}
|
||||
at, err := rv.r.AtVersion(ctx, version) // the index by version, never every reservation (H-A-271)
|
||||
if err != nil {
|
||||
return "the reservations at " + version + " cannot be read: " + err.Error()
|
||||
}
|
||||
var out []string
|
||||
for _, v := range at {
|
||||
out = append(out, fmt.Sprintf("%s holds %d built digests %v, published %v (a published heard before the build recorded its digests is kept until it does)", v.Module, len(v.Artifacts), v.Artifacts, v.Published))
|
||||
for _, r := range read {
|
||||
v := r.Reserved
|
||||
if v.Version != version {
|
||||
continue
|
||||
}
|
||||
out = append(out, fmt.Sprintf("%s holds %d built digests %v, published %v", v.Module, len(v.Artifacts), v.Artifacts, v.Published))
|
||||
}
|
||||
standing := "the number standing at it is not read"
|
||||
if keys, err := rv.r.IndexedAt(ctx, version); err == nil {
|
||||
standing = fmt.Sprintf("%d reservations stand at it, the built ones not read", len(keys))
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return "no reservation holds version " + version
|
||||
return "no reservation read holds version " + version + " unbuilt or with that digest (" + standing + ")"
|
||||
}
|
||||
return strings.Join(out, "; ")
|
||||
return strings.Join(out, "; ") + " (" + standing + "; a published heard before the build recorded its digests is kept until it does)"
|
||||
}
|
||||
|
||||
// reserveDoneAct is builder_reserve_done's act (design 47 G A-455): the next version of the series is reserved for a done trunk build
|
||||
@@ -414,7 +420,7 @@ func (rv *reservations) ReleasedFor(ctx context.Context, module, fingerprint str
|
||||
func (rv *reservations) markHeld(ctx context.Context, v versions.ReservedVersion, held []versions.HeldArtifact) {
|
||||
for _, h := range held {
|
||||
rv.logf("%s %s: %s holds %s %s with the built digest %s: marked published from its record", v.Module, v.Version, h.Seat, h.Name, h.Version, h.Digest)
|
||||
rv.published(ctx, versions.Outcome{Name: h.Name, Version: h.Version, Digest: h.Digest, Holder: h.Seat})
|
||||
rv.published(ctx, versions.Outcome{Name: h.Name, Version: h.Version, Digest: h.Digest, Holder: h.Seat}, nil)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -119,10 +119,10 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) {
|
||||
if !bounds.keys[boundKey] {
|
||||
t.Fatalf("the publish's bound is not armed: %v", bounds.keys)
|
||||
}
|
||||
rv.published(ctx, versions.Outcome{Name: "m-a", Version: "1.4.3", Digest: "sha256:a"})
|
||||
rv.published(ctx, versions.Outcome{Name: "m-a", Version: "1.4.3", Digest: "sha256:a"}, nil)
|
||||
// Design 47 G §52, A-57: the holder's published is said once, per artifact, as the seat mesh-builds' package-published with the
|
||||
// branch and source revision of the reservation; a second published of the same artifact says nothing.
|
||||
rv.published(ctx, versions.Outcome{Name: "m-a", Version: "1.4.3", Digest: "sha256:a"})
|
||||
rv.published(ctx, versions.Outcome{Name: "m-a", Version: "1.4.3", Digest: "sha256:a"}, nil)
|
||||
if len(said) != 1 || said[0].Artifact != "m:a" || said[0].Version != "1.4.3" || said[0].Digest != "sha256:a" || said[0].Branch != "main" || said[0].Source != 41 {
|
||||
t.Fatalf("package-published said %+v, want one of m:a at 1.4.3 for main at source 41", said)
|
||||
}
|
||||
@@ -236,8 +236,8 @@ func TestAPublishedHeardBeforeTheDigestsIsSaidWhenTheBuildRecordsThem(t *testing
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rv.published(ctx, versions.Outcome{Name: "m-a", Version: res.Version, Digest: "sha256:a"})
|
||||
rv.published(ctx, versions.Outcome{Name: "m-x", Version: res.Version, Digest: "sha256:x"})
|
||||
rv.published(ctx, versions.Outcome{Name: "m-a", Version: res.Version, Digest: "sha256:a"}, nil)
|
||||
rv.published(ctx, versions.Outcome{Name: "m-x", Version: res.Version, Digest: "sha256:x"}, nil)
|
||||
if len(said) != 0 {
|
||||
t.Fatalf("package-published said before the digests are recorded: %+v", said)
|
||||
}
|
||||
@@ -259,7 +259,7 @@ func TestAPublishedHeardBeforeTheDigestsIsSaidWhenTheBuildRecordsThem(t *testing
|
||||
t.Fatalf("no finding raised for the digest the holder holds: %s", raised)
|
||||
}
|
||||
// Taken in once: a published of m:a heard again says nothing more.
|
||||
rv.published(ctx, versions.Outcome{Name: "m-a", Version: res.Version, Digest: "sha256:a"})
|
||||
rv.published(ctx, versions.Outcome{Name: "m-a", Version: res.Version, Digest: "sha256:a"}, nil)
|
||||
if len(said) != 1 {
|
||||
t.Fatalf("package-published said again: %+v", said)
|
||||
}
|
||||
@@ -450,7 +450,7 @@ func TestARegistrysRecordEndsOrMovesAPublish(t *testing.T) {
|
||||
if err := rv.Built(ctx, module, again.Version, map[string]string{"node-tools:program": "sha256:fresh"}, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
rv.published(ctx, versions.Outcome{Name: "node-tools/program", Version: again.Version, Digest: "sha256:fresh"})
|
||||
rv.published(ctx, versions.Outcome{Name: "node-tools/program", Version: again.Version, Digest: "sha256:fresh"}, nil)
|
||||
return nil
|
||||
}
|
||||
cur, _, _ = rv.r.Of(ctx, "node-tools", "0.1.1")
|
||||
@@ -477,3 +477,58 @@ func TestARegistrysRecordEndsOrMovesAPublish(t *testing.T) {
|
||||
t.Errorf("no version-held-elsewhere naming both digests: %s", raised)
|
||||
}
|
||||
}
|
||||
|
||||
// countedReserve counts the reads of reservation keys (two tokens).
|
||||
type countedReserve struct {
|
||||
*memReserve
|
||||
reads int
|
||||
}
|
||||
|
||||
func (c *countedReserve) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
if !strings.HasPrefix(key, "index.") {
|
||||
c.reads++
|
||||
}
|
||||
return c.memReserve.Get(ctx, key)
|
||||
}
|
||||
|
||||
// TestAPublishedThatMarksNothingIsSaidFromWhatItRead: design 47 H H-A-280 — a published whose digest no reservation holds, at a
|
||||
// version many reservations share, reads only the unbuilt ones there, each once; its journal line is made from those reads with
|
||||
// no second read; and the handler is told to say in progress before that pass.
|
||||
func TestAPublishedThatMarksNothingIsSaidFromWhatItRead(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := &countedReserve{memReserve: &memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}}
|
||||
none := func(string, any) (json.RawMessage, error) { return json.Marshal(meshcatalog.VersionsAnswer{}) }
|
||||
var logs []string
|
||||
rv := newReservations(store, none, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} },
|
||||
&findings{module: builds.BuilderModule, w: &memFindings{vals: map[string][]byte{}}, logf: t.Logf, now: time.Now}, nil,
|
||||
func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) })
|
||||
rv.filled.Store(true)
|
||||
for i := range 14 {
|
||||
m := fmt.Sprintf("m%02d", i)
|
||||
res, err := rv.Reserve(ctx, m, "0.1", []string{m + ":a"}, versions.Source{Fingerprint: "fp" + m})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if i > 0 { // one stays unbuilt
|
||||
if err := rv.Built(ctx, m, res.Version, map[string]string{m + ":a": "sha256:" + m}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
store.reads = 0
|
||||
told := 0
|
||||
rv.published(ctx, versions.Outcome{Name: "x", Version: "0.1.0", Digest: "sha256:none"}, func() { told++ })
|
||||
if store.reads != 1 || told != 1 {
|
||||
t.Errorf("%d reservation reads, in progress told %d times: want the one unbuilt read once, and in progress said before", store.reads, told)
|
||||
}
|
||||
if line := strings.Join(logs, "\n"); !strings.Contains(line, "kept on") {
|
||||
t.Errorf("the journal says %q, want the published kept on the unbuilt reservation", line)
|
||||
}
|
||||
logs, store.reads = nil, 0
|
||||
// A digest of a built one whose artifact is published already marks nothing: the line names it from the one read.
|
||||
rv.published(ctx, versions.Outcome{Name: "m05-a", Version: "0.1.0", Digest: "sha256:m05"}, nil)
|
||||
rv.published(ctx, versions.Outcome{Name: "m05-a", Version: "0.1.0", Digest: "sha256:m05"}, nil)
|
||||
if store.reads != 2 || !strings.Contains(strings.Join(logs, "\n"), "m05 holds 1 built digests") || !strings.Contains(strings.Join(logs, "\n"), "14 reservations stand at it") {
|
||||
t.Errorf("%d reads for two publisheds, journal %q: want one read each and the line made from it", store.reads, logs)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,7 +103,12 @@ func (h *seatHearer) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
h.pub.heard(ctx, se.seat, se.event, o)
|
||||
}
|
||||
if h.res != nil && se.event == versions.HeardPublished {
|
||||
h.res.published(ctx, o)
|
||||
// A pass over many reservations may outlast a third of the ack wait: the bus is told in progress before it (H-A-280).
|
||||
h.res.published(ctx, o, func() {
|
||||
if err := m.InProgress(); err != nil {
|
||||
h.logf("in progress of %s not said: %v", m.Subject(), err)
|
||||
}
|
||||
})
|
||||
}
|
||||
if se.event == versions.HeardPublished && o.Kind == versions.KindPackage && h.b != nil {
|
||||
// A package registry seat's published pins a package an entry names with none, at the version it carries (H-A-192).
|
||||
|
||||
@@ -19,9 +19,12 @@ import (
|
||||
const (
|
||||
indexDigestPrefix = "index.digest."
|
||||
indexVersionPrefix = "index.version."
|
||||
indexUnbuiltPrefix = "index.unbuilt."
|
||||
// IndexDigestFilter and IndexVersionFilter are the filters of the index's two halves.
|
||||
IndexDigestFilter = indexDigestPrefix + "*"
|
||||
IndexVersionFilter = indexVersionPrefix + "*"
|
||||
// IndexUnbuiltFilter is the filter of the reservations at a version whose build has not recorded every digest (H-A-280).
|
||||
IndexUnbuiltFilter = indexUnbuiltPrefix + "*"
|
||||
)
|
||||
|
||||
// IndexDigestKey is the index key of the reservations holding digest as a built artifact's.
|
||||
@@ -30,6 +33,10 @@ func IndexDigestKey(digest string) string { return indexDigestPrefix + EscapeTok
|
||||
// IndexVersionKey is the index key of the reservations standing at version (MAJOR.MINOR.PATCH).
|
||||
func IndexVersionKey(version string) string { return indexVersionPrefix + EscapeToken(version) }
|
||||
|
||||
// IndexUnbuiltKey is the index key of the reservations at version whose build has not recorded every digest (design 47 H
|
||||
// H-A-280): the only ones a published whose digest no reservation holds is kept on (A-415), kept apart from the built.
|
||||
func IndexUnbuiltKey(version string) string { return indexUnbuiltPrefix + EscapeToken(version) }
|
||||
|
||||
// IndexEntry is an index key's value: the reservation keys, sorted.
|
||||
type IndexEntry struct {
|
||||
Keys []string `json:"keys"`
|
||||
@@ -90,6 +97,9 @@ func indexKeys(v ReservedVersion) []string {
|
||||
out := []string{}
|
||||
if v.Version != "" {
|
||||
out = append(out, IndexVersionKey(v.Version))
|
||||
if v.BuiltAt.IsZero() {
|
||||
out = append(out, IndexUnbuiltKey(v.Version))
|
||||
}
|
||||
}
|
||||
for _, d := range v.Artifacts {
|
||||
if d != "" && !slices.Contains(out, IndexDigestKey(d)) {
|
||||
@@ -122,6 +132,11 @@ func (r *Reserver) reindex(ctx context.Context, key string, was, now ReservedVer
|
||||
return nil
|
||||
}
|
||||
|
||||
// IndexedAt are the keys of the reservations standing at version, by the index by version alone (H-A-280): no reservation read.
|
||||
func (r *Reserver) IndexedAt(ctx context.Context, version string) ([]string, error) {
|
||||
return r.indexed(ctx, IndexVersionKey(version))
|
||||
}
|
||||
|
||||
// AtVersion are the reservations standing at version, read through the index by version (H-A-271): one read of the index and
|
||||
// one of each reservation it names.
|
||||
func (r *Reserver) AtVersion(ctx context.Context, version string) ([]ReservedVersion, error) {
|
||||
@@ -160,7 +175,7 @@ func (r *Reserver) Reindex(ctx context.Context) (int, error) {
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, filter := range []string{IndexDigestFilter, IndexVersionFilter} {
|
||||
for _, filter := range []string{IndexDigestFilter, IndexVersionFilter, IndexUnbuiltFilter} {
|
||||
keys, err := r.Store.KeysOf(ctx, filter)
|
||||
if err != nil {
|
||||
return len(all), fmt.Errorf("the index cannot be listed: %w", err)
|
||||
|
||||
@@ -62,21 +62,37 @@ func TestTheIndexFindsAPublishedReservationInOneRead(t *testing.T) {
|
||||
t.Errorf("the published listed %d times and read %d reservations, want none listed and one read", l, n)
|
||||
}
|
||||
})
|
||||
t.Run("a digest no reservation holds is judged from its version's keys", func(t *testing.T) {
|
||||
t.Run("a digest no reservation holds is judged from its version's unbuilt keys", func(t *testing.T) {
|
||||
r, store := fill(t)
|
||||
marked, kept, err := r.Published(ctx, "m11-a", "0.1.0", "sha256:early")
|
||||
if err != nil || len(marked) != 0 {
|
||||
t.Fatalf("an unbuilt digest marked %v (%v)", marked, err)
|
||||
}
|
||||
// The 20 unbuilt reservations at 0.1.0 keep it as heard (A-415); the built ones are read and left.
|
||||
if len(kept) != 20 || store.lists.Load() != 0 || store.reads.Load() != 40 {
|
||||
t.Errorf("kept on %d (want 20), listed %d (want 0), read %d (want the 40 at 0.1.0)", len(kept), store.lists.Load(), store.reads.Load())
|
||||
// The 20 unbuilt reservations at 0.1.0 keep it as heard (A-415); the 20 built ones are not read at all (H-A-280).
|
||||
if len(kept) != 20 || store.lists.Load() != 0 || store.reads.Load() != 20 {
|
||||
t.Errorf("kept on %d (want 20), listed %d (want 0), read %d (want only the 20 unbuilt at 0.1.0)", len(kept), store.lists.Load(), store.reads.Load())
|
||||
}
|
||||
at, err := r.AtVersion(ctx, "0.9.9")
|
||||
if err != nil || len(at) != 0 || store.lists.Load() != 0 {
|
||||
t.Errorf("a version no reservation stands at answered %v (%v), listed %d", at, err, store.lists.Load())
|
||||
}
|
||||
})
|
||||
t.Run("a published at a version many share reads each reservation once and hands them on", func(t *testing.T) {
|
||||
r, store := fill(t)
|
||||
// A stale digest entry naming unbuilt reservations too: the second pass takes them as the first read them.
|
||||
stale, _ := json.Marshal(IndexEntry{Keys: []string{ReservedKey("m01", "0.1"), ReservedKey("m03", "0.1")}})
|
||||
if _, err := store.Create(ctx, IndexDigestKey("sha256:stale"), stale); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
slow := 0
|
||||
out, err := r.PublishedRead(ctx, "x", "0.1.0", "sha256:stale", func() { slow++ })
|
||||
if err != nil || len(out.Kept) != 20 || len(out.Read) != 20 || slow != 1 {
|
||||
t.Fatalf("kept %d, read %d, told slow %d (%v): want 20 kept, 20 read and handed on, slow told once", len(out.Kept), len(out.Read), slow, err)
|
||||
}
|
||||
if n := store.reads.Load(); n != 20 {
|
||||
t.Errorf("%d reservation reads, want each of the 20 read once", n)
|
||||
}
|
||||
})
|
||||
t.Run("a release and a drop take the reservation out of the index", func(t *testing.T) {
|
||||
r, store := fill(t)
|
||||
if _, err := r.Released(ctx, "m10", "0.1.0"); err != nil {
|
||||
|
||||
@@ -601,88 +601,138 @@ func (m Marked) PackagePublished(digest string, at time.Time) meshbuilds.Package
|
||||
Version: m.Reserved.Version, Digest: digest, Dependencies: m.Reserved.Dependencies[m.Artifact]}
|
||||
}
|
||||
|
||||
// Published takes a registry seat's published in: the artifacts of a reservation at version (a leading v is the Go form)
|
||||
// whose built digest it names are marked published. It answers each artifact it newly marked, with the reservation as marked.
|
||||
// A published that marks no artifact of a reservation at version whose build has not recorded every digest is kept on that
|
||||
// reservation as heard (name, version, digest), and the keys it is kept on are answered too (design 47 G A-415): Kept marks
|
||||
// it when the build records its digests. The reservation is found through the index (H-A-269 (1), H-A-271): the reservations
|
||||
// holding the digest in one read, and only when none holds it at version, those standing at the version in one more; never a
|
||||
// scan of every reservation.
|
||||
// Published takes a registry seat's published in, as PublishedRead does, answering the artifacts newly marked and the keys it is
|
||||
// kept on.
|
||||
func (r *Reserver) Published(ctx context.Context, name, version, digest string) ([]Marked, []string, error) {
|
||||
out, err := r.PublishedRead(ctx, name, version, digest, nil)
|
||||
return out.Marked, out.Kept, err
|
||||
}
|
||||
|
||||
// ReadReservation is a reservation a published read, as it stood after the published was taken in, and whether one of its built
|
||||
// digests is the published's (design 47 H H-A-280).
|
||||
type ReadReservation struct {
|
||||
Key string
|
||||
Reserved ReservedVersion
|
||||
Matched bool
|
||||
}
|
||||
|
||||
// PublishedOutcome is what a published did: the artifacts it newly marked, the keys it is kept on (A-415), and every reservation it
|
||||
// read, each once, which the journal line is made from (H-A-280).
|
||||
type PublishedOutcome struct {
|
||||
Marked []Marked
|
||||
Kept []string
|
||||
Read []ReadReservation
|
||||
}
|
||||
|
||||
// PublishedRead takes a registry seat's published in: the artifacts of a reservation at version (a leading v is the Go form)
|
||||
// whose built digest it names are marked published. A published that marks no artifact of a reservation at version whose build
|
||||
// has not recorded every digest is kept on that reservation as heard (name, version, digest), and the keys it is kept on are
|
||||
// answered (design 47 G A-415): Kept marks it when the build records its digests. The reservation is found through the index
|
||||
// (H-A-269 (1), H-A-271, H-A-280): the reservations holding the digest in one read, and only when none at version holds it, those
|
||||
// at the version whose build has not recorded every digest in one more; never a scan of every reservation, and each reservation
|
||||
// read at most once (again only on a write that met another). slow is told before the second pass, which may read many, so the
|
||||
// caller can say in progress to the bus; nil is told nothing.
|
||||
func (r *Reserver) PublishedRead(ctx context.Context, name, version, digest string, slow func()) (PublishedOutcome, error) {
|
||||
version = strings.TrimPrefix(version, "v")
|
||||
var out PublishedOutcome
|
||||
if version == "" || digest == "" {
|
||||
return nil, nil, nil
|
||||
return out, nil
|
||||
}
|
||||
keys, err := r.indexed(ctx, IndexDigestKey(digest))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the index of digest %s cannot be read: %w", digest, err)
|
||||
return out, fmt.Errorf("the index of digest %s cannot be read: %w", digest, err)
|
||||
}
|
||||
marked, kept, matched, err := r.publishedOn(ctx, keys, name, version, digest, false)
|
||||
read := map[string]readOne{}
|
||||
matched, err := r.publishedOn(ctx, keys, name, version, digest, false, read, &out)
|
||||
if err != nil || matched {
|
||||
return marked, kept, err
|
||||
return out, err
|
||||
}
|
||||
keys, err = r.indexed(ctx, IndexVersionKey(version))
|
||||
keys, err = r.indexed(ctx, IndexUnbuiltKey(version))
|
||||
if err != nil {
|
||||
return marked, kept, fmt.Errorf("the index of version %s cannot be read: %w", version, err)
|
||||
return out, fmt.Errorf("the index of the unbuilt at version %s cannot be read: %w", version, err)
|
||||
}
|
||||
m, k, _, err := r.publishedOn(ctx, keys, name, version, digest, true)
|
||||
return append(marked, m...), append(kept, k...), err
|
||||
if len(keys) > 0 && slow != nil {
|
||||
slow()
|
||||
}
|
||||
_, err = r.publishedOn(ctx, keys, name, version, digest, true, read, &out)
|
||||
return out, err
|
||||
}
|
||||
|
||||
// readOne is a reservation as one pass read it, kept so the next pass reads it no more.
|
||||
type readOne struct {
|
||||
v ReservedVersion
|
||||
rev uint64
|
||||
ok bool
|
||||
}
|
||||
|
||||
// publishedOn takes a published in on the reservations under keys: an artifact whose built digest it names is marked; with keep,
|
||||
// a reservation at version whose build has not recorded every digest and that it matches nothing of keeps it (A-415). It also
|
||||
// answers whether a reservation at version holds the digest.
|
||||
func (r *Reserver) publishedOn(ctx context.Context, keys []string, name, version, digest string, keep bool) ([]Marked, []string, bool, error) {
|
||||
var marked []Marked
|
||||
var kept []string
|
||||
// a reservation at version whose build has not recorded every digest and that it matches nothing of keeps it (A-415). A
|
||||
// reservation read by an earlier pass (read) is taken as read; each read is added to out.Read once. It answers whether a
|
||||
// reservation at version holds the digest.
|
||||
func (r *Reserver) publishedOn(ctx context.Context, keys []string, name, version, digest string, keep bool, read map[string]readOne, out *PublishedOutcome) (bool, error) {
|
||||
anyMatched := false
|
||||
for _, k := range keys {
|
||||
again:
|
||||
cur, rev, found, err := r.get(ctx, k)
|
||||
if err != nil {
|
||||
return marked, kept, anyMatched, err
|
||||
}
|
||||
if !found || cur.Version != version {
|
||||
continue
|
||||
}
|
||||
changed, matched, keepIt := false, false, false
|
||||
var arts []string
|
||||
for a, d := range cur.Artifacts {
|
||||
if d != digest {
|
||||
continue
|
||||
}
|
||||
matched = true
|
||||
if !cur.Published[a] {
|
||||
if cur.Published == nil {
|
||||
cur.Published = map[string]bool{}
|
||||
one, seen := read[k]
|
||||
for {
|
||||
if !seen {
|
||||
cur, rev, found, err := r.get(ctx, k)
|
||||
if err != nil {
|
||||
return anyMatched, err
|
||||
}
|
||||
cur.Published[a], changed = true, true
|
||||
arts = append(arts, a)
|
||||
one = readOne{v: cur, rev: rev, ok: found}
|
||||
}
|
||||
}
|
||||
anyMatched = anyMatched || matched
|
||||
sort.Strings(arts)
|
||||
if keep && !matched && cur.BuiltAt.IsZero() && !cur.keeps(name, digest) {
|
||||
cur.Kept = append(cur.Kept, KeptPublished{Name: name, Version: version, Digest: digest})
|
||||
changed, keepIt = true, true
|
||||
}
|
||||
if !changed {
|
||||
continue
|
||||
}
|
||||
raw, _ := json.Marshal(cur)
|
||||
if _, err := r.Store.Update(ctx, k, raw, rev); errors.Is(err, ErrConflict) {
|
||||
goto again
|
||||
} else if err != nil {
|
||||
return marked, kept, anyMatched, fmt.Errorf("%s cannot be written: %w", k, err)
|
||||
}
|
||||
for _, a := range arts {
|
||||
marked = append(marked, Marked{Reserved: cur, Artifact: a})
|
||||
}
|
||||
if keepIt {
|
||||
kept = append(kept, k)
|
||||
seen = false // a write that meets another reads again
|
||||
cur := one.v
|
||||
if !one.ok || cur.Version != version {
|
||||
read[k] = one
|
||||
break
|
||||
}
|
||||
changed, matched, keepIt := false, false, false
|
||||
var arts []string
|
||||
for a, d := range cur.Artifacts {
|
||||
if d != digest {
|
||||
continue
|
||||
}
|
||||
matched = true
|
||||
if !cur.Published[a] {
|
||||
cur.Published = maps.Clone(cur.Published)
|
||||
if cur.Published == nil {
|
||||
cur.Published = map[string]bool{}
|
||||
}
|
||||
cur.Published[a], changed = true, true
|
||||
arts = append(arts, a)
|
||||
}
|
||||
}
|
||||
sort.Strings(arts)
|
||||
if keep && !matched && cur.BuiltAt.IsZero() && !cur.keeps(name, digest) {
|
||||
cur.Kept = append(slices.Clone(cur.Kept), KeptPublished{Name: name, Version: version, Digest: digest})
|
||||
changed, keepIt = true, true
|
||||
}
|
||||
rev := one.rev
|
||||
if changed {
|
||||
raw, _ := json.Marshal(cur)
|
||||
var err error
|
||||
if rev, err = r.Store.Update(ctx, k, raw, one.rev); errors.Is(err, ErrConflict) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return anyMatched, fmt.Errorf("%s cannot be written: %w", k, err)
|
||||
}
|
||||
for _, a := range arts {
|
||||
out.Marked = append(out.Marked, Marked{Reserved: cur, Artifact: a})
|
||||
}
|
||||
if keepIt {
|
||||
out.Kept = append(out.Kept, k)
|
||||
}
|
||||
}
|
||||
anyMatched = anyMatched || matched
|
||||
if _, was := read[k]; !was {
|
||||
out.Read = append(out.Read, ReadReservation{Key: k, Reserved: cur, Matched: matched})
|
||||
}
|
||||
read[k] = readOne{v: cur, rev: rev, ok: true}
|
||||
break
|
||||
}
|
||||
}
|
||||
return marked, kept, anyMatched, nil
|
||||
return anyMatched, nil
|
||||
}
|
||||
|
||||
func (v ReservedVersion) keeps(name, digest string) bool {
|
||||
|
||||
Reference in New Issue
Block a user