From ece9d6b18fc3b31066828bde5e6b63431d429dfb Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 16:45:25 +0100 Subject: [PATCH] Builder: a published reads each reservation once, and only the unbuilt ones on a miss Live, a published at a version fourteen reservations share cost three seconds: the fallback by version read every reservation there, and the journal line read them all again, so the backlog drained at fifty a minute. The published now hands the reservations it read (each once, a second pass taking what the first read) to its journal line. The fallback reads the new index key of the reservations at a version whose build has not recorded every digest (kept at reserve and where a digest is recorded, written again at the start), the only ones a published can be kept on; the line counts the rest from the index by version without reading them. Before that pass the handler says in progress to the bus. Builds: design 47 H H-A-280, H-A-271 --- .../cmd/mesh-delivery-builder/reserve.go | 46 +++-- .../cmd/mesh-delivery-builder/reserve_test.go | 67 ++++++- .../cmd/mesh-delivery-builder/seats.go | 7 +- .../internal/versions/index.go | 17 +- .../internal/versions/index_test.go | 24 ++- .../internal/versions/reserve.go | 174 +++++++++++------- 6 files changed, 241 insertions(+), 94 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go index 1e3bbd28..e062050a 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go @@ -288,19 +288,21 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool { return true } -// published marks the artifacts of the reservation a registry seat's published names (its version and digest). -func (rv *reservations) published(ctx context.Context, o versions.Outcome) { - marked, kept, err := rv.r.Published(ctx, o.Name, o.Version, o.Digest) +// published marks the artifacts of the reservation a registry seat's published names (its version and digest). inProgress is told +// before a pass that may read many reservations, so the handler says in progress to the bus (design 47 H H-A-280); nil is told +// nothing. +func (rv *reservations) published(ctx context.Context, o versions.Outcome, inProgress func()) { + out, err := rv.r.PublishedRead(ctx, o.Name, o.Version, o.Digest, inProgress) if err != nil { rv.logf("a published of %s@%s is not recorded in the reservations: %v", o.Name, o.Version, err) } - if len(kept) > 0 { - rv.logf("the published of %s@%s (digest %s, holder %s) is kept on %s until the build records its digests", o.Name, o.Version, o.Digest, o.Holder, strings.Join(kept, ", ")) - } else if len(marked) == 0 { - rv.logf("package-published not said: the published of %s@%s (digest %s, holder %s) marks no artifact of a reservation: %s", o.Name, o.Version, o.Digest, o.Holder, rv.noMark(ctx, o)) + if len(out.Kept) > 0 { + rv.logf("the published of %s@%s (digest %s, holder %s) is kept on %s until the build records its digests", o.Name, o.Version, o.Digest, o.Holder, strings.Join(out.Kept, ", ")) + } else if len(out.Marked) == 0 { + rv.logf("package-published not said: the published of %s@%s (digest %s, holder %s) marks no artifact of a reservation: %s", o.Name, o.Version, o.Digest, o.Holder, rv.noMark(ctx, o, out.Read)) } // Each artifact the holder's published newly marked is said as the seat mesh-builds' package-published (design 47 G §52, §334). - for _, m := range marked { + for _, m := range out.Marked { rv.sayPublished(ctx, m, o.Digest) } } @@ -363,25 +365,29 @@ func (rv *reservations) tagged(ctx context.Context, ev meshcatalog.VersionReleas return nil } -// noMark says why a published marked nothing: which reservation holds that version, and what it holds of digests, read through the -// index by version (H-A-269 (1), H-A-271). -func (rv *reservations) noMark(ctx context.Context, o versions.Outcome) string { +// noMark says why a published marked nothing, from the reservations the published read (design 47 H H-A-280: no second read): +// what each holds of digests, and how many stand at that version by the index by version (its one key, no reservation read). +func (rv *reservations) noMark(ctx context.Context, o versions.Outcome, read []versions.ReadReservation) string { version := strings.TrimPrefix(o.Version, "v") if version == "" || o.Digest == "" { return "it names no version or no digest (a by-digest publish)" } - at, err := rv.r.AtVersion(ctx, version) // the index by version, never every reservation (H-A-271) - if err != nil { - return "the reservations at " + version + " cannot be read: " + err.Error() - } var out []string - for _, v := range at { - out = append(out, fmt.Sprintf("%s holds %d built digests %v, published %v (a published heard before the build recorded its digests is kept until it does)", v.Module, len(v.Artifacts), v.Artifacts, v.Published)) + for _, r := range read { + v := r.Reserved + if v.Version != version { + continue + } + out = append(out, fmt.Sprintf("%s holds %d built digests %v, published %v", v.Module, len(v.Artifacts), v.Artifacts, v.Published)) + } + standing := "the number standing at it is not read" + if keys, err := rv.r.IndexedAt(ctx, version); err == nil { + standing = fmt.Sprintf("%d reservations stand at it, the built ones not read", len(keys)) } if len(out) == 0 { - return "no reservation holds version " + version + return "no reservation read holds version " + version + " unbuilt or with that digest (" + standing + ")" } - return strings.Join(out, "; ") + return strings.Join(out, "; ") + " (" + standing + "; a published heard before the build recorded its digests is kept until it does)" } // reserveDoneAct is builder_reserve_done's act (design 47 G A-455): the next version of the series is reserved for a done trunk build @@ -414,7 +420,7 @@ func (rv *reservations) ReleasedFor(ctx context.Context, module, fingerprint str func (rv *reservations) markHeld(ctx context.Context, v versions.ReservedVersion, held []versions.HeldArtifact) { for _, h := range held { rv.logf("%s %s: %s holds %s %s with the built digest %s: marked published from its record", v.Module, v.Version, h.Seat, h.Name, h.Version, h.Digest) - rv.published(ctx, versions.Outcome{Name: h.Name, Version: h.Version, Digest: h.Digest, Holder: h.Seat}) + rv.published(ctx, versions.Outcome{Name: h.Name, Version: h.Version, Digest: h.Digest, Holder: h.Seat}, nil) } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go index 84beb089..94058086 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go @@ -119,10 +119,10 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) { if !bounds.keys[boundKey] { t.Fatalf("the publish's bound is not armed: %v", bounds.keys) } - rv.published(ctx, versions.Outcome{Name: "m-a", Version: "1.4.3", Digest: "sha256:a"}) + rv.published(ctx, versions.Outcome{Name: "m-a", Version: "1.4.3", Digest: "sha256:a"}, nil) // Design 47 G §52, A-57: the holder's published is said once, per artifact, as the seat mesh-builds' package-published with the // branch and source revision of the reservation; a second published of the same artifact says nothing. - rv.published(ctx, versions.Outcome{Name: "m-a", Version: "1.4.3", Digest: "sha256:a"}) + rv.published(ctx, versions.Outcome{Name: "m-a", Version: "1.4.3", Digest: "sha256:a"}, nil) if len(said) != 1 || said[0].Artifact != "m:a" || said[0].Version != "1.4.3" || said[0].Digest != "sha256:a" || said[0].Branch != "main" || said[0].Source != 41 { t.Fatalf("package-published said %+v, want one of m:a at 1.4.3 for main at source 41", said) } @@ -236,8 +236,8 @@ func TestAPublishedHeardBeforeTheDigestsIsSaidWhenTheBuildRecordsThem(t *testing if err != nil { t.Fatal(err) } - rv.published(ctx, versions.Outcome{Name: "m-a", Version: res.Version, Digest: "sha256:a"}) - rv.published(ctx, versions.Outcome{Name: "m-x", Version: res.Version, Digest: "sha256:x"}) + rv.published(ctx, versions.Outcome{Name: "m-a", Version: res.Version, Digest: "sha256:a"}, nil) + rv.published(ctx, versions.Outcome{Name: "m-x", Version: res.Version, Digest: "sha256:x"}, nil) if len(said) != 0 { t.Fatalf("package-published said before the digests are recorded: %+v", said) } @@ -259,7 +259,7 @@ func TestAPublishedHeardBeforeTheDigestsIsSaidWhenTheBuildRecordsThem(t *testing t.Fatalf("no finding raised for the digest the holder holds: %s", raised) } // Taken in once: a published of m:a heard again says nothing more. - rv.published(ctx, versions.Outcome{Name: "m-a", Version: res.Version, Digest: "sha256:a"}) + rv.published(ctx, versions.Outcome{Name: "m-a", Version: res.Version, Digest: "sha256:a"}, nil) if len(said) != 1 { t.Fatalf("package-published said again: %+v", said) } @@ -450,7 +450,7 @@ func TestARegistrysRecordEndsOrMovesAPublish(t *testing.T) { if err := rv.Built(ctx, module, again.Version, map[string]string{"node-tools:program": "sha256:fresh"}, nil); err != nil { return err } - rv.published(ctx, versions.Outcome{Name: "node-tools/program", Version: again.Version, Digest: "sha256:fresh"}) + rv.published(ctx, versions.Outcome{Name: "node-tools/program", Version: again.Version, Digest: "sha256:fresh"}, nil) return nil } cur, _, _ = rv.r.Of(ctx, "node-tools", "0.1.1") @@ -477,3 +477,58 @@ func TestARegistrysRecordEndsOrMovesAPublish(t *testing.T) { t.Errorf("no version-held-elsewhere naming both digests: %s", raised) } } + +// countedReserve counts the reads of reservation keys (two tokens). +type countedReserve struct { + *memReserve + reads int +} + +func (c *countedReserve) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) { + if !strings.HasPrefix(key, "index.") { + c.reads++ + } + return c.memReserve.Get(ctx, key) +} + +// TestAPublishedThatMarksNothingIsSaidFromWhatItRead: design 47 H H-A-280 — a published whose digest no reservation holds, at a +// version many reservations share, reads only the unbuilt ones there, each once; its journal line is made from those reads with +// no second read; and the handler is told to say in progress before that pass. +func TestAPublishedThatMarksNothingIsSaidFromWhatItRead(t *testing.T) { + ctx := context.Background() + store := &countedReserve{memReserve: &memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}} + none := func(string, any) (json.RawMessage, error) { return json.Marshal(meshcatalog.VersionsAnswer{}) } + var logs []string + rv := newReservations(store, none, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, + &findings{module: builds.BuilderModule, w: &memFindings{vals: map[string][]byte{}}, logf: t.Logf, now: time.Now}, nil, + func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }) + rv.filled.Store(true) + for i := range 14 { + m := fmt.Sprintf("m%02d", i) + res, err := rv.Reserve(ctx, m, "0.1", []string{m + ":a"}, versions.Source{Fingerprint: "fp" + m}) + if err != nil { + t.Fatal(err) + } + if i > 0 { // one stays unbuilt + if err := rv.Built(ctx, m, res.Version, map[string]string{m + ":a": "sha256:" + m}, nil); err != nil { + t.Fatal(err) + } + } + } + store.reads = 0 + told := 0 + rv.published(ctx, versions.Outcome{Name: "x", Version: "0.1.0", Digest: "sha256:none"}, func() { told++ }) + if store.reads != 1 || told != 1 { + t.Errorf("%d reservation reads, in progress told %d times: want the one unbuilt read once, and in progress said before", store.reads, told) + } + if line := strings.Join(logs, "\n"); !strings.Contains(line, "kept on") { + t.Errorf("the journal says %q, want the published kept on the unbuilt reservation", line) + } + logs, store.reads = nil, 0 + // A digest of a built one whose artifact is published already marks nothing: the line names it from the one read. + rv.published(ctx, versions.Outcome{Name: "m05-a", Version: "0.1.0", Digest: "sha256:m05"}, nil) + rv.published(ctx, versions.Outcome{Name: "m05-a", Version: "0.1.0", Digest: "sha256:m05"}, nil) + if store.reads != 2 || !strings.Contains(strings.Join(logs, "\n"), "m05 holds 1 built digests") || !strings.Contains(strings.Join(logs, "\n"), "14 reservations stand at it") { + t.Errorf("%d reads for two publisheds, journal %q: want one read each and the line made from it", store.reads, logs) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seats.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seats.go index 3bcc405d..d45ffe18 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seats.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seats.go @@ -103,7 +103,12 @@ func (h *seatHearer) heard(ctx context.Context, m jetstream.Msg) bool { h.pub.heard(ctx, se.seat, se.event, o) } if h.res != nil && se.event == versions.HeardPublished { - h.res.published(ctx, o) + // A pass over many reservations may outlast a third of the ack wait: the bus is told in progress before it (H-A-280). + h.res.published(ctx, o, func() { + if err := m.InProgress(); err != nil { + h.logf("in progress of %s not said: %v", m.Subject(), err) + } + }) } if se.event == versions.HeardPublished && o.Kind == versions.KindPackage && h.b != nil { // A package registry seat's published pins a package an entry names with none, at the version it carries (H-A-192). diff --git a/modules/mesh-delivery-builder/internal/versions/index.go b/modules/mesh-delivery-builder/internal/versions/index.go index 937048c5..f3316af0 100644 --- a/modules/mesh-delivery-builder/internal/versions/index.go +++ b/modules/mesh-delivery-builder/internal/versions/index.go @@ -19,9 +19,12 @@ import ( const ( indexDigestPrefix = "index.digest." indexVersionPrefix = "index.version." + indexUnbuiltPrefix = "index.unbuilt." // IndexDigestFilter and IndexVersionFilter are the filters of the index's two halves. IndexDigestFilter = indexDigestPrefix + "*" IndexVersionFilter = indexVersionPrefix + "*" + // IndexUnbuiltFilter is the filter of the reservations at a version whose build has not recorded every digest (H-A-280). + IndexUnbuiltFilter = indexUnbuiltPrefix + "*" ) // IndexDigestKey is the index key of the reservations holding digest as a built artifact's. @@ -30,6 +33,10 @@ func IndexDigestKey(digest string) string { return indexDigestPrefix + EscapeTok // IndexVersionKey is the index key of the reservations standing at version (MAJOR.MINOR.PATCH). func IndexVersionKey(version string) string { return indexVersionPrefix + EscapeToken(version) } +// IndexUnbuiltKey is the index key of the reservations at version whose build has not recorded every digest (design 47 H +// H-A-280): the only ones a published whose digest no reservation holds is kept on (A-415), kept apart from the built. +func IndexUnbuiltKey(version string) string { return indexUnbuiltPrefix + EscapeToken(version) } + // IndexEntry is an index key's value: the reservation keys, sorted. type IndexEntry struct { Keys []string `json:"keys"` @@ -90,6 +97,9 @@ func indexKeys(v ReservedVersion) []string { out := []string{} if v.Version != "" { out = append(out, IndexVersionKey(v.Version)) + if v.BuiltAt.IsZero() { + out = append(out, IndexUnbuiltKey(v.Version)) + } } for _, d := range v.Artifacts { if d != "" && !slices.Contains(out, IndexDigestKey(d)) { @@ -122,6 +132,11 @@ func (r *Reserver) reindex(ctx context.Context, key string, was, now ReservedVer return nil } +// IndexedAt are the keys of the reservations standing at version, by the index by version alone (H-A-280): no reservation read. +func (r *Reserver) IndexedAt(ctx context.Context, version string) ([]string, error) { + return r.indexed(ctx, IndexVersionKey(version)) +} + // AtVersion are the reservations standing at version, read through the index by version (H-A-271): one read of the index and // one of each reservation it names. func (r *Reserver) AtVersion(ctx context.Context, version string) ([]ReservedVersion, error) { @@ -160,7 +175,7 @@ func (r *Reserver) Reindex(ctx context.Context) (int, error) { } } } - for _, filter := range []string{IndexDigestFilter, IndexVersionFilter} { + for _, filter := range []string{IndexDigestFilter, IndexVersionFilter, IndexUnbuiltFilter} { keys, err := r.Store.KeysOf(ctx, filter) if err != nil { return len(all), fmt.Errorf("the index cannot be listed: %w", err) diff --git a/modules/mesh-delivery-builder/internal/versions/index_test.go b/modules/mesh-delivery-builder/internal/versions/index_test.go index c31d3425..d946002a 100644 --- a/modules/mesh-delivery-builder/internal/versions/index_test.go +++ b/modules/mesh-delivery-builder/internal/versions/index_test.go @@ -62,21 +62,37 @@ func TestTheIndexFindsAPublishedReservationInOneRead(t *testing.T) { t.Errorf("the published listed %d times and read %d reservations, want none listed and one read", l, n) } }) - t.Run("a digest no reservation holds is judged from its version's keys", func(t *testing.T) { + t.Run("a digest no reservation holds is judged from its version's unbuilt keys", func(t *testing.T) { r, store := fill(t) marked, kept, err := r.Published(ctx, "m11-a", "0.1.0", "sha256:early") if err != nil || len(marked) != 0 { t.Fatalf("an unbuilt digest marked %v (%v)", marked, err) } - // The 20 unbuilt reservations at 0.1.0 keep it as heard (A-415); the built ones are read and left. - if len(kept) != 20 || store.lists.Load() != 0 || store.reads.Load() != 40 { - t.Errorf("kept on %d (want 20), listed %d (want 0), read %d (want the 40 at 0.1.0)", len(kept), store.lists.Load(), store.reads.Load()) + // The 20 unbuilt reservations at 0.1.0 keep it as heard (A-415); the 20 built ones are not read at all (H-A-280). + if len(kept) != 20 || store.lists.Load() != 0 || store.reads.Load() != 20 { + t.Errorf("kept on %d (want 20), listed %d (want 0), read %d (want only the 20 unbuilt at 0.1.0)", len(kept), store.lists.Load(), store.reads.Load()) } at, err := r.AtVersion(ctx, "0.9.9") if err != nil || len(at) != 0 || store.lists.Load() != 0 { t.Errorf("a version no reservation stands at answered %v (%v), listed %d", at, err, store.lists.Load()) } }) + t.Run("a published at a version many share reads each reservation once and hands them on", func(t *testing.T) { + r, store := fill(t) + // A stale digest entry naming unbuilt reservations too: the second pass takes them as the first read them. + stale, _ := json.Marshal(IndexEntry{Keys: []string{ReservedKey("m01", "0.1"), ReservedKey("m03", "0.1")}}) + if _, err := store.Create(ctx, IndexDigestKey("sha256:stale"), stale); err != nil { + t.Fatal(err) + } + slow := 0 + out, err := r.PublishedRead(ctx, "x", "0.1.0", "sha256:stale", func() { slow++ }) + if err != nil || len(out.Kept) != 20 || len(out.Read) != 20 || slow != 1 { + t.Fatalf("kept %d, read %d, told slow %d (%v): want 20 kept, 20 read and handed on, slow told once", len(out.Kept), len(out.Read), slow, err) + } + if n := store.reads.Load(); n != 20 { + t.Errorf("%d reservation reads, want each of the 20 read once", n) + } + }) t.Run("a release and a drop take the reservation out of the index", func(t *testing.T) { r, store := fill(t) if _, err := r.Released(ctx, "m10", "0.1.0"); err != nil { diff --git a/modules/mesh-delivery-builder/internal/versions/reserve.go b/modules/mesh-delivery-builder/internal/versions/reserve.go index 7fcbaf71..8b819733 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve.go @@ -601,88 +601,138 @@ func (m Marked) PackagePublished(digest string, at time.Time) meshbuilds.Package Version: m.Reserved.Version, Digest: digest, Dependencies: m.Reserved.Dependencies[m.Artifact]} } -// Published takes a registry seat's published in: the artifacts of a reservation at version (a leading v is the Go form) -// whose built digest it names are marked published. It answers each artifact it newly marked, with the reservation as marked. -// A published that marks no artifact of a reservation at version whose build has not recorded every digest is kept on that -// reservation as heard (name, version, digest), and the keys it is kept on are answered too (design 47 G A-415): Kept marks -// it when the build records its digests. The reservation is found through the index (H-A-269 (1), H-A-271): the reservations -// holding the digest in one read, and only when none holds it at version, those standing at the version in one more; never a -// scan of every reservation. +// Published takes a registry seat's published in, as PublishedRead does, answering the artifacts newly marked and the keys it is +// kept on. func (r *Reserver) Published(ctx context.Context, name, version, digest string) ([]Marked, []string, error) { + out, err := r.PublishedRead(ctx, name, version, digest, nil) + return out.Marked, out.Kept, err +} + +// ReadReservation is a reservation a published read, as it stood after the published was taken in, and whether one of its built +// digests is the published's (design 47 H H-A-280). +type ReadReservation struct { + Key string + Reserved ReservedVersion + Matched bool +} + +// PublishedOutcome is what a published did: the artifacts it newly marked, the keys it is kept on (A-415), and every reservation it +// read, each once, which the journal line is made from (H-A-280). +type PublishedOutcome struct { + Marked []Marked + Kept []string + Read []ReadReservation +} + +// PublishedRead takes a registry seat's published in: the artifacts of a reservation at version (a leading v is the Go form) +// whose built digest it names are marked published. A published that marks no artifact of a reservation at version whose build +// has not recorded every digest is kept on that reservation as heard (name, version, digest), and the keys it is kept on are +// answered (design 47 G A-415): Kept marks it when the build records its digests. The reservation is found through the index +// (H-A-269 (1), H-A-271, H-A-280): the reservations holding the digest in one read, and only when none at version holds it, those +// at the version whose build has not recorded every digest in one more; never a scan of every reservation, and each reservation +// read at most once (again only on a write that met another). slow is told before the second pass, which may read many, so the +// caller can say in progress to the bus; nil is told nothing. +func (r *Reserver) PublishedRead(ctx context.Context, name, version, digest string, slow func()) (PublishedOutcome, error) { version = strings.TrimPrefix(version, "v") + var out PublishedOutcome if version == "" || digest == "" { - return nil, nil, nil + return out, nil } keys, err := r.indexed(ctx, IndexDigestKey(digest)) if err != nil { - return nil, nil, fmt.Errorf("the index of digest %s cannot be read: %w", digest, err) + return out, fmt.Errorf("the index of digest %s cannot be read: %w", digest, err) } - marked, kept, matched, err := r.publishedOn(ctx, keys, name, version, digest, false) + read := map[string]readOne{} + matched, err := r.publishedOn(ctx, keys, name, version, digest, false, read, &out) if err != nil || matched { - return marked, kept, err + return out, err } - keys, err = r.indexed(ctx, IndexVersionKey(version)) + keys, err = r.indexed(ctx, IndexUnbuiltKey(version)) if err != nil { - return marked, kept, fmt.Errorf("the index of version %s cannot be read: %w", version, err) + return out, fmt.Errorf("the index of the unbuilt at version %s cannot be read: %w", version, err) } - m, k, _, err := r.publishedOn(ctx, keys, name, version, digest, true) - return append(marked, m...), append(kept, k...), err + if len(keys) > 0 && slow != nil { + slow() + } + _, err = r.publishedOn(ctx, keys, name, version, digest, true, read, &out) + return out, err +} + +// readOne is a reservation as one pass read it, kept so the next pass reads it no more. +type readOne struct { + v ReservedVersion + rev uint64 + ok bool } // publishedOn takes a published in on the reservations under keys: an artifact whose built digest it names is marked; with keep, -// a reservation at version whose build has not recorded every digest and that it matches nothing of keeps it (A-415). It also -// answers whether a reservation at version holds the digest. -func (r *Reserver) publishedOn(ctx context.Context, keys []string, name, version, digest string, keep bool) ([]Marked, []string, bool, error) { - var marked []Marked - var kept []string +// a reservation at version whose build has not recorded every digest and that it matches nothing of keeps it (A-415). A +// reservation read by an earlier pass (read) is taken as read; each read is added to out.Read once. It answers whether a +// reservation at version holds the digest. +func (r *Reserver) publishedOn(ctx context.Context, keys []string, name, version, digest string, keep bool, read map[string]readOne, out *PublishedOutcome) (bool, error) { anyMatched := false for _, k := range keys { - again: - cur, rev, found, err := r.get(ctx, k) - if err != nil { - return marked, kept, anyMatched, err - } - if !found || cur.Version != version { - continue - } - changed, matched, keepIt := false, false, false - var arts []string - for a, d := range cur.Artifacts { - if d != digest { - continue - } - matched = true - if !cur.Published[a] { - if cur.Published == nil { - cur.Published = map[string]bool{} + one, seen := read[k] + for { + if !seen { + cur, rev, found, err := r.get(ctx, k) + if err != nil { + return anyMatched, err } - cur.Published[a], changed = true, true - arts = append(arts, a) + one = readOne{v: cur, rev: rev, ok: found} } - } - anyMatched = anyMatched || matched - sort.Strings(arts) - if keep && !matched && cur.BuiltAt.IsZero() && !cur.keeps(name, digest) { - cur.Kept = append(cur.Kept, KeptPublished{Name: name, Version: version, Digest: digest}) - changed, keepIt = true, true - } - if !changed { - continue - } - raw, _ := json.Marshal(cur) - if _, err := r.Store.Update(ctx, k, raw, rev); errors.Is(err, ErrConflict) { - goto again - } else if err != nil { - return marked, kept, anyMatched, fmt.Errorf("%s cannot be written: %w", k, err) - } - for _, a := range arts { - marked = append(marked, Marked{Reserved: cur, Artifact: a}) - } - if keepIt { - kept = append(kept, k) + seen = false // a write that meets another reads again + cur := one.v + if !one.ok || cur.Version != version { + read[k] = one + break + } + changed, matched, keepIt := false, false, false + var arts []string + for a, d := range cur.Artifacts { + if d != digest { + continue + } + matched = true + if !cur.Published[a] { + cur.Published = maps.Clone(cur.Published) + if cur.Published == nil { + cur.Published = map[string]bool{} + } + cur.Published[a], changed = true, true + arts = append(arts, a) + } + } + sort.Strings(arts) + if keep && !matched && cur.BuiltAt.IsZero() && !cur.keeps(name, digest) { + cur.Kept = append(slices.Clone(cur.Kept), KeptPublished{Name: name, Version: version, Digest: digest}) + changed, keepIt = true, true + } + rev := one.rev + if changed { + raw, _ := json.Marshal(cur) + var err error + if rev, err = r.Store.Update(ctx, k, raw, one.rev); errors.Is(err, ErrConflict) { + continue + } else if err != nil { + return anyMatched, fmt.Errorf("%s cannot be written: %w", k, err) + } + for _, a := range arts { + out.Marked = append(out.Marked, Marked{Reserved: cur, Artifact: a}) + } + if keepIt { + out.Kept = append(out.Kept, k) + } + } + anyMatched = anyMatched || matched + if _, was := read[k]; !was { + out.Read = append(out.Read, ReadReservation{Key: k, Reserved: cur, Matched: matched}) + } + read[k] = readOne{v: cur, rev: rev, ok: true} + break } } - return marked, kept, anyMatched, nil + return anyMatched, nil } func (v ReservedVersion) keeps(name, digest string) bool { -- 2.54.0