diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index e09db687..b7068896 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -239,6 +239,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // and version-released heard; nothing is reserved before the fill. reservedKV := process.AwaitBucket(ctx, js, conn, own.reserved, "no version is reserved: builds are made unreserved", logger.Printf) reserved := newReservations(awaitedReserved{awaitedVersions{b: reservedKV, bucket: own.reserved}}, calls.Call, pubBoundsKV.KV, found, b.Reconsider, logger.Printf) + // The index by digest and by version beside the reservations is written again once the state is bound (H-A-269 (1), H-A-271). + reservedKV.Ready(func(jetstream.KeyValue) { reserved.reindex(ctx) }) // Each artifact the holder says published at a reserved version is said as the seat mesh-builds' package-published (design 47 G §52). reserved.say = func(ctx context.Context, p meshbuilds.PackagePublished) error { return emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectPackagePublished, "package-published-"+p.Module+"-"+p.Version+"-"+p.Artifact, p) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go index 7d0d4f77..87f09a7a 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go @@ -549,8 +549,14 @@ func (m memVersions) Update(_ context.Context, key string, v []byte, _ uint64) ( m[key] = v return 2, nil } -func (m memVersions) KeysOf(context.Context, string) ([]string, error) { - return slices.Collect(maps.Keys(m)), nil +func (m memVersions) KeysOf(_ context.Context, filter string) ([]string, error) { + var out []string + for k := range m { + if versions.FilterMatches(filter, k) { // as the bus reads a filter + out = append(out, k) + } + } + return out, nil } // memBuilds is the builder's builds in memory. diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go index 867af03e..cb73e20b 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go @@ -77,10 +77,21 @@ func newReservations(store versions.ReserveStore, call toolCaller, bounds func() } return out, nil }} - rv.r.Took = rv.took + rv.r.Took, rv.r.Logf = rv.took, logf return rv } +// reindex writes the reservations' index again from the reservations, once at the builder's start (H-A-269 (1), H-A-271), and +// says so. +func (rv *reservations) reindex(ctx context.Context) { + n, err := rv.r.Reindex(ctx) + if err != nil { + rv.logf("the reservations' index is written again from %d reservations, not whole: %v", n, err) + return + } + rv.logf("the reservations' index is written again from %d reservations", n) +} + // took is a build's takeover of the reservation of an older build never released (design 47 G A-455): the older version is // dropped, its bound ended and its finding cleared (the newer build's reservation has its own), and the builder's log says so. func (rv *reservations) took(ctx context.Context, older, newer versions.ReservedVersion) { @@ -349,21 +360,19 @@ func (rv *reservations) tagged(ctx context.Context, ev meshcatalog.VersionReleas return nil } -// noMark says why a published marked nothing: which reservation holds that version, and what it holds of digests. +// noMark says why a published marked nothing: which reservation holds that version, and what it holds of digests, read through the +// index by version (H-A-269 (1), H-A-271). func (rv *reservations) noMark(ctx context.Context, o versions.Outcome) string { version := strings.TrimPrefix(o.Version, "v") if version == "" || o.Digest == "" { return "it names no version or no digest (a by-digest publish)" } - all, err := rv.r.All(ctx) + at, err := rv.r.AtVersion(ctx, version) // the index by version, never every reservation (H-A-271) if err != nil { - return "the reservations cannot be read: " + err.Error() + return "the reservations at " + version + " cannot be read: " + err.Error() } var out []string - for _, v := range all { - if v.Version != version { - continue - } + for _, v := range at { out = append(out, fmt.Sprintf("%s holds %d built digests %v, published %v (a published heard before the build recorded its digests is kept until it does)", v.Module, len(v.Artifacts), v.Artifacts, v.Published)) } if len(out) == 0 { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go index 6051e802..e74573a0 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go @@ -48,7 +48,7 @@ func (m *memReserve) Update(_ context.Context, key string, v []byte, rev uint64) func (m *memReserve) KeysOf(_ context.Context, filter string) ([]string, error) { var out []string for k := range m.vals { - if filter == versions.AllFilter || strings.HasPrefix(k, strings.TrimSuffix(filter, "*")) { + if versions.FilterMatches(filter, k) { // as the bus reads a filter out = append(out, k) } } diff --git a/modules/mesh-delivery-builder/internal/build/reserve_test.go b/modules/mesh-delivery-builder/internal/build/reserve_test.go index 5d7e5009..871eede0 100644 --- a/modules/mesh-delivery-builder/internal/build/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/build/reserve_test.go @@ -360,7 +360,10 @@ func (r *realReserve) Built(ctx context.Context, module, version string, digests type reserveMap struct{ *memStore } -func (m reserveMap) KeysOf(ctx context.Context, _ string) ([]string, error) { return m.Keys(ctx) } +func (m reserveMap) KeysOf(ctx context.Context, filter string) ([]string, error) { + all, err := m.Keys(ctx) + return slices.DeleteFunc(all, func(k string) bool { return !versions.FilterMatches(filter, k) }), err // as the bus reads a filter +} func (m reserveMap) Delete(_ context.Context, key string) error { m.mu.Lock() defer m.mu.Unlock() diff --git a/modules/mesh-delivery-builder/internal/versions/index.go b/modules/mesh-delivery-builder/internal/versions/index.go new file mode 100644 index 00000000..937048c5 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/versions/index.go @@ -0,0 +1,190 @@ +package versions + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "sort" +) + +// **The reservations' index by digest and by version** (design 47 H H-A-269 (1), H-A-271): beside the reservations, in the same +// state, the builder keeps which reservation keys hold a built digest and which stand at a version, so a registry seat's published +// finds its reservation in one read and a published whose digest no reservation holds is judged from its version's keys in one +// more, never a scan of every reservation. The index keys have three tokens, so the reservations' filter (two tokens) never lists +// them. It is maintained at reserve, at a digest recorded, at release and at drop; an entry left behind by a write that ended +// halfway is harmless (every read checks the reservation it names), and a missing one is written again by Reindex at the start. + +const ( + indexDigestPrefix = "index.digest." + indexVersionPrefix = "index.version." + // IndexDigestFilter and IndexVersionFilter are the filters of the index's two halves. + IndexDigestFilter = indexDigestPrefix + "*" + IndexVersionFilter = indexVersionPrefix + "*" +) + +// IndexDigestKey is the index key of the reservations holding digest as a built artifact's. +func IndexDigestKey(digest string) string { return indexDigestPrefix + EscapeToken(digest) } + +// IndexVersionKey is the index key of the reservations standing at version (MAJOR.MINOR.PATCH). +func IndexVersionKey(version string) string { return indexVersionPrefix + EscapeToken(version) } + +// IndexEntry is an index key's value: the reservation keys, sorted. +type IndexEntry struct { + Keys []string `json:"keys"` +} + +// indexed reads an index key's reservation keys. +func (r *Reserver) indexed(ctx context.Context, key string) ([]string, error) { + raw, _, found, err := r.Store.Get(ctx, key) + if err != nil || !found { + return nil, err + } + var e IndexEntry + if err := json.Unmarshal(raw, &e); err != nil { + return nil, fmt.Errorf("%s holds what is no index entry: %w", key, err) + } + return e.Keys, nil +} + +// index adds (or, with add false, removes) reserved under the index key key, by compare-and-set; an entry left empty is deleted. +func (r *Reserver) index(ctx context.Context, key, reserved string, add bool) error { + for { + raw, rev, found, err := r.Store.Get(ctx, key) + if err != nil { + return err + } + var e IndexEntry + if found { + _ = json.Unmarshal(raw, &e) + } + has := slices.Contains(e.Keys, reserved) + if has == add { + return nil + } + if add { + e.Keys = append(e.Keys, reserved) + sort.Strings(e.Keys) + } else { + e.Keys = slices.DeleteFunc(e.Keys, func(k string) bool { return k == reserved }) + } + if len(e.Keys) == 0 { + return r.Store.Delete(ctx, key) + } + out, _ := json.Marshal(e) + if found { + _, err = r.Store.Update(ctx, key, out, rev) + } else { + _, err = r.Store.Create(ctx, key, out) + } + if errors.Is(err, ErrConflict) { + continue + } + return err + } +} + +// indexKeys are the index keys a reservation stands under: its version's and each built digest's. +func indexKeys(v ReservedVersion) []string { + out := []string{} + if v.Version != "" { + out = append(out, IndexVersionKey(v.Version)) + } + for _, d := range v.Artifacts { + if d != "" && !slices.Contains(out, IndexDigestKey(d)) { + out = append(out, IndexDigestKey(d)) + } + } + return out +} + +// reindex moves reservation key from the index keys of was (its value before, zero when there was none) to those of now (zero when +// it is gone). +func (r *Reserver) reindex(ctx context.Context, key string, was, now ReservedVersion) error { + add, drop := indexKeys(now), indexKeys(was) + var errs []error + for _, k := range add { + if err := r.index(ctx, k, key, true); err != nil { + errs = append(errs, err) + } + } + for _, k := range drop { + if !slices.Contains(add, k) { + if err := r.index(ctx, k, key, false); err != nil { + errs = append(errs, err) + } + } + } + if err := errors.Join(errs...); err != nil { + return fmt.Errorf("the index of %s is not written (Reindex writes it at the start): %w", key, err) + } + return nil +} + +// AtVersion are the reservations standing at version, read through the index by version (H-A-271): one read of the index and +// one of each reservation it names. +func (r *Reserver) AtVersion(ctx context.Context, version string) ([]ReservedVersion, error) { + keys, err := r.indexed(ctx, IndexVersionKey(version)) + if err != nil { + return nil, err + } + var out []ReservedVersion + for _, k := range keys { + cur, _, found, err := r.get(ctx, k) + if err != nil { + return out, err + } + if found && cur.Version == version { + out = append(out, cur) + } + } + return out, nil +} + +// Reindex writes the index again from the reservations, once at the builder's start (H-A-269 (1), H-A-271): each reservation is +// added under its keys, and a reservation an index key names that does not stand under it (read again) is removed from it. Both +// are the same compare-and-set as every other index write, so a reservation written meanwhile keeps its entries. It answers how +// many reservations it read. +func (r *Reserver) Reindex(ctx context.Context) (int, error) { + all, err := r.All(ctx) + if err != nil { + return 0, err + } + var errs []error + for _, v := range all { + key := ReservedKey(v.Module, v.Series) + for _, k := range indexKeys(v) { + if err := r.index(ctx, k, key, true); err != nil { + errs = append(errs, err) + } + } + } + for _, filter := range []string{IndexDigestFilter, IndexVersionFilter} { + keys, err := r.Store.KeysOf(ctx, filter) + if err != nil { + return len(all), fmt.Errorf("the index cannot be listed: %w", err) + } + for _, k := range keys { + named, err := r.indexed(ctx, k) + if err != nil { + errs = append(errs, err) + continue + } + for _, rk := range named { + cur, _, found, err := r.get(ctx, rk) + if err != nil { + errs = append(errs, err) + continue + } + if found && slices.Contains(indexKeys(cur), k) { + continue + } + if err := r.index(ctx, k, rk, false); err != nil { + errs = append(errs, err) + } + } + } + } + return len(all), errors.Join(errs...) +} diff --git a/modules/mesh-delivery-builder/internal/versions/index_test.go b/modules/mesh-delivery-builder/internal/versions/index_test.go new file mode 100644 index 00000000..c31d3425 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/versions/index_test.go @@ -0,0 +1,128 @@ +package versions + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "sync/atomic" + "testing" +) + +// countingReserve counts the listings of the reservations and the reads of a reservation key (two tokens), so a test sees that +// a published reads its reservation and no other. +type countingReserve struct { + *reserveMem + lists, reads atomic.Int64 +} + +func (c *countingReserve) KeysOf(ctx context.Context, filter string) ([]string, error) { + c.lists.Add(1) + return c.reserveMem.KeysOf(ctx, filter) +} + +func (c *countingReserve) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) { + if !strings.HasPrefix(key, "index.") { + c.reads.Add(1) + } + return c.reserveMem.Get(ctx, key) +} + +// TestTheIndexFindsAPublishedReservationInOneRead: design 47 H H-A-269 (1), H-A-271 — a registry seat's published finds its +// reservation through the index by digest, reading that reservation alone and listing none; a published whose digest no +// reservation holds is judged from its version's keys (kept on a build that recorded no digest yet, A-415), never a scan; the +// index goes with a release and a drop, and is written again at the start from reservations written before it existed. +func TestTheIndexFindsAPublishedReservationInOneRead(t *testing.T) { + ctx := context.Background() + fill := func(t *testing.T) (*Reserver, *countingReserve) { + store := &countingReserve{reserveMem: &reserveMem{mem: newMem()}} + r := &Reserver{Store: store, ReleasedOf: func(context.Context, string) ([]string, error) { return nil, nil }} + for i := range 40 { + m := fmt.Sprintf("m%02d", i) + if _, err := r.Reserve(ctx, m, "0.1", []string{m + ":a"}, Source{Fingerprint: "fp" + m}); err != nil { + t.Fatal(err) + } + if i%2 == 0 { // half are built, each with its own digest + if _, err := r.Built(ctx, m, "0.1.0", m+":a", "sha256:"+m, nil); err != nil { + t.Fatal(err) + } + } + } + store.lists.Store(0) + store.reads.Store(0) + return r, store + } + t.Run("a published by digest reads its one reservation", func(t *testing.T) { + r, store := fill(t) + marked, _, err := r.Published(ctx, "m10-a", "v0.1.0", "sha256:m10") + if err != nil || len(marked) != 1 || marked[0].Reserved.Module != "m10" || marked[0].Artifact != "m10:a" { + t.Fatalf("the published marked %+v (%v), want m10:a", marked, err) + } + if l, n := store.lists.Load(), store.reads.Load(); l != 0 || n != 1 { + t.Errorf("the published listed %d times and read %d reservations, want none listed and one read", l, n) + } + }) + t.Run("a digest no reservation holds is judged from its version's keys", func(t *testing.T) { + r, store := fill(t) + marked, kept, err := r.Published(ctx, "m11-a", "0.1.0", "sha256:early") + if err != nil || len(marked) != 0 { + t.Fatalf("an unbuilt digest marked %v (%v)", marked, err) + } + // The 20 unbuilt reservations at 0.1.0 keep it as heard (A-415); the built ones are read and left. + if len(kept) != 20 || store.lists.Load() != 0 || store.reads.Load() != 40 { + t.Errorf("kept on %d (want 20), listed %d (want 0), read %d (want the 40 at 0.1.0)", len(kept), store.lists.Load(), store.reads.Load()) + } + at, err := r.AtVersion(ctx, "0.9.9") + if err != nil || len(at) != 0 || store.lists.Load() != 0 { + t.Errorf("a version no reservation stands at answered %v (%v), listed %d", at, err, store.lists.Load()) + } + }) + t.Run("a release and a drop take the reservation out of the index", func(t *testing.T) { + r, store := fill(t) + if _, err := r.Released(ctx, "m10", "0.1.0"); err != nil { + t.Fatal(err) + } + if _, err := r.Drop(ctx, "m12", "0.1"); err != nil { + t.Fatal(err) + } + for _, k := range []string{IndexDigestKey("sha256:m10"), IndexDigestKey("sha256:m12")} { + if _, _, found, _ := store.Get(ctx, k); found { + t.Errorf("%s stands after its reservation went", k) + } + } + keys, _ := r.indexed(ctx, IndexVersionKey("0.1.0")) + for _, k := range keys { + if k == ReservedKey("m10", "0.1") || k == ReservedKey("m12", "0.1") { + t.Errorf("the version's index still names %s", k) + } + } + }) + t.Run("the index is written again at the start from the reservations", func(t *testing.T) { + store := &countingReserve{reserveMem: &reserveMem{mem: newMem()}} + r := &Reserver{Store: store} + // A reservation written before the index existed, and an index entry naming a reservation that is gone. + raw, _ := json.Marshal(ReservedVersion{V: 1, Module: "old", Series: "0.2", Version: "0.2.4", Artifacts: map[string]string{"old:a": "sha256:old"}}) + if _, err := store.Create(ctx, ReservedKey("old", "0.2"), raw); err != nil { + t.Fatal(err) + } + stale, _ := json.Marshal(IndexEntry{Keys: []string{ReservedKey("gone", "0.1")}}) + if _, err := store.Create(ctx, IndexDigestKey("sha256:gone"), stale); err != nil { + t.Fatal(err) + } + if n, err := r.Reindex(ctx); err != nil || n != 1 { + t.Fatalf("the reindex read %d reservations (%v), want 1", n, err) + } + if _, _, found, _ := store.Get(ctx, IndexDigestKey("sha256:gone")); found { + t.Error("the index entry of a reservation that is gone stands after the reindex") + } + if keys, _ := store.KeysOf(ctx, AllFilter); len(keys) != 1 { + t.Errorf("the reservations list %v: the index keys are listed among them", keys) + } + store.lists.Store(0) + store.reads.Store(0) + marked, _, err := r.Published(ctx, "old-a", "0.2.4", "sha256:old") + if err != nil || len(marked) != 1 || store.lists.Load() != 0 || store.reads.Load() != 1 { + t.Errorf("after the reindex the published marked %v (%v), listed %d, read %d", marked, err, store.lists.Load(), store.reads.Load()) + } + }) +} diff --git a/modules/mesh-delivery-builder/internal/versions/republish.go b/modules/mesh-delivery-builder/internal/versions/republish.go index bf68e038..a04b963f 100644 --- a/modules/mesh-delivery-builder/internal/versions/republish.go +++ b/modules/mesh-delivery-builder/internal/versions/republish.go @@ -77,6 +77,9 @@ type QueuedPublish struct { Begun bool `json:"begun,omitempty"` } +// QueueFilter is the filter of every queued version: a queue key has three tokens, which AllFilter (two) never lists on the bus. +const QueueFilter = "*.*.*" + // QueueKey is the key of holder's queued version of module. func QueueKey(holder, module, version string) string { return EscapeToken(holder) + "." + EscapeToken(module) + "." + EscapeToken(version) @@ -402,7 +405,7 @@ const PublishAgainDescription = "Publish reserved versions again (design 47 G se // Resume takes up, at the builder's start, every queued version again, a begun one sent anew (§588). func (r *Republisher) Resume(ctx context.Context) error { - keys, err := r.Queue.KeysOf(ctx, AllFilter) + keys, err := r.Queue.KeysOf(ctx, QueueFilter) if err != nil { return fmt.Errorf("the state %s cannot be listed: %w", PublishQueueState, err) } diff --git a/modules/mesh-delivery-builder/internal/versions/reserve.go b/modules/mesh-delivery-builder/internal/versions/reserve.go index fcae79ec..03db4b9a 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve.go @@ -135,6 +135,16 @@ type Reserver struct { // DefaultReleasedKept. ReleasedKept int Now func() time.Time + // Logf says an index write that failed (H-A-269 (1)): the reservation stands, and Reindex writes its index at the next + // start. Nil says nothing. + Logf func(string, ...any) +} + +// said is told the outcome of a reindex: a failure is said, the reservation's own write stands. +func (r *Reserver) said(err error) { + if err != nil && r.Logf != nil { + r.Logf("%v", err) + } } func (r *Reserver) now() time.Time { @@ -231,6 +241,7 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts if err != nil { return ReservedVersion{}, fmt.Errorf("%s cannot be reserved: %w", key, err) } + r.said(r.reindex(ctx, key, cur, out)) // cur is zero when there was none if found && r.Took != nil { r.Took(ctx, cur, out) } @@ -263,6 +274,8 @@ func (r *Reserver) Built(ctx context.Context, module, version, artifact, digest if err != nil || !found || cur.Version != version { return false, err } + was := cur + was.Artifacts = maps.Clone(cur.Artifacts) cur.Artifacts[artifact] = digest if len(deps) > 0 { if cur.Dependencies == nil { @@ -279,6 +292,7 @@ func (r *Reserver) Built(ctx context.Context, module, version, artifact, digest } else if err != nil { return false, fmt.Errorf("%s cannot be written: %w", key, err) } + r.said(r.reindex(ctx, key, was, cur)) return true, nil } } @@ -397,6 +411,7 @@ func (r *Reserver) Released(ctx context.Context, module, version string) ([]stri if err := r.Store.Delete(ctx, k); err != nil { return gone, fmt.Errorf("%s cannot be deleted: %w", k, err) } + r.said(r.reindex(ctx, k, cur, ReservedVersion{})) gone = append(gone, k) } sort.Strings(gone) @@ -484,11 +499,15 @@ func (r *Reserver) TakenOver(ctx context.Context, module, fingerprint string) (R // log. It answers whether there was one. func (r *Reserver) Drop(ctx context.Context, module, series string) (bool, error) { key := ReservedKey(module, series) - _, _, found, err := r.get(ctx, key) + cur, _, found, err := r.get(ctx, key) if err != nil || !found { return false, err } - return true, r.Store.Delete(ctx, key) + if err := r.Store.Delete(ctx, key); err != nil { + return true, err + } + r.said(r.reindex(ctx, key, cur, ReservedVersion{})) + return true, nil } // Marked is an artifact of a reservation a registry seat's published newly marked: the reservation as it stands then. @@ -511,29 +530,47 @@ func (m Marked) PackagePublished(digest string, at time.Time) meshbuilds.Package // whose built digest it names are marked published. It answers each artifact it newly marked, with the reservation as marked. // A published that marks no artifact of a reservation at version whose build has not recorded every digest is kept on that // reservation as heard (name, version, digest), and the keys it is kept on are answered too (design 47 G A-415): Kept marks -// it when the build records its digests. +// it when the build records its digests. The reservation is found through the index (H-A-269 (1), H-A-271): the reservations +// holding the digest in one read, and only when none holds it at version, those standing at the version in one more; never a +// scan of every reservation. func (r *Reserver) Published(ctx context.Context, name, version, digest string) ([]Marked, []string, error) { version = strings.TrimPrefix(version, "v") if version == "" || digest == "" { return nil, nil, nil } - keys, err := r.Store.KeysOf(ctx, AllFilter) + keys, err := r.indexed(ctx, IndexDigestKey(digest)) if err != nil { - return nil, nil, fmt.Errorf("the reservations cannot be listed: %w", err) + return nil, nil, fmt.Errorf("the index of digest %s cannot be read: %w", digest, err) } - sort.Strings(keys) + marked, kept, matched, err := r.publishedOn(ctx, keys, name, version, digest, false) + if err != nil || matched { + return marked, kept, err + } + keys, err = r.indexed(ctx, IndexVersionKey(version)) + if err != nil { + return marked, kept, fmt.Errorf("the index of version %s cannot be read: %w", version, err) + } + m, k, _, err := r.publishedOn(ctx, keys, name, version, digest, true) + return append(marked, m...), append(kept, k...), err +} + +// publishedOn takes a published in on the reservations under keys: an artifact whose built digest it names is marked; with keep, +// a reservation at version whose build has not recorded every digest and that it matches nothing of keeps it (A-415). It also +// answers whether a reservation at version holds the digest. +func (r *Reserver) publishedOn(ctx context.Context, keys []string, name, version, digest string, keep bool) ([]Marked, []string, bool, error) { var marked []Marked var kept []string + anyMatched := false for _, k := range keys { again: cur, rev, found, err := r.get(ctx, k) if err != nil { - return marked, kept, err + return marked, kept, anyMatched, err } if !found || cur.Version != version { continue } - changed, matched, keep := false, false, false + changed, matched, keepIt := false, false, false var arts []string for a, d := range cur.Artifacts { if d != digest { @@ -548,10 +585,11 @@ func (r *Reserver) Published(ctx context.Context, name, version, digest string) arts = append(arts, a) } } + anyMatched = anyMatched || matched sort.Strings(arts) - if !matched && cur.BuiltAt.IsZero() && !cur.keeps(name, digest) { + if keep && !matched && cur.BuiltAt.IsZero() && !cur.keeps(name, digest) { cur.Kept = append(cur.Kept, KeptPublished{Name: name, Version: version, Digest: digest}) - changed, keep = true, true + changed, keepIt = true, true } if !changed { continue @@ -560,16 +598,16 @@ func (r *Reserver) Published(ctx context.Context, name, version, digest string) if _, err := r.Store.Update(ctx, k, raw, rev); errors.Is(err, ErrConflict) { goto again } else if err != nil { - return marked, kept, fmt.Errorf("%s cannot be written: %w", k, err) + return marked, kept, anyMatched, fmt.Errorf("%s cannot be written: %w", k, err) } for _, a := range arts { marked = append(marked, Marked{Reserved: cur, Artifact: a}) } - if keep { + if keepIt { kept = append(kept, k) } } - return marked, kept, nil + return marked, kept, anyMatched, nil } func (v ReservedVersion) keeps(name, digest string) bool { diff --git a/modules/mesh-delivery-builder/internal/versions/store.go b/modules/mesh-delivery-builder/internal/versions/store.go index 66fb89fe..5cdb30ca 100644 --- a/modules/mesh-delivery-builder/internal/versions/store.go +++ b/modules/mesh-delivery-builder/internal/versions/store.go @@ -4,12 +4,28 @@ import ( "context" "errors" "fmt" + "strings" "time" ) // AllFilter is the key filter of every version of every name: two tokens. const AllFilter = "*.*" +// FilterMatches says whether key is under filter as the bus reads a key filter: token by token, a `*` any one token, so AllFilter +// lists the two-token keys alone and never the index's three-token keys (H-A-269 (1)). +func FilterMatches(filter, key string) bool { + ft, kt := strings.Split(filter, "."), strings.Split(key, ".") + if len(ft) != len(kt) { + return false + } + for i := range ft { + if ft[i] != "*" && ft[i] != kt[i] { + return false + } + } + return true +} + // ErrConflict is what a Store answers when a Create finds the key or an Update finds another revision. var ErrConflict = errors.New("the key exists or changed") diff --git a/modules/mesh-delivery-builder/internal/versions/tools.go b/modules/mesh-delivery-builder/internal/versions/tools.go index 09ef4f7a..990dacbc 100644 --- a/modules/mesh-delivery-builder/internal/versions/tools.go +++ b/modules/mesh-delivery-builder/internal/versions/tools.go @@ -45,13 +45,9 @@ func (s readStore) KeysOf(_ context.Context, filter string) ([]string, error) { if err != nil { return nil, err } - if filter == AllFilter { - return all, nil - } - prefix := strings.TrimSuffix(filter, "*") var out []string for _, k := range all { - if strings.HasPrefix(k, prefix) && !strings.Contains(k[len(prefix):], ".") { + if FilterMatches(filter, k) { out = append(out, k) } } diff --git a/modules/mesh-delivery-builder/internal/versions/versions_test.go b/modules/mesh-delivery-builder/internal/versions/versions_test.go index 4999f0b6..5d464da2 100644 --- a/modules/mesh-delivery-builder/internal/versions/versions_test.go +++ b/modules/mesh-delivery-builder/internal/versions/versions_test.go @@ -66,21 +66,9 @@ func (m *mem) KeysOf(_ context.Context, filter string) ([]string, error) { if m.down != nil { return nil, m.down } - prefix := strings.TrimSuffix(filter, "*") var out []string for k := range m.vals { - if filter == AllFilter { - out = append(out, k) - continue - } - if strings.Count(filter, ".") == 2 { // a three-token filter, `a.*.*` as the bus reads it - ft, kt := strings.Split(filter, "."), strings.Split(k, ".") - if len(kt) == 3 && kt[0] == ft[0] { - out = append(out, k) - } - continue - } - if strings.HasPrefix(k, prefix) && !strings.Contains(k[len(prefix):], ".") { + if FilterMatches(filter, k) { // token by token, as the bus reads a filter out = append(out, k) } }