diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go index a406d3a1..e5dd1e7c 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go @@ -14,7 +14,8 @@ import ( "mesh-delivery-builder/internal/versions" ) -// readerOf is a kept state as the versions package's reader. +// readerOf is a kept state as the versions package's reader: its values read in one page, one request for many (design 47 H +// H-A-290 (5), H-A-292), where a key-by-key read of a queue of two thousand keys outlasts the tool's call. func readerOf(s stdio.KeptState) versions.Reader { return versions.Reader{Keys: s.Keys, Get: func(key string) (json.RawMessage, bool, error) { e, err := s.Get(key) @@ -22,7 +23,34 @@ func readerOf(s stdio.KeptState) versions.Reader { return nil, false, err } return e.Value, true, nil - }} + }, All: func() (versions.StatePage, error) { return onePage(s.Page, s.Info) }} +} + +// statePageBytes and statePageBudget bound the one page of a state read, so its answer fits one bus message and its time. +const ( + statePageBytes = 768 * 1024 + statePageBudget = 10 * time.Second +) + +// onePage is the first page of a state's values, and never a second (H-A-292, H-A-293): when the state holds more, the page says +// so with the state's size from its info. A runtime that answers no page says errors.ErrUnsupported. +func onePage(page func(after string, maxBytes int, budget time.Duration) (stdio.StatePage, error), info func() (*stdio.StateInfo, error)) (versions.StatePage, error) { + pg, err := page("", statePageBytes, statePageBudget) + if err != nil { + return versions.StatePage{}, err + } + out := versions.StatePage{Values: make(map[string]json.RawMessage, len(pg.Entries)), More: pg.Next != ""} + for _, e := range pg.Entries { + out.Values[e.Key] = e.Value + } + if out.More { + if in, err := info(); err != nil || in == nil { + out.InfoWords = fmt.Sprintf("the state's info cannot be read: %v", err) + } else { + out.Keys, out.Bytes = in.Keys, in.Bytes + } + } + return out, nil } // republishTools are builder_publish_again and builder_publish_queue (design 47 G §588). The publish again is a hand act: @@ -59,7 +87,11 @@ func republishTools(queue, acted, reasks versions.Reader, ask queueAsker, now fu return map[string]any{"key": key, "queued": a.Queued, "answer": a.Answer}, nil }}, {Name: versions.PublishQueueTool, Description: versions.PublishQueueDescription, - Input: map[string]any{"type": "object", "properties": map[string]any{}}, - Run: func(map[string]any) (any, error) { return versions.AnswerQueue(queue, acted, reasks) }}, + Input: map[string]any{"type": "object", "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "list only this module's queued versions, each with its class"}, + "version": map[string]any{"type": "string", "description": "with module: only this version, MAJOR.MINOR.PATCH"}}}, + Run: func(args map[string]any) (any, error) { + return versions.AnswerQueue(queue, acted, reasks, str(args, "module"), str(args, "version")) + }}, } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go new file mode 100644 index 00000000..595ebf91 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go @@ -0,0 +1,44 @@ +package main + +import ( + "encoding/json" + "errors" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// TestAStateIsReadInOnePage: design 47 H H-A-292, H-A-293 — builder_publish_queue reads each of its own states in one page and +// never asks a second: a state past the page answers that page with More and the state's size from its info; one the page holds +// answers whole without reading the info. +func TestAStateIsReadInOnePage(t *testing.T) { + for _, c := range []struct { + name string + next string + }{{"past one page", "k1"}, {"held by one page", ""}} { + t.Run(c.name, func(t *testing.T) { + var asked []string + infos := 0 + page := func(after string, maxBytes int, budget time.Duration) (stdio.StatePage, error) { + asked = append(asked, after) + if after != "" { + return stdio.StatePage{Entries: []stdio.StateEntry{{Key: "k2", Value: json.RawMessage(`2`)}}}, nil + } + return stdio.StatePage{Entries: []stdio.StateEntry{{Key: "k1", Value: json.RawMessage(`1`)}}, Next: c.next}, nil + } + info := func() (*stdio.StateInfo, error) { infos++; return &stdio.StateInfo{Keys: 2, Bytes: 640}, nil } + pg, err := onePage(page, info) + more := c.next != "" + if err != nil || len(asked) != 1 || len(pg.Values) != 1 || pg.More != more || (more && (pg.Keys != 2 || pg.Bytes != 640 || infos != 1)) || (!more && infos != 0) { + t.Errorf("asked the pages after %q, read %v, more %v, size %d keys %d bytes, info read %d times (%v): want one page only, its rest said with the state's size", asked, pg.Values, pg.More, pg.Keys, pg.Bytes, infos, err) + } + }) + } + unsupported := func(string, int, time.Duration) (stdio.StatePage, error) { + return stdio.StatePage{}, errors.ErrUnsupported + } + if _, err := onePage(unsupported, nil); !errors.Is(err, errors.ErrUnsupported) { + t.Errorf("a runtime with no pages answered %v, want ErrUnsupported so the reader reads key by key", err) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index bd8a8fe3..2bfff267 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -261,7 +261,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro republish := &versions.Republisher{Reserved: reserved.r, Queue: awaitedQueue{awaitedReserved{awaitedVersions{b: queueKV, bucket: own.queue}}}, Entry: entryVia(calls.Call), Send: sendVia(calls.Call), Unreleased: reserved.unreleasedWords, Logf: logger.Printf, - Holders: holders} // a registry seat's holder from the seat-holders view, from the start (design 47 G A-416, A-416a) + Holders: holders, // a registry seat's holder from the seat-holders view, from the start (design 47 G A-416, A-416a) + Started: time.Now()} // a build ended from now on is of the live class, sent first (H-A-290 (1)) // A number a registry seat already holds is never reserved (H-A-275): read before reserving, through the seats' verb versions. reserved.r.HeldNumbers = func(ctx context.Context, module, fp string, refs []string) ([]string, error) { return republish.HeldNumbers(ctx, module, fp, refs, registryVersionsVia(calls.Call)) @@ -303,10 +304,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return republish.Held(ctx, v, registryVersionsVia(calls.Call)) }, markHeld: reserved.markHeld, moveElsewhere: reserved.movedElsewhere, - allDone: func(ctx context.Context) ([]versions.ReservedVersion, error) { - all, err := reserved.r.All(ctx) - return builtOf(all), err - }} + allDone: func(ctx context.Context) ([]versions.ReservedVersion, error) { return reserved.r.All(ctx) }, + retired: reserved.r.Retired} // only each module's newest done build is asked at the start (H-A-291 (1)) + // A version that never publishes (withdrawn or taken over) has its publish item ended by the queue too (H-A-290 (2)). + republish.Ended = items.Done reasksKV.Ready(func(jetstream.KeyValue) { items.AskAllDone(ctx) }) wirePublishItems(reserved, items, awaitedReserved{awaitedVersions{b: actedKV, bucket: own.acted}}, releasedKept) acts.Republish = republish.Republish diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go index f61ac764..65206476 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go @@ -46,7 +46,11 @@ type publishItems struct { reserved func(ctx context.Context, module, version string) (versions.ReservedVersion, bool, error) republish func(ctx context.Context, module, version string) ([]string, error) endedWith func(ctx context.Context, v versions.ReservedVersion, why string) // raises the finding version-unreleased - allDone func(ctx context.Context) ([]versions.ReservedVersion, error) + // allDone are the reservations, done or not: those whose build is done are asked at the start (builtOf). + allDone func(ctx context.Context) ([]versions.ReservedVersion, error) + // retired are the done builds a newer reservation or release of the same module supersedes, with why (H-A-291 (1)); nil + // retires none. + retired func(ctx context.Context, all []versions.ReservedVersion) (map[string]string, error) // unheard is how many messages the builder's durable on the events stream holds pending and unacknowledged, read before each // try (design 47 H H-A-269 (2), H-A-272); nil reads none. unheard func(ctx context.Context) (int, error) @@ -98,12 +102,28 @@ func (p *publishItems) Ask(ctx context.Context, module, version string) error { // AskAllDone asks the publish item of every reservation whose build is done and that has no release: at the builder's start, so // what stood before this change moves too, whatever its artifacts. func (p *publishItems) AskAllDone(ctx context.Context) { - vs, err := p.allDone(ctx) + all, err := p.allDone(ctx) if err != nil { p.logf("the reservations of done builds are not asked to publish at the start: %v", err) return } - for _, v := range vs { + var retired map[string]string + if p.retired != nil { + if retired, err = p.retired(ctx, all); err != nil { + p.logf("the superseded builds are not retired at the start: %v", err) + } + } + for _, v := range builtOf(all) { + if why := retired[v.Module+" "+v.Version]; why != "" { // only each module's newest is asked (H-A-291 (1), H-A-275) + p.logf("%s %s is not asked to publish at the start: %s; its publish item is ended", v.Module, v.Version, why) + p.Done(ctx, v.Module, v.Version) + continue + } + if why := versions.NotStanding(v, true); why != "" { // withdrawn: it never publishes (H-A-290 (2)) + p.logf("%s %s is not asked to publish at the start: %s; its publish item is ended", v.Module, v.Version, why) + p.Done(ctx, v.Module, v.Version) + continue + } if err := p.Ask(ctx, v.Module, v.Version); err != nil { p.logf("%v", err) } @@ -137,7 +157,8 @@ func (p *publishItems) Try(ctx context.Context, kv jetstream.KeyValue, e jetstre p.logf("%s %s: its reservation cannot be read, the publish item waits: %v", r.Module, r.Version, err) return } - if !held { // dropped, taken over or released: never because its artifacts are published (A-456b) + if why := versions.NotStanding(v, held); why != "" { // dropped, taken over, released or withdrawn: never because its artifacts are published (A-456b, H-A-290 (2)) + p.logf("%s %s: the publish item is ended: %s", r.Module, r.Version, why) _ = kv.Delete(ctx, e.Key()) return } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem_test.go index 1feffe0c..4e2c7ea2 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem_test.go @@ -4,9 +4,12 @@ import ( "context" "encoding/json" "errors" + "maps" "os" "path/filepath" + "slices" "strings" + "sync" "testing" "time" @@ -19,9 +22,10 @@ import ( "mesh-delivery-builder/internal/versions" ) -// itemKV is the re-ask state in memory: one revision per key. +// itemKV is the re-ask state in memory: one revision per key, its maps locked (a try writes from the re-ask watch's goroutine). type itemKV struct { jetstream.KeyValue + mu sync.Mutex vals map[string][]byte revs map[string]uint64 rev uint64 @@ -42,13 +46,25 @@ func (e itemEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValueP func (k *itemKV) entry(key string) jetstream.KeyValueEntry { return itemEntry{key: key, value: k.vals[key], rev: k.revs[key]} } + +// has says whether key holds a value. +func (k *itemKV) has(key string) bool { + k.mu.Lock() + defer k.mu.Unlock() + _, ok := k.vals[key] + return ok +} func (k *itemKV) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) { + k.mu.Lock() + defer k.mu.Unlock() if _, ok := k.vals[key]; !ok { return nil, jetstream.ErrKeyNotFound } return k.entry(key), nil } func (k *itemKV) Create(_ context.Context, key string, v []byte, _ ...jetstream.KVCreateOpt) (uint64, error) { + k.mu.Lock() + defer k.mu.Unlock() if _, ok := k.vals[key]; ok { return 0, jetstream.ErrKeyExists } @@ -57,6 +73,8 @@ func (k *itemKV) Create(_ context.Context, key string, v []byte, _ ...jetstream. return k.rev, nil } func (k *itemKV) Update(_ context.Context, key string, v []byte, rev uint64) (uint64, error) { + k.mu.Lock() + defer k.mu.Unlock() if k.revs[key] != rev { return 0, jetstream.ErrKeyExists } @@ -65,6 +83,8 @@ func (k *itemKV) Update(_ context.Context, key string, v []byte, rev uint64) (ui return k.rev, nil } func (k *itemKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error { + k.mu.Lock() + defer k.mu.Unlock() delete(k.vals, key) return nil } @@ -140,21 +160,49 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) { kv.vals, kv.revs = map[string][]byte{}, map[string]uint64{} p.allDone = func(context.Context) ([]versions.ReservedVersion, error) { published := versions.ReservedVersion{Module: "n", Version: "0.2.0", Artifacts: map[string]string{"n:a": "sha256:a"}, Published: map[string]bool{"n:a": true}, BuiltAt: now} - return builtOf([]versions.ReservedVersion{v, published, {Module: "o", Version: "0.1.0"}}), nil // the last build has not ended + withdrawn := versions.ReservedVersion{Module: "w", Version: "0.4.0", BuiltAt: now, Withdrawn: map[string]string{"0.4.0": "fp"}} + return builtOf([]versions.ReservedVersion{v, published, withdrawn, {Module: "o", Version: "0.1.0"}}), nil // the last build has not ended } + _ = p.Ask(ctx, "w", "0.4.0") // asked before its reservation was withdrawn p.AskAllDone(ctx) - if len(kv.vals) != 2 { - t.Errorf("the start asked %d items, want one per done unreleased reservation", len(kv.vals)) + if _, ok := kv.vals[versions.PublishItemKey("w", "0.4.0")]; len(kv.vals) != 2 || ok { + t.Errorf("the start left %d items (withdrawn's kept: %v), want one per done unreleased reservation and none for a withdrawn one (H-A-290 (2))", len(kv.vals), ok) } + // A start with three unreleased done versions of one module asks only the newest and retires the two (H-A-291 (1), H-A-275). + kv.vals, kv.revs = map[string][]byte{}, map[string]uint64{} + _ = p.Ask(ctx, "s", "0.1.3") // asked in an earlier run + p.allDone = func(context.Context) ([]versions.ReservedVersion, error) { + var out []versions.ReservedVersion + for _, ver := range []string{"0.1.3", "0.1.10", "0.1.2"} { // one series (H-A-293) + out = append(out, versions.ReservedVersion{Module: "s", Version: ver, Artifacts: map[string]string{"s:a": "sha256:a"}, BuiltAt: now}) + } + return out, nil + } + p.retired = (&versions.Reserver{}).Retired + p.AskAllDone(ctx) + if _, ok := kv.vals[versions.PublishItemKey("s", "0.1.10")]; len(kv.vals) != 1 || !ok { + t.Errorf("the start left the items %v, want only the newest, s 0.1.10, asked and the two older retired", slices.Collect(maps.Keys(kv.vals))) + } + kv.vals, kv.revs = map[string][]byte{}, map[string]uint64{} // A released version's item is deleted; a version no longer reserved has it deleted at its next try. p.Done(ctx, "m", "0.1.2") p.Done(ctx, "n", "0.2.0") if len(kv.vals) != 0 { t.Errorf("the release left %v", kv.vals) } + // A withdrawn reservation has its item ended at its try, nothing queued (H-A-290 (2)). + queued = nil + _ = p.Ask(ctx, "m", "0.1.2") + v.Withdrawn = map[string]string{"0.1.2": "fp"} + e, _ := kv.Get(ctx, key) + p.Try(ctx, kv, e, decodeItem(t, e.Value())) + if len(kv.vals) != 0 || len(queued) != 0 { + t.Errorf("a withdrawn reservation left its item %v, queued %v", kv.vals, queued) + } + v.Withdrawn = nil _ = p.Ask(ctx, "m", "0.1.2") reserved = false - e, _ := kv.Get(ctx, key) + e, _ = kv.Get(ctx, key) p.Try(ctx, kv, e, decodeItem(t, e.Value())) if len(kv.vals) != 0 { t.Errorf("a dropped reservation left its item %v", kv.vals) @@ -220,7 +268,7 @@ func TestThePublishItemsAreWiredToTheReservationsAndTheReAskWatch(t *testing.T) } first := done("src2:one") key := versions.PublishItemKey("m", first.Version) - if _, ok := kv.vals[key]; !ok { + if !kv.has(key) { t.Fatalf("a done build has no publish item: %v", kv.vals) } // A build whose artifacts are all published still gets its item, until its release is written (A-456b). @@ -231,7 +279,7 @@ func TestThePublishItemsAreWiredToTheReservationsAndTheReAskWatch(t *testing.T) if err := rv.Built(ctx, "m", first.Version, map[string]string{"m:a": "sha256:a"}, nil); err != nil { t.Fatal(err) } - if _, ok := kv.vals[key]; !ok { + if !kv.has(key) { t.Fatalf("a build with every artifact published has no item: %v", kv.vals) } // The watch hands the item to its try, which queues the publish. @@ -247,11 +295,11 @@ func TestThePublishItemsAreWiredToTheReservationsAndTheReAskWatch(t *testing.T) } // A takeover by another build deletes the older's item and asks the newer's. second := done("src2:two") - if _, ok := kv.vals[key]; ok { + if kv.has(key) { t.Errorf("the taken-over reservation kept its item") } secondKey := versions.PublishItemKey("m", second.Version) - if _, ok := kv.vals[secondKey]; !ok { + if !kv.has(secondKey) { t.Fatalf("the newer build has no item: %v", kv.vals) } // version-released deletes the item and keeps the release. @@ -259,7 +307,7 @@ func TestThePublishItemsAreWiredToTheReservationsAndTheReAskWatch(t *testing.T) if !rv.heard(ctx, msg) || !msg.acked { t.Fatal("version-released was not taken in") } - if _, ok := kv.vals[secondKey]; ok { + if kv.has(secondKey) { t.Errorf("a released version kept its publish item") } if ok, _ := rv.ReleasedFor(ctx, "m", "src2:two"); !ok { @@ -270,7 +318,7 @@ func TestThePublishItemsAreWiredToTheReservationsAndTheReAskWatch(t *testing.T) if _, err := rv.dropped(ctx, "m", "0.1", "op", "never", time.Unix(0, 0)); err != nil { t.Fatal(err) } - if _, ok := kv.vals[versions.PublishItemKey("m", third.Version)]; ok { + if kv.has(versions.PublishItemKey("m", third.Version)) { t.Errorf("a dropped reservation kept its item") } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish.go index e723ca63..0ea48b01 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish.go @@ -33,8 +33,8 @@ func (s awaitedQueue) Bytes(ctx context.Context) (uint64, error) { // entryVia reads a catalog graph entry by its fingerprint through the seat mesh-catalog's verb entry. func entryVia(call toolCaller) func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) { - return func(_ context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) { - raw, err := call("seat:"+meshcatalog.Seat+"."+meshcatalog.VerbEntry, meshcatalog.EntryInput{Fingerprint: fingerprint}) + return func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) { + raw, err := callWithin(ctx, call, "seat:"+meshcatalog.Seat+"."+meshcatalog.VerbEntry, meshcatalog.EntryInput{Fingerprint: fingerprint}) if err != nil { return meshcatalog.CatalogEntry{}, err } @@ -50,8 +50,8 @@ func entryVia(call toolCaller) func(ctx context.Context, fingerprint string) (me // artifact store, addressed by digest. The four seats' publish take one wire form; an answer that lacks parts, or a fault of // the call, is the holder's refusal, said in its words (not stored when the holder lacks the bytes). func sendVia(call toolCaller) func(ctx context.Context, p versions.Publish) error { - return func(_ context.Context, p versions.Publish) error { - raw, err := call("seat:"+p.Seat+"."+versions.VerbPublish, publishBody(p)) + return func(ctx context.Context, p versions.Publish) error { + raw, err := callWithin(ctx, call, "seat:"+p.Seat+"."+versions.VerbPublish, publishBody(p)) if err != nil { return fmt.Errorf("%s's publish of %s %s: %w", p.Seat, p.Name, p.Version, err) } @@ -66,6 +66,29 @@ func sendVia(call toolCaller) func(ctx context.Context, p versions.Publish) erro } } +// callWithin is call within ctx's bound (design 47 H H-A-290 (4)): its answer, or, once the bound ends first, that it was not +// answered within it. The call itself ends by its own bound (the SDK's call timeout), its answer then dropped. +func callWithin(ctx context.Context, call toolCaller, key string, body any) (json.RawMessage, error) { + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("%s is not called: %w", key, context.Cause(ctx)) + } + type answer struct { + raw json.RawMessage + err error + } + done := make(chan answer, 1) + go func() { + raw, err := call(key, body) + done <- answer{raw, err} + }() + select { + case a := <-done: + return a.raw, a.err + case <-ctx.Done(): + return nil, fmt.Errorf("%s is not answered within its bound: %w", key, context.Cause(ctx)) + } +} + // publishBody is the seat's publish input for p: the digest, its kind and name and version, no parts. func publishBody(p versions.Publish) gopackageregistry.PublishInput { return gopackageregistry.PublishInput{Commit: p.Commit, Digest: p.Digest, Fingerprint: p.Fingerprint, Kind: p.Kind, Name: p.Name, Transfer: p.Transfer, Version: p.Version} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish_test.go index d78396ca..51770fed 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish_test.go @@ -57,6 +57,26 @@ func TestRepublishSend(t *testing.T) { t.Errorf("%s: asked %q, error %v, want %q", name, asked, err, c.want) } } + // A send and its entry read honour the bound they are given (design 47 H H-A-290 (4)): a call still unanswered when it ends + // is said as not answered within it, and the send goes on. + release := make(chan struct{}) + defer close(release) + called := make(chan struct{}, 2) + slow := func(string, any) (json.RawMessage, error) { + called <- struct{}{} + <-release + return json.RawMessage(`{}`), nil + } + for name, call := range map[string]func(context.Context) error{ + "the publish": func(ctx context.Context) error { return sendVia(slow)(ctx, p) }, + "the entry": func(ctx context.Context) error { _, err := entryVia(slow)(ctx, "fp"); return err }, + } { + ctx, cancel := context.WithCancel(context.Background()) + go func() { <-called; cancel() }() + if err := call(ctx); err == nil || !strings.Contains(err.Error(), "within its bound") { + t.Errorf("%s past its bound answered %v, want it not answered within its bound", name, err) + } + } if _, _, err := build.NewRepublish("", "0.1.3", "why", "me", versionsNow()); err == nil { t.Error("a version with no module is accepted") } diff --git a/modules/mesh-delivery-builder/internal/build/claim_test.go b/modules/mesh-delivery-builder/internal/build/claim_test.go index ebbe4ec6..320c4ffd 100644 --- a/modules/mesh-delivery-builder/internal/build/claim_test.go +++ b/modules/mesh-delivery-builder/internal/build/claim_test.go @@ -102,7 +102,10 @@ func seed(t *testing.T, st *memStore, b builds.BuilderBuild) string { func valueOf(t *testing.T, st *memStore, key string) builds.BuilderBuild { t.Helper() - b, err := builds.DecodeBuilderBuild(st.vals[key]) + st.mu.Lock() + raw := st.vals[key] + st.mu.Unlock() + b, err := builds.DecodeBuilderBuild(raw) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-builder/internal/build/retakeask_test.go b/modules/mesh-delivery-builder/internal/build/retakeask_test.go index 35c461d3..b30b3c0f 100644 --- a/modules/mesh-delivery-builder/internal/build/retakeask_test.go +++ b/modules/mesh-delivery-builder/internal/build/retakeask_test.go @@ -95,10 +95,14 @@ func theOperatorAnswers(t *testing.T, option string) { if err != nil || h != occ.Acted { t.Fatalf("%s: %v, %v", option, h, err) } - for i := 0; i < 50 && option == OptionBuildOnceMore; i++ { // the build the answer starts runs on its own goroutine - r.b.Wait() - if valueOf(t, r.st, r.key).State == builds.BuildDone { + // The build the answer starts runs on its own goroutine: the builder is waited on only once that build has claimed the key + // (its start counted before its claim, read through the store's lock), never while its start may still be counted. + for i := 0; i < 50 && option == OptionBuildOnceMore; i++ { + if st := valueOf(t, r.st, r.key).State; st == builds.BuildDone { break + } else if st != builds.BuildFailed { + r.b.Wait() + continue } time.Sleep(time.Millisecond) } diff --git a/modules/mesh-delivery-builder/internal/versions/republish.go b/modules/mesh-delivery-builder/internal/versions/republish.go index 99ba763d..6e205cc6 100644 --- a/modules/mesh-delivery-builder/internal/versions/republish.go +++ b/modules/mesh-delivery-builder/internal/versions/republish.go @@ -7,6 +7,7 @@ import ( "fmt" "slices" "sort" + "strconv" "strings" "sync" "time" @@ -38,6 +39,15 @@ const ( // (H-A-272). VerbPublish = "publish" VerbVersions = "versions" + // PublishClassLive is the class of a publish whose build ended in this run, which a pipeline's build step waits on; + // PublishClassStart that of a publish asked again at a start, of a build that ended before it. A holder's queue sends every + // live one before any of the start, each class oldest first (design 47 H H-A-290 (1)). + PublishClassLive = "build-ended-this-run" + PublishClassStart = "asked-again-at-start" + // PublishSendsAtOnce bounds the sends of one holder's queue that run at once (H-A-290 (3)). + PublishSendsAtOnce = 4 + // PublishQueueListCap caps the queued versions and the publish items builder_publish_queue lists (H-A-290 (5)). + PublishQueueListCap = 50 ) // RepublishTransfer is a publish again's transfer name, the builder's own (the registry part's publish: the caller names its @@ -78,6 +88,26 @@ type QueuedPublish struct { Artifacts map[string]string `json:"artifacts"` Queued time.Time `json:"queued"` Begun bool `json:"begun,omitempty"` + // Class is PublishClassLive or PublishClassStart (H-A-290 (1)); a value written before it had one is of the start. + Class string `json:"class,omitempty"` +} + +// rank is the place of q's class in a holder's queue: the live first. +func (q QueuedPublish) rank() int { + if q.Class == PublishClassLive { + return 0 + } + return 1 +} + +// sortQueued sorts a holder's queued versions in the order they are sent: the live class first, each class oldest first. +func sortQueued(qs []QueuedPublish) { + sort.SliceStable(qs, func(i, j int) bool { + if a, b := qs[i].rank(), qs[j].rank(); a != b { + return a < b + } + return qs[i].Queued.Before(qs[j].Queued) + }) } // QueueFilter is the filter of every queued version: a queue key has three tokens, which AllFilter (two) never lists on the bus. @@ -121,9 +151,18 @@ type Republisher struct { // Holders answers a registry seat's holders: the catalog's SeatHolders, the part's one holder function that its FollowSeat // of the assignments takes too, read from the builder's start (design 47 G A-416, A-416a); nil answers none. Holders func(seat string) []string + // Started is when this run of the builder started: a version whose build ended at or after it is of the live class, one + // that ended before it of the start's (H-A-290 (1)). Zero makes every version live. + Started time.Time + // Ended ends the publish item of a version that never publishes, its reservation withdrawn or taken over (H-A-290 (2)); + // nil ends none. + Ended func(ctx context.Context, module, version string) mu sync.Mutex busy map[string]bool + // relist marks a holder whose pass lists again before its next start-class send: a live publish was queued meanwhile + // (H-A-291 (2)). + relist map[string]bool // lastQueued is the newest time a version was queued at: a clock that has not moved between two versions (a coarse clock, // or a fixed one) must not make the queue's order the order of its keys, so each is queued after the last. lastQueued time.Time @@ -155,6 +194,127 @@ func (r *Republisher) logf(f string, a ...any) { } } +// classOf is the class of v's publish: live when its build ended in this run (H-A-290 (1)). +func (r *Republisher) classOf(v ReservedVersion) string { + if v.BuiltAt.Before(r.Started) { + return PublishClassStart + } + return PublishClassLive +} + +// NotStanding says why the reservation of a version, as Of answers it (found), never publishes: it stands no more at the +// version (released, dropped or taken over by a newer build), or it was withdrawn (design 47 H H-A-225, H-A-279). Empty when it +// stands (H-A-290 (2)). +func NotStanding(v ReservedVersion, found bool) string { + switch { + case !found: + return "its reservation stands no more at the version (released, dropped or taken over)" + case hasKey(v.Withdrawn, v.Version): + return "its reservation was withdrawn" + } + return "" +} + +// newerVersion says whether MAJOR.MINOR.PATCH a is above b, number by number. +func newerVersion(a, b string) bool { + pa, pb := strings.Split(a, "."), strings.Split(b, ".") + if len(pa) != 3 || len(pb) != 3 { + return false + } + for i := range 3 { + x, errx := strconv.Atoi(pa[i]) + y, erry := strconv.Atoi(pb[i]) + if errx != nil || erry != nil { + return false + } + if x != y { + return x > y + } + } + return false +} + +// Retired are the done builds without a release among all that a newer reservation or a newer release of the same module and +// series (MAJOR.MINOR) supersedes, each with why (design 47 H H-A-291 (1), H-A-293; H-A-275: the older digest is dead): keyed +// ` `. A standing newer series never retires an older one's build. The releases are the builder's own record +// of them (Releases), read once per module. +func (r *Reserver) Retired(ctx context.Context, all []ReservedVersion) (map[string]string, error) { + seriesOf := func(v ReservedVersion) string { + s, _, err := SplitVersion(v.Version) + if err != nil { + return "" + } + return v.Module + " " + s + } + newest := map[string]string{} // module and series -> its newest reserved version + for _, v := range all { + k := seriesOf(v) + if n, ok := newest[k]; k != "" && (!ok || newerVersion(v.Version, n)) { + newest[k] = v.Version + } + } + kept := map[string][]string{} // module -> the releases the record keeps + read := map[string]bool{} + out := map[string]string{} + for _, v := range all { + k := seriesOf(v) + if v.BuiltAt.IsZero() || k == "" { + continue + } + if n := newest[k]; newerVersion(n, v.Version) { + out[v.Module+" "+v.Version] = "the newer reservation " + n + " of " + v.Module + " supersedes it" + continue + } + if !read[v.Module] && r.Releases != nil { + read[v.Module] = true + raw, _, found, err := r.Releases.Get(ctx, ReleasedKey(v.Module)) + if err != nil { + return out, fmt.Errorf("the releases of %s cannot be read: %w", v.Module, err) + } + var rec ReleasedRecord + if found && json.Unmarshal(raw, &rec) == nil { + for _, rb := range rec.Kept { + kept[v.Module] = append(kept[v.Module], rb.Version) + } + } + } + series, _, _ := SplitVersion(v.Version) + rel := "" + for _, kv := range kept[v.Module] { + if s, _, err := SplitVersion(kv); err == nil && s == series && newerVersion(kv, v.Version) && (rel == "" || newerVersion(kv, rel)) { + rel = kv + } + } + if rel != "" { + out[v.Module+" "+v.Version] = "the newer release " + rel + " of " + v.Module + " supersedes it" + } + } + return out, nil +} + +// drop deletes every queued key of module's version and ends its publish item: it never publishes (H-A-290 (2)), said with why. +func (r *Republisher) drop(ctx context.Context, module, version, why string) { + ctx = context.WithoutCancel(ctx) + keys, err := r.Queue.KeysOf(ctx, "*."+EscapeToken(module)+"."+EscapeToken(version)) + if err != nil { + r.logf("publish again: %s %s never publishes (%s), and its queued keys cannot be listed: %v", module, version, why, err) + } + for _, k := range keys { + if err := r.Queue.Delete(ctx, k); err != nil { + r.logf("%s cannot be deleted from %s: %v", k, PublishQueueState, err) + } + } + r.end(ctx, module, version, why) +} + +// end ends module's version's publish item, said with why. +func (r *Republisher) end(ctx context.Context, module, version, why string) { + r.logf("publish again: %s %s is not published: %s; its queued key is dropped and its publish item ended", module, version, why) + if r.Ended != nil { + r.Ended(context.WithoutCancel(ctx), module, version) + } +} + // ActedKey is the acted key of a subject. func ActedKey(subject string) string { return EscapeToken(subject) } @@ -189,6 +349,10 @@ func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf if max > 0 && len(queued) >= max { break } + if why := NotStanding(v, true); why != "" { + r.drop(ctx, v.Module, v.Version, why) + continue + } if v.Fingerprint == "" || v.BuiltAt.IsZero() { r.logf("publish again: %s %s is not queued: it holds no fingerprint or its build has not ended with every digest", v.Module, v.Version) continue @@ -251,7 +415,7 @@ func (r *Republisher) enqueue(ctx context.Context, v ReservedVersion, holder str r.logf("publish again: %s %s is queued at %s already", v.Module, v.Version, holder) return false, nil } - raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: holder, Module: v.Module, Version: v.Version, Artifacts: arts, Queued: r.queuedAt()}) + raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: holder, Module: v.Module, Version: v.Version, Artifacts: arts, Queued: r.queuedAt(), Class: r.classOf(v)}) used, err := r.Queue.Bytes(ctx) if err != nil { return false, fmt.Errorf("the size of the state %s cannot be read: %w", PublishQueueState, err) @@ -268,7 +432,15 @@ func (r *Republisher) enqueue(ctx context.Context, v ReservedVersion, holder str } return false, fmt.Errorf("%s cannot be queued: %w", key, err) } - r.logf("publish again: %s %s is queued to publish at %s: %d artifacts", v.Module, v.Version, holder, len(arts)) + r.logf("publish again: %s %s is queued to publish at %s: %d artifacts, class %s", v.Module, v.Version, holder, len(arts), r.classOf(v)) + if r.classOf(v) == PublishClassLive { // a pass sending a start list lists again, so this one goes next (H-A-291 (2)) + r.mu.Lock() + if r.relist == nil { + r.relist = map[string]bool{} + } + r.relist[holder] = true + r.mu.Unlock() + } return true, nil } @@ -305,13 +477,43 @@ func (r *Republisher) Republish(ctx context.Context, module, version string) ([] return r.queue(ctx, vs, r.holdersFn, 0) } -// Queued are the queued versions, per holder in the order they are sent, and the acted keys with their sequences: -// builder_publish_queue. +// Queued is builder_publish_queue's answer, small at any size of the queue (design 47 H H-A-290 (5)): the queued versions' +// count per holder and class with the oldest and newest queue time, the first PublishQueueListCap of them in the order they are +// sent (each with its class), the acted keys with their sequences, and the publish items, counted and capped alike. type Queued struct { - Queue []QueuedPublish `json:"queue"` - Acted map[string]uint64 `json:"acted"` - // Items are the publish items (design 47 G A-457a), each with its tries and its next try; an item that ended says why. - Items []PublishItem `json:"items"` + Total int `json:"total"` + Counts []QueueCount `json:"counts"` + // Queue are the first queued versions, per holder in the order they are sent (the live class first, each class oldest first); + // with a module named, only that module's. + Queue []QueuedPublish `json:"queue"` + // Listed says how many of Total the answer lists. + Listed int `json:"listed"` + Acted map[string]uint64 `json:"acted"` + // ItemCount is how many publish items stand; Items the first of them by their next try (design 47 G A-457a), each with its + // tries and its next try; an item that ended says why. + ItemCount int `json:"item-count"` + Items []PublishItem `json:"items"` + // Capped says a state held more than the one page read of it (H-A-293); Read names each such state, the keys read and its + // size. Every count above is then of the keys read. + Capped bool `json:"capped,omitempty"` + Read []StateCapped `json:"read,omitempty"` +} + +// capped notes a state read only in part. +func (q *Queued) capped(c *StateCapped) { + if c != nil { + q.Capped = true + q.Read = append(q.Read, *c) + } +} + +// QueueCount is how many versions one holder's queue holds of one class, and when the oldest and the newest were queued. +type QueueCount struct { + Holder string `json:"holder"` + Class string `json:"class"` + Count int `json:"count"` + Oldest time.Time `json:"oldest"` + Newest time.Time `json:"newest"` } // PublishItemKeyPrefix opens the key of a publish item in the builder's re-ask state. @@ -335,69 +537,113 @@ func PublishItemKey(module, version string) string { return PublishItemKeyPrefix + EscapeToken(module) + "." + EscapeToken(version) } -// AnswerQueue is builder_publish_queue over the two sections as the tools bundle reads them. -func AnswerQueue(queue, acted, reasks Reader) (Queued, error) { - out := Queued{Queue: []QueuedPublish{}, Acted: map[string]uint64{}, Items: []PublishItem{}} - keys, err := queue.Keys() +// AnswerQueue is builder_publish_queue over the three states as the tools bundle reads them (H-A-290 (5)): each state read in +// bulk (Reader.All), the answer counted and capped at PublishQueueListCap whatever the queue holds; module (and version) narrow the +// listed queue to one module's (one version's), so an item's class can be read. +func AnswerQueue(queue, acted, reasks Reader, module, version string) (Queued, error) { + out := Queued{Counts: []QueueCount{}, Queue: []QueuedPublish{}, Acted: map[string]uint64{}, Items: []PublishItem{}} + vals, c, err := queue.values(PublishQueueState) + out.capped(c) if err != nil { - return out, fmt.Errorf("the state %s cannot be listed: %w", PublishQueueState, err) + return out, fmt.Errorf("the state %s cannot be read: %w", PublishQueueState, err) } - for _, k := range keys { - raw, ok, err := queue.Get(k) - if err != nil || !ok { + counts := map[[2]string]*QueueCount{} + var listed []QueuedPublish + for _, raw := range vals { + var q QueuedPublish + if json.Unmarshal(raw, &q) != nil { continue } - var q QueuedPublish - if json.Unmarshal(raw, &q) == nil { + if q.Class == "" { + q.Class = PublishClassStart + } + out.Total++ + c := counts[[2]string{q.Holder, q.Class}] + if c == nil { + c = &QueueCount{Holder: q.Holder, Class: q.Class, Oldest: q.Queued, Newest: q.Queued} + counts[[2]string{q.Holder, q.Class}] = c + } + c.Count++ + if q.Queued.Before(c.Oldest) { + c.Oldest = q.Queued + } + if q.Queued.After(c.Newest) { + c.Newest = q.Queued + } + if (module == "" || q.Module == module) && (version == "" || q.Version == version) { + listed = append(listed, q) + } + } + for _, c := range counts { + out.Counts = append(out.Counts, *c) + } + sort.Slice(out.Counts, func(i, j int) bool { + if out.Counts[i].Holder != out.Counts[j].Holder { + return out.Counts[i].Holder < out.Counts[j].Holder + } + return out.Counts[i].Class < out.Counts[j].Class + }) + sort.SliceStable(listed, func(i, j int) bool { return listed[i].Holder < listed[j].Holder }) + byHolder := map[string][]QueuedPublish{} + var holders []string + for _, q := range listed { + if byHolder[q.Holder] == nil { + holders = append(holders, q.Holder) + } + byHolder[q.Holder] = append(byHolder[q.Holder], q) + } + for _, h := range holders { + qs := byHolder[h] + sortQueued(qs) + for _, q := range qs { + if len(out.Queue) == PublishQueueListCap { + break + } out.Queue = append(out.Queue, q) } } - sort.SliceStable(out.Queue, func(i, j int) bool { - if out.Queue[i].Holder != out.Queue[j].Holder { - return out.Queue[i].Holder < out.Queue[j].Holder - } - return out.Queue[i].Queued.Before(out.Queue[j].Queued) - }) - akeys, err := acted.Keys() + out.Listed = len(out.Queue) + avals, c, err := acted.values(ActedState) + out.capped(c) if err != nil { - return out, fmt.Errorf("the state %s cannot be listed: %w", ActedState, err) + return out, fmt.Errorf("the state %s cannot be read: %w", ActedState, err) } - for _, k := range akeys { - raw, ok, err := acted.Get(k) - if err != nil || !ok { - continue - } + for k, raw := range avals { var a Acted if json.Unmarshal(raw, &a) == nil { out.Acted[k] = a.Sequence } } - rkeys, err := reasks.Keys() + rvals, c, err := reasks.values("reasks") + out.capped(c) if err != nil { - return out, fmt.Errorf("the re-ask state cannot be listed: %w", err) + return out, fmt.Errorf("the re-ask state cannot be read: %w", err) } - for _, k := range rkeys { + for k, raw := range rvals { if !strings.HasPrefix(k, PublishItemKeyPrefix) { continue } - raw, ok, err := reasks.Get(k) - if err != nil || !ok { - continue - } var it PublishItem if json.Unmarshal(raw, &it) == nil { out.Items = append(out.Items, it) } } + out.ItemCount = len(out.Items) sort.SliceStable(out.Items, func(i, j int) bool { return out.Items[i].NextTry.Before(out.Items[j].NextTry) }) + if len(out.Items) > PublishQueueListCap { + out.Items = out.Items[:PublishQueueListCap] + } return out, nil } // PublishQueueDescription is builder_publish_queue's description. -const PublishQueueDescription = "The versions the builder will publish again (its state " + PublishQueueState + ", design 47 G section 588): per registry " + - "seat holder, in the order they are sent, each with its artifacts' digests, when it was queued and whether its publish has begun; and the " + - "keys of the state " + ActedState + ", the last stream sequence the builder acted on for each subject; and the publish items (design 47 G " + - "A-457a), each version's tries and next try, and why one ended after its tries. Read-only." +const PublishQueueDescription = "The versions the builder will publish again (its state " + PublishQueueState + ", design 47 G section 588, H H-A-290): " + + "how many each registry seat holder's queue holds per class (build-ended-this-run, sent first, or asked-again-at-start) with the oldest and newest " + + "queue time; the first 50 in the order they are sent, each with its class, its artifacts' digests, when it was queued and whether its publish has " + + "begun (module, and version, list only that module's, so one item's class can be read); the keys of the state " + ActedState + ", the last stream " + + "sequence the builder acted on for each subject; and the publish items (design 47 G A-457a), counted, the first 50 by next try, each version's " + + "tries and next try, and why one ended after its tries. Each state is read in one page; a state past it is answered capped, naming the " + + "keys read and the state's size (H-A-293). Read-only." // PublishAgainDescription is builder_publish_again's description. const PublishAgainDescription = "Publish reserved versions again (design 47 G section 588), which builds nothing: with module and version, that reserved " + @@ -406,22 +652,57 @@ const PublishAgainDescription = "Publish reserved versions again (design 47 G se "with the versions it queued; each outcome is said as package-published, or left in version-unreleased with the holder's words. A hand act, " + "recorded in the hand-act log." -// Resume takes up, at the builder's start, every queued version again, a begun one sent anew (§588). +// Resume takes up, at the builder's start, every queued version again, a begun one sent anew (§588): a version whose reservation +// was withdrawn or taken over is dropped with its publish item ended (H-A-290 (2)), and one of the live class whose build ended +// before this run is of the start's now (H-A-290 (1)). It reads the reservations once. func (r *Republisher) Resume(ctx context.Context) error { keys, err := r.Queue.KeysOf(ctx, QueueFilter) if err != nil { return fmt.Errorf("the state %s cannot be listed: %w", PublishQueueState, err) } + all, err := r.Reserved.All(ctx) + if err != nil { + return fmt.Errorf("the queued publishes are not taken up: %w", err) + } + reserved := make(map[string]ReservedVersion, len(all)) + for _, v := range all { + reserved[v.Module+" "+v.Version] = v + } + retired, err := r.Reserved.Retired(ctx, all) + if err != nil { + r.logf("publish again: the superseded builds are not retired at the start: %v", err) + } holders := map[string]bool{} for _, k := range keys { - raw, _, found, err := r.Queue.Get(ctx, k) + raw, rev, found, err := r.Queue.Get(ctx, k) if err != nil || !found { continue } var q QueuedPublish - if json.Unmarshal(raw, &q) == nil { - holders[q.Holder] = true + if json.Unmarshal(raw, &q) != nil { + continue } + v, ok := reserved[q.Module+" "+q.Version] + why := NotStanding(v, ok) + if why == "" { + why = retired[q.Module+" "+q.Version] + } + if why != "" { + if err := r.Queue.Delete(ctx, k); err != nil { + r.logf("%s cannot be deleted from %s: %v", k, PublishQueueState, err) + } + r.end(ctx, q.Module, q.Version, why) + continue + } + if class := r.classOf(v); q.Class != class { + q.Class = class + if raw, err := json.Marshal(q); err == nil { + if _, err := r.Queue.Update(ctx, k, raw, rev); err != nil { + r.logf("%s keeps its class %s at the start: %v", k, q.Class, err) + } + } + } + holders[q.Holder] = true } for h := range holders { r.drainLater(ctx, h) @@ -451,16 +732,22 @@ func (r *Republisher) drainLater(ctx context.Context, holder string) { go run() } -// Drain sends holder's queued versions one at a time, oldest first, each bounded by UnreleasedBound; a key is deleted when -// its outcome is said. +// Drain sends holder's queued versions in passes (H-A-290 (3)): a pass lists the holder's keys once, reads each once, and sends +// them in their order (the live class first, each class oldest first), PublishSendsAtOnce at a time; the keys are listed again +// only when the pass is spent, or when a live publish was queued while it sends a start list: then the sends in flight end and +// the keys are listed again, so the live one goes next (H-A-291 (2)); a start-class one queued meanwhile waits for the pass's end. +// Each send is bounded by UnreleasedBound; a key is deleted when its outcome is said. func (r *Republisher) Drain(ctx context.Context, holder string) { for ctx.Err() == nil { + r.mu.Lock() + delete(r.relist, holder) + r.mu.Unlock() keys, err := r.Queue.KeysOf(ctx, EscapeToken(holder)+".*.*") if err != nil { r.logf("the queue of %s cannot be listed: %v", holder, err) return } - var items []QueuedPublish + items := make([]QueuedPublish, 0, len(keys)) for _, k := range keys { if raw, _, found, err := r.Queue.Get(ctx, k); err == nil && found { var q QueuedPublish @@ -472,12 +759,35 @@ func (r *Republisher) Drain(ctx context.Context, holder string) { if len(items) == 0 { return } - sort.SliceStable(items, func(i, j int) bool { return items[i].Queued.Before(items[j].Queued) }) - r.send(ctx, items[0]) + sortQueued(items) + sem := make(chan struct{}, PublishSendsAtOnce) + var wg sync.WaitGroup + for _, q := range items { + if ctx.Err() != nil { + break + } + sem <- struct{}{} + r.mu.Lock() + again := q.Class != PublishClassLive && r.relist[holder] + r.mu.Unlock() + if again { + <-sem + r.logf("publish again: %s's pass lists again after its sends in flight: a live publish was queued", holder) + break + } + wg.Add(1) + go func() { + defer func() { <-sem; wg.Done() }() + r.send(ctx, q) + }() + } + wg.Wait() } } -// send publishes one queued version's artifacts and says the outcome, then deletes its key. +// send publishes one queued version's artifacts and says the outcome, then deletes its key. Every call it makes honours the one +// bound it is given, UnreleasedBound (a context's bound, no timer, ADR 0437 §8), and a send that returns without sending says why +// (H-A-290 (4)). func (r *Republisher) send(ctx context.Context, q QueuedPublish) { key := QueueKey(q.Holder, q.Module, q.Version) defer func() { @@ -485,18 +795,44 @@ func (r *Republisher) send(ctx context.Context, q QueuedPublish) { r.logf("%s cannot be deleted from %s: %v", key, PublishQueueState, err) } }() + bound, cancel := context.WithTimeout(ctx, UnreleasedBound) + defer cancel() if !q.Begun { q.Begun = true raw, _ := json.Marshal(q) - if _, rev, found, err := r.Queue.Get(ctx, key); err == nil && found { - _, _ = r.Queue.Update(ctx, key, raw, rev) + if _, rev, found, err := r.Queue.Get(bound, key); err == nil && found { + _, _ = r.Queue.Update(bound, key, raw, rev) } } - v, found, err := r.Reserved.Of(ctx, q.Module, q.Version) - if err != nil || !found { - return // released meanwhile, or dropped + all, err := r.Reserved.All(bound) + if err != nil { + r.logf("publish again: %s %s is not sent to %s: its reservation cannot be read: %v; its publish item asks it again", q.Module, q.Version, q.Holder, err) + return } - e, err := r.Entry(ctx, v.Fingerprint) + var v ReservedVersion + found := false + var mine []ReservedVersion + for _, w := range all { + if w.Module == q.Module { + mine = append(mine, w) + if w.Version == q.Version { + v, found = w, true + } + } + } + why := NotStanding(v, found) + if why == "" { + retired, err := r.Reserved.Retired(bound, mine) + if err != nil { + r.logf("publish again: %s %s: whether a newer version supersedes it cannot be read, so it is sent: %v", q.Module, q.Version, err) + } + why = retired[q.Module+" "+q.Version] + } + if why != "" { + r.end(ctx, q.Module, q.Version, why) + return + } + e, err := r.Entry(bound, v.Fingerprint) if err != nil { r.say(ctx, v, fmt.Sprintf("the catalog entry %s cannot be read: %v", v.Fingerprint, err)) return @@ -512,10 +848,8 @@ func (r *Republisher) send(ctx context.Context, q QueuedPublish) { r.say(ctx, v, ref+": the catalog entry names no registry seat for it") return } - bound, cancel := context.WithTimeout(ctx, UnreleasedBound) // a context's bound, no timer (ADR 0437 §8) err := r.Send(bound, Publish{Seat: seat, Kind: kind, Name: name, Version: v.Version, Digest: q.Artifacts[ref], Fingerprint: v.Fingerprint, Commit: v.Commit, Transfer: RepublishTransfer(q.Module, v.Version, strings.TrimPrefix(ref, q.Module+":"))}) - cancel() if err == nil { r.logf("publish again: %s %s: %s sent to %s as %s %s, digest %s; the package-published is said when the seat says published", q.Module, q.Version, ref, seat, name, v.Version, q.Artifacts[ref]) } @@ -523,7 +857,7 @@ func (r *Republisher) send(ctx context.Context, q QueuedPublish) { r.logf("publish again: %s %s: %s refused by %s: %v", q.Module, q.Version, ref, seat, err) words := err.Error() if strings.Contains(words, strings.TrimSuffix(NotStoredWords, ": ")) { - if ok, merr := r.Reserved.MarkUnstored(ctx, q.Module, q.Version, ref, words); merr != nil { + if ok, merr := r.Reserved.MarkUnstored(bound, q.Module, q.Version, ref, words); merr != nil { r.logf("%s %s: the unstored %s is not kept: %v", q.Module, q.Version, ref, merr) } else if ok { r.say(ctx, v, ref+" "+NotStoredWords+words) diff --git a/modules/mesh-delivery-builder/internal/versions/republish_test.go b/modules/mesh-delivery-builder/internal/versions/republish_test.go index 642be137..51d1256a 100644 --- a/modules/mesh-delivery-builder/internal/versions/republish_test.go +++ b/modules/mesh-delivery-builder/internal/versions/republish_test.go @@ -5,8 +5,13 @@ import ( "encoding/json" "errors" "fmt" + "maps" + "slices" "strings" + "sync" + "sync/atomic" "testing" + "testing/synctest" "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" @@ -15,10 +20,19 @@ import ( type queueMem struct { *reserveMem bytes uint64 + // listings counts the listings of one holder's keys (a filter `.*.*`), what a drain's pass lists. + listings atomic.Int32 } func (q *queueMem) Bytes(context.Context) (uint64, error) { return q.bytes, nil } +func (q *queueMem) KeysOf(ctx context.Context, filter string) ([]string, error) { + if strings.HasSuffix(filter, ".*.*") && filter != QueueFilter { + q.listings.Add(1) + } + return q.reserveMem.KeysOf(ctx, filter) +} + // republishBed is a Republisher over memory states: reservations of module m (a bundle by the oci seat and a Go package by // the Go seat), one holder module `oci` that claims the oci seat, and a Send and a Unreleased that record. type republishBed struct { @@ -26,8 +40,10 @@ type republishBed struct { reserved *reserveMem queue *queueMem now time.Time + mu sync.Mutex // sent and said, which a holder's sends at once write sent []Publish said []string + ended []string refuse string // the words Send fails with, empty for none // holders are the registry seats' holders the seat-holders view names (design 47 G A-416). holders map[string][]string @@ -48,6 +64,8 @@ func newRepublishBed(t *testing.T) *republishBed { {Ref: m + ":lib", Kind: "package", Language: "go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/" + m + "/go"}}}}, nil }, Send: func(_ context.Context, p Publish) error { + b.mu.Lock() + defer b.mu.Unlock() b.sent = append(b.sent, p) if b.refuse != "" { return errors.New(b.refuse) @@ -55,8 +73,15 @@ func newRepublishBed(t *testing.T) *republishBed { return nil }, Unreleased: func(_ context.Context, v ReservedVersion, why string) { + b.mu.Lock() + defer b.mu.Unlock() b.said = append(b.said, v.Module+" "+v.Version+": "+why) }, + Ended: func(_ context.Context, module, version string) { + b.mu.Lock() + defer b.mu.Unlock() + b.ended = append(b.ended, module+" "+version) + }, Logf: t.Logf} b.Holders = func(seat string) []string { return b.holders[seat] } return b @@ -190,7 +215,7 @@ func TestRepublish(t *testing.T) { return json.RawMessage(`{"mode":"publish","module":"m","version":"0.1.0","tries":3,"next-try":"2026-11-17T01:00:00Z","why":"w"}`), true, nil }} none := Reader{Keys: func() ([]string, error) { return nil, nil }} - answer, err := AnswerQueue(none, none, items) + answer, err := AnswerQueue(none, none, items, "", "") if err != nil || len(answer.Items) != 1 || answer.Items[0].Tries != 3 || answer.Items[0].Version != "0.1.0" { t.Errorf("builder_publish_queue items = %+v (%v), want the one publish item with its tries", answer.Items, err) } @@ -235,3 +260,279 @@ func TestTheRegistriesRecordSaysWhetherAVersionIsHeld(t *testing.T) { t.Errorf("a version with no missing artifact is judged %+v (%v), want all held without a read", j, err) } } + +// queueAt writes a queued version of module m at holder oci directly, as a start finds it. +func (b *republishBed) queueAt(t *testing.T, v ReservedVersion, class string, at time.Time) { + t.Helper() + raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: v.Module, Version: v.Version, Artifacts: map[string]string{v.Module + ":prog": "sha256:aa"}, Queued: at, Class: class}) + if _, err := b.queue.Create(context.Background(), QueueKey("oci", v.Module, v.Version), raw); err != nil { + t.Fatal(err) + } +} + +// TestAHoldersQueueSendsLiveFirstInPasses: design 47 H H-A-290 (1)-(3) — a publish whose build ended in this run is sent before +// any of a build that ended before the start, each class oldest first; a pass lists the holder's keys once and sends them, at most +// four at a time; a build whose reservation was withdrawn or taken over never publishes, its key dropped and its publish item ended, +// at enqueue, at the pass and at the start. +func TestAHoldersQueueSendsLiveFirstInPasses(t *testing.T) { + ctx := context.Background() + t.Run("the live class first, at most four sends at once", func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + b := newRepublishBed(t) + var old []ReservedVersion + for _, m := range []string{"o1", "o2", "o3", "o4", "o5"} { + old = append(old, b.reserve(t, m)) + } + b.Started = b.now + b.reserve(t, "live") + for i, v := range old { + b.queueAt(t, v, PublishClassStart, b.now.Add(time.Duration(i)*time.Second)) + } + b.now = b.now.Add(time.Hour) // the live one is queued last, after every old one + b.holders = map[string][]string{SeatArtifacts: {"oci"}} + release := make(chan struct{}) + var arrived []string + var inflight, most int + send := b.Send + b.Send = func(ctx context.Context, p Publish) error { + b.mu.Lock() + arrived = append(arrived, p.Name) + inflight++ + most = max(most, inflight) + b.mu.Unlock() + <-release + b.mu.Lock() + inflight-- + b.mu.Unlock() + return send(ctx, p) + } + b.Go = func(f func()) { go f() } + if q, err := b.Republish(ctx, "live", ""); err != nil || len(q) != 1 { + t.Fatalf("queued %v (%v), want the live version", q, err) + } + synctest.Wait() // every send the bound lets run waits on release + b.mu.Lock() + first := slices.Clone(arrived) + b.mu.Unlock() + if len(first) != PublishSendsAtOnce || !slices.Contains(first, "live/prog") { + t.Errorf("the sends running at once are %v: want %d, the live one among them", first, PublishSendsAtOnce) + } + close(release) + synctest.Wait() + if len(b.sent) != 6 || most != PublishSendsAtOnce { + t.Errorf("sent %d, at most %d at once: want all six, %d at once", len(b.sent), most, PublishSendsAtOnce) + } + }) + }) + t.Run("a pass lists the keys once for all its sends", func(t *testing.T) { + b := newRepublishBed(t) + var vs []ReservedVersion + for i := range 6 { + vs = append(vs, b.reserve(t, fmt.Sprintf("m%d", i))) + } + for i, v := range vs { + b.queueAt(t, v, PublishClassStart, b.now.Add(time.Duration(i)*time.Second)) + } + b.Drain(ctx, "oci") + if n := b.queue.listings.Load(); len(b.sent) != 6 || n != 2 { + t.Errorf("sent %d with %d listings, want six sends from one pass and one listing that finds it spent", len(b.sent), n) + } + }) + t.Run("a start makes a live key of a build before it a start key", func(t *testing.T) { + b := newRepublishBed(t) + v := b.reserve(t, "m") + b.Started = b.now + b.queueAt(t, v, PublishClassLive, b.now) + b.Go = func(func()) {} // the start's drain is not run here + if err := b.Resume(ctx); err != nil { + t.Fatal(err) + } + raw, _, _, _ := b.queue.Get(ctx, QueueKey("oci", "m", v.Version)) + var q QueuedPublish + if json.Unmarshal(raw, &q) != nil || q.Class != PublishClassStart { + t.Errorf("the key after the start is %s, want the class %s: its build did not end in this run", raw, PublishClassStart) + } + }) + for _, at := range []string{"enqueue", "pass", "start"} { + t.Run("a withdrawn build never publishes, at the "+at, func(t *testing.T) { + b := newRepublishBed(t) + v := b.reserve(t, "m") + b.holders = map[string][]string{SeatArtifacts: {"oci"}} + if _, ok, err := b.Reserved.Withdraw(ctx, "m", v.Version); err != nil || !ok { + t.Fatalf("withdrawn %v (%v)", ok, err) + } + switch at { + case "enqueue": + b.queueAt(t, v, PublishClassLive, b.now) + b.Go = func(func()) {} // a key queued before the withdrawal is dropped by the enqueue itself + if q, _ := b.Republish(ctx, "m", v.Version); len(q) != 0 { + t.Errorf("queued %v", q) + } + case "pass": + b.queueAt(t, v, PublishClassLive, b.now) + b.Drain(ctx, "oci") + case "start": + b.queueAt(t, v, PublishClassLive, b.now) + b.Go = func(func()) {} + if err := b.Resume(ctx); err != nil { + t.Fatal(err) + } + } + keys, _ := b.queue.KeysOf(ctx, QueueFilter) + if len(b.sent) != 0 || len(keys) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" { + t.Errorf("sent %v, keys %v, items ended %v: want nothing sent, the key dropped and m %s's item ended", b.sent, keys, b.ended, v.Version) + } + }) + } + t.Run("a live publish queued mid-pass goes after the sends in flight, before the rest of the start list", func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + b := newRepublishBed(t) + var old []ReservedVersion + for _, m := range []string{"o1", "o2", "o3", "o4", "o5", "o6"} { + old = append(old, b.reserve(t, m)) + } + b.Started = b.now + b.reserve(t, "live") + for i, v := range old { + b.queueAt(t, v, PublishClassStart, b.now.Add(time.Duration(i)*time.Second)) + } + b.now = b.now.Add(time.Hour) + b.holders = map[string][]string{SeatArtifacts: {"oci"}} + first, then := make(chan struct{}), make(chan struct{}) // the four in flight wait on first, every later send on then + var arrived []string + send := b.Send + b.Send = func(ctx context.Context, p Publish) error { + b.mu.Lock() + arrived = append(arrived, p.Name) + gate := map[bool]chan struct{}{true: first, false: then}[len(arrived) <= PublishSendsAtOnce] + b.mu.Unlock() + <-gate + return send(ctx, p) + } + b.Go = func(f func()) { go f() } + b.drainLater(ctx, "oci") + synctest.Wait() // four start sends in flight, the pass holding o5 and o6 + if q, err := b.Republish(ctx, "live", ""); err != nil || len(q) != 1 { + t.Fatalf("queued %v (%v)", q, err) + } + close(first) + synctest.Wait() // the sends after the four: with the live one among them, or o5 and o6 alone when the pass goes on + b.mu.Lock() + next := slices.Clone(arrived[PublishSendsAtOnce:]) + b.mu.Unlock() + if !slices.Contains(next, "live/prog") { + t.Errorf("after the four in flight the sends are %v, want the live one among them, the pass listed again", next) + } + close(then) + synctest.Wait() + if len(arrived) != 7 { + t.Errorf("sent %v, want all seven", arrived) + } + }) + }) + t.Run("a build a newer patch or release of its series supersedes is retired at the pass and at the start, never by a newer series", func(t *testing.T) { + b := newRepublishBed(t) + v := b.reserve(t, "m") + b.holders = map[string][]string{SeatArtifacts: {"oci"}} + if _, err := b.Reserved.Reserve(ctx, "m", "0.2", []string{"m:prog"}, Source{Fingerprint: "src2:m"}); err != nil { + t.Fatal(err) + } + b.queueAt(t, v, PublishClassStart, b.now) + b.Drain(ctx, "oci") + if len(b.sent) != 1 || len(b.ended) != 0 { + t.Errorf("with a standing 0.2: sent %v, ended %v, want m %s sent, its series not superseded (H-A-293)", b.sent, b.ended, v.Version) + } + rel := &reserveMem{mem: newMem()} + raw, _ := json.Marshal(ReleasedRecord{Kept: []ReleasedBuild{{Fingerprint: "f", Version: "0.2.3"}, {Fingerprint: "g", Version: "0.1.9"}}}) + _, _ = rel.Create(ctx, ReleasedKey("m"), raw) + b.Reserved.Releases = rel + b.sent = nil + b.queueAt(t, v, PublishClassStart, b.now) + b.Drain(ctx, "oci") + if len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" { + t.Errorf("at the pass: sent %v, ended %v, want m %s retired by the release 0.1.9", b.sent, b.ended, v.Version) + } + b.ended = nil + b.queueAt(t, v, PublishClassStart, b.now) + b.Go = func(func()) {} + if err := b.Resume(ctx); err != nil { + t.Fatal(err) + } + if keys, _ := b.queue.KeysOf(ctx, QueueFilter); len(keys) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" { + t.Errorf("at the start: keys %v, ended %v, want m %s retired", keys, b.ended, v.Version) + } + only := &reserveMem{mem: newMem()} + raw, _ = json.Marshal(ReleasedRecord{Kept: []ReleasedBuild{{Fingerprint: "f", Version: "0.2.0"}}}) + _, _ = only.Create(ctx, ReleasedKey("n"), raw) + got, err := (&Reserver{Releases: only}).Retired(ctx, []ReservedVersion{{Module: "n", Version: "0.1.4", BuiltAt: b.now}, {Module: "n", Version: "0.1.6"}, + {Module: "n", Version: "0.3.0", BuiltAt: b.now}, {Module: "p", Version: "0.1.2", BuiltAt: b.now}}) + if err != nil || len(got) != 1 || !strings.Contains(got["n 0.1.4"], "reservation 0.1.6") { + t.Errorf("retired %v (%v), want only n 0.1.4, by the newer patch 0.1.6; n 0.3.0 and p 0.1.2 kept, a release of 0.2 retiring neither", got, err) + } + }) + t.Run("a version taken over is not sent, and its item ended", func(t *testing.T) { + b := newRepublishBed(t) + v := b.reserve(t, "m") + b.queueAt(t, v, PublishClassStart, b.now) + if _, err := b.Reserved.Reserve(ctx, "m", "0.1", []string{"m:prog"}, Source{Fingerprint: "src2:other"}); err != nil { + t.Fatal(err) + } + b.Drain(ctx, "oci") + if len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" { + t.Errorf("sent %v, ended %v: want nothing sent and the item ended", b.sent, b.ended) + } + }) +} + +// TestThePublishQueueToolAnswersAtAnySize: design 47 H H-A-290 (5) — builder_publish_queue over two thousand queued versions answers +// within a second and small: counts per holder and class with the oldest and newest queue time, a list capped at fifty in the +// order they are sent, and one module's item with its class; each state read in bulk where the reader can. +func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) { + at := time.Date(2026, 11, 20, 17, 0, 0, 0, time.UTC) + vals := map[string]json.RawMessage{} + for i := range 2000 { + holder, class := "oci", PublishClassStart + if i%2 == 1 { + holder = "go" + } + if i >= 1990 { + class = PublishClassLive + } + q := QueuedPublish{V: 1, Holder: holder, Module: fmt.Sprintf("m%d", i), Version: "0.1.0", Artifacts: map[string]string{"a": "sha256:aa"}, Queued: at.Add(time.Duration(i) * time.Second), Class: class} + vals[QueueKey(holder, q.Module, q.Version)], _ = json.Marshal(q) + } + gets := 0 + queue := Reader{All: func() (StatePage, error) { return StatePage{Values: vals}, nil }, + Keys: func() ([]string, error) { return slices.Collect(maps.Keys(vals)), nil }, + Get: func(k string) (json.RawMessage, bool, error) { gets++; v, ok := vals[k]; return v, ok, nil }} + none := Reader{Keys: func() ([]string, error) { return nil, nil }} + start := time.Now() + a, err := AnswerQueue(queue, none, none, "", "") + took := time.Since(start) + raw, _ := json.Marshal(a) + if err != nil || took > time.Second || a.Total != 2000 || len(a.Queue) != PublishQueueListCap || a.Listed != PublishQueueListCap || len(raw) > 64<<10 || gets != 0 { + t.Fatalf("answered %d of %d in %s, %d bytes, %d single reads (%v): want all counted, fifty listed, within a second, small, in bulk", len(a.Queue), a.Total, took, len(raw), gets, err) + } + if fmt.Sprintln(a.Counts[1].Holder, a.Counts[1].Class, a.Counts[1].Count, a.Counts[1].Oldest.Equal(at.Add(1991*time.Second)), a.Counts[1].Newest.Equal(at.Add(1999*time.Second))) != "go "+PublishClassLive+" 5 true true\n" || len(a.Counts) != 4 { + t.Errorf("counts %+v, want per holder and class with their oldest and newest", a.Counts) + } + if a.Queue[0].Class != PublishClassLive || a.Queue[0].Holder != "go" { + t.Errorf("the list opens with %+v, want the go holder's live class, sent first", a.Queue[0]) + } + one, err := AnswerQueue(queue, none, none, "m1995", "") + if err != nil || len(one.Queue) != 1 || one.Queue[0].Class != PublishClassLive || one.Total != 2000 { + t.Errorf("one module's answer %+v (%v), want its item with its class", one.Queue, err) + } + if a.Capped || len(a.Read) != 0 { + t.Errorf("a state one page holds is answered capped: %+v", a.Read) + } + queue.All = func() (StatePage, error) { return StatePage{Values: vals, More: true, Keys: 5000, Bytes: 3 << 20}, nil } + if c, err := AnswerQueue(queue, none, none, "", ""); err != nil || !c.Capped || len(c.Read) != 1 || c.Read[0].State != PublishQueueState || + c.Read[0].KeysRead != 2000 || c.Read[0].StateKeys != 5000 || c.Read[0].StateBytes != 3<<20 || c.Total != 2000 { + t.Errorf("a state past its one page answered capped %v, read %+v (%v): want capped, 2000 keys read of 5000, its size said (H-A-293)", c.Capped, c.Read, err) + } + queue.All = func() (StatePage, error) { return StatePage{}, errors.ErrUnsupported } + if b, err := AnswerQueue(queue, none, none, "", ""); err != nil || b.Total != 2000 || gets != 2000 { + t.Errorf("a reader with no bulk read answered %d (%v) with %d single reads, want every key read one by one", b.Total, err, gets) + } +} diff --git a/modules/mesh-delivery-builder/internal/versions/tools.go b/modules/mesh-delivery-builder/internal/versions/tools.go index 990dacbc..8a562cbf 100644 --- a/modules/mesh-delivery-builder/internal/versions/tools.go +++ b/modules/mesh-delivery-builder/internal/versions/tools.go @@ -21,10 +21,59 @@ const ( OldFormClosedEvent = "version.old-form-closed" ) -// Reader reads the state as the tools bundle may: every key, and one. +// Reader reads the state as the tools bundle may: every key, and one; and, where All is set, the state's values in one page, +// one request for many (design 47 H H-A-290 (5), H-A-292). type Reader struct { Keys func() ([]string, error) Get func(key string) (json.RawMessage, bool, error) + // All is one page of the state's values; nil, or an error satisfying errors.Is(err, errors.ErrUnsupported), reads key by key. + All func() (StatePage, error) +} + +// StatePage is one page of a state (H-A-292, H-A-293): its values, and, when the state holds more than the page, More with the +// state's size as its info says it (Keys, Bytes; InfoWords when the info cannot be read). No second page is asked. +type StatePage struct { + Values map[string]json.RawMessage + More bool + Keys, Bytes uint64 + InfoWords string +} + +// StateCapped is what an answer says of a state it read only the first page of (H-A-293): how many keys it read, and the +// state's size from its info. +type StateCapped struct { + State string `json:"state"` + KeysRead int `json:"keys-read"` + StateKeys uint64 `json:"state-keys"` + StateBytes uint64 `json:"state-bytes"` + InfoWords string `json:"info,omitempty"` +} + +// values is the state's values: one page where the reader can (capped says what a page past the state's end left out), else +// key by key. +func (r Reader) values(state string) (map[string]json.RawMessage, *StateCapped, error) { + if r.All != nil { + pg, err := r.All() + if !errors.Is(err, errors.ErrUnsupported) { + if err != nil || !pg.More { + return pg.Values, nil, err + } + return pg.Values, &StateCapped{State: state, KeysRead: len(pg.Values), StateKeys: pg.Keys, StateBytes: pg.Bytes, InfoWords: pg.InfoWords}, nil + } + } + keys, err := r.Keys() + if err != nil { + return nil, nil, err + } + out := make(map[string]json.RawMessage, len(keys)) + for _, k := range keys { + raw, ok, err := r.Get(k) + if err != nil || !ok { + continue + } + out[k] = raw + } + return out, nil, nil } // readStore is a Store over a Reader that refuses every write: the tools bundle only reads versions.