H-A-290: the builder's publish queue sends live publishes first, in passes, never a withdrawn build

The tool runner's release waited forty minutes behind old versions: every start asked a publish again for every done
build, and each send relisted and reread the whole queue. A publish whose build ended in this run now goes before any
asked again at a start (the class kept on the key's value, a live key of a build before the start made a start key at
the start); a pass lists a holder's keys once and sends them four at a time; a withdrawn or taken-over build is dropped
with its publish item ended at enqueue, at the pass and at the start; a send that returns unsent says why, and every call
it makes honours its one bound; builder_publish_queue reads its states in pages and answers counts per holder and class,
the oldest and newest queue time, a list capped at fifty and one module's items with their class.

Builds: design 47 H H-A-290 (1)-(5), H-A-269, H-A-271, H-A-275
This commit is contained in:
2026-11-21 18:59:58 +01:00
parent 5547fd028e
commit bda61ae64c
9 changed files with 592 additions and 77 deletions
@@ -14,7 +14,8 @@ import (
"mesh-delivery-builder/internal/versions"
)
// readerOf is a kept state as the versions package's reader.
// readerOf is a kept state as the versions package's reader: every value read in its pages, one request for many (design 47 H
// H-A-290 (5)), where a key-by-key read of a queue of two thousand keys outlasts the tool's call.
func readerOf(s stdio.KeptState) versions.Reader {
return versions.Reader{Keys: s.Keys, Get: func(key string) (json.RawMessage, bool, error) {
e, err := s.Get(key)
@@ -22,7 +23,31 @@ func readerOf(s stdio.KeptState) versions.Reader {
return nil, false, err
}
return e.Value, true, nil
}}
}, All: func() (map[string]json.RawMessage, error) { return allPages(s.Page) }}
}
// statePageBytes and statePageBudget bound one page of a whole state read, so its answer fits one bus message and its time.
const (
statePageBytes = 768 * 1024
statePageBudget = 10 * time.Second
)
// allPages is every key's value of a state, page after page; a runtime that answers no page says errors.ErrUnsupported.
func allPages(page func(after string, maxBytes int, budget time.Duration) (stdio.StatePage, error)) (map[string]json.RawMessage, error) {
out := map[string]json.RawMessage{}
for after := ""; ; {
pg, err := page(after, statePageBytes, statePageBudget)
if err != nil {
return nil, err
}
for _, e := range pg.Entries {
out[e.Key] = e.Value
}
if pg.Next == "" || pg.Next == after {
return out, nil
}
after = pg.Next
}
}
// republishTools are builder_publish_again and builder_publish_queue (design 47 G §588). The publish again is a hand act:
@@ -59,7 +84,11 @@ func republishTools(queue, acted, reasks versions.Reader, ask queueAsker, now fu
return map[string]any{"key": key, "queued": a.Queued, "answer": a.Answer}, nil
}},
{Name: versions.PublishQueueTool, Description: versions.PublishQueueDescription,
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) { return versions.AnswerQueue(queue, acted, reasks) }},
Input: map[string]any{"type": "object", "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "list only this module's queued versions, each with its class"},
"version": map[string]any{"type": "string", "description": "with module: only this version, MAJOR.MINOR.PATCH"}}},
Run: func(args map[string]any) (any, error) {
return versions.AnswerQueue(queue, acted, reasks, str(args, "module"), str(args, "version"))
}},
}
}
@@ -261,7 +261,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
republish := &versions.Republisher{Reserved: reserved.r, Queue: awaitedQueue{awaitedReserved{awaitedVersions{b: queueKV, bucket: own.queue}}},
Entry: entryVia(calls.Call), Send: sendVia(calls.Call),
Unreleased: reserved.unreleasedWords, Logf: logger.Printf,
Holders: holders} // a registry seat's holder from the seat-holders view, from the start (design 47 G A-416, A-416a)
Holders: holders, // a registry seat's holder from the seat-holders view, from the start (design 47 G A-416, A-416a)
Started: time.Now()} // a build ended from now on is of the live class, sent first (H-A-290 (1))
// A number a registry seat already holds is never reserved (H-A-275): read before reserving, through the seats' verb versions.
reserved.r.HeldNumbers = func(ctx context.Context, module, fp string, refs []string) ([]string, error) {
return republish.HeldNumbers(ctx, module, fp, refs, registryVersionsVia(calls.Call))
@@ -307,6 +308,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
all, err := reserved.r.All(ctx)
return builtOf(all), err
}}
// A version that never publishes (withdrawn or taken over) has its publish item ended by the queue too (H-A-290 (2)).
republish.Ended = items.Done
reasksKV.Ready(func(jetstream.KeyValue) { items.AskAllDone(ctx) })
wirePublishItems(reserved, items, awaitedReserved{awaitedVersions{b: actedKV, bucket: own.acted}}, releasedKept)
acts.Republish = republish.Republish
@@ -104,6 +104,11 @@ func (p *publishItems) AskAllDone(ctx context.Context) {
return
}
for _, v := range vs {
if why := versions.NotStanding(v, true); why != "" { // withdrawn: it never publishes (H-A-290 (2))
p.logf("%s %s is not asked to publish at the start: %s; its publish item is ended", v.Module, v.Version, why)
p.Done(ctx, v.Module, v.Version)
continue
}
if err := p.Ask(ctx, v.Module, v.Version); err != nil {
p.logf("%v", err)
}
@@ -137,7 +142,8 @@ func (p *publishItems) Try(ctx context.Context, kv jetstream.KeyValue, e jetstre
p.logf("%s %s: its reservation cannot be read, the publish item waits: %v", r.Module, r.Version, err)
return
}
if !held { // dropped, taken over or released: never because its artifacts are published (A-456b)
if why := versions.NotStanding(v, held); why != "" { // dropped, taken over, released or withdrawn: never because its artifacts are published (A-456b, H-A-290 (2))
p.logf("%s %s: the publish item is ended: %s", r.Module, r.Version, why)
_ = kv.Delete(ctx, e.Key())
return
}
@@ -140,11 +140,13 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) {
kv.vals, kv.revs = map[string][]byte{}, map[string]uint64{}
p.allDone = func(context.Context) ([]versions.ReservedVersion, error) {
published := versions.ReservedVersion{Module: "n", Version: "0.2.0", Artifacts: map[string]string{"n:a": "sha256:a"}, Published: map[string]bool{"n:a": true}, BuiltAt: now}
return builtOf([]versions.ReservedVersion{v, published, {Module: "o", Version: "0.1.0"}}), nil // the last build has not ended
withdrawn := versions.ReservedVersion{Module: "w", Version: "0.4.0", BuiltAt: now, Withdrawn: map[string]string{"0.4.0": "fp"}}
return builtOf([]versions.ReservedVersion{v, published, withdrawn, {Module: "o", Version: "0.1.0"}}), nil // the last build has not ended
}
_ = p.Ask(ctx, "w", "0.4.0") // asked before its reservation was withdrawn
p.AskAllDone(ctx)
if len(kv.vals) != 2 {
t.Errorf("the start asked %d items, want one per done unreleased reservation", len(kv.vals))
if _, ok := kv.vals[versions.PublishItemKey("w", "0.4.0")]; len(kv.vals) != 2 || ok {
t.Errorf("the start left %d items (withdrawn's kept: %v), want one per done unreleased reservation and none for a withdrawn one (H-A-290 (2))", len(kv.vals), ok)
}
// A released version's item is deleted; a version no longer reserved has it deleted at its next try.
p.Done(ctx, "m", "0.1.2")
@@ -152,9 +154,19 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) {
if len(kv.vals) != 0 {
t.Errorf("the release left %v", kv.vals)
}
// A withdrawn reservation has its item ended at its try, nothing queued (H-A-290 (2)).
queued = nil
_ = p.Ask(ctx, "m", "0.1.2")
v.Withdrawn = map[string]string{"0.1.2": "fp"}
e, _ := kv.Get(ctx, key)
p.Try(ctx, kv, e, decodeItem(t, e.Value()))
if len(kv.vals) != 0 || len(queued) != 0 {
t.Errorf("a withdrawn reservation left its item %v, queued %v", kv.vals, queued)
}
v.Withdrawn = nil
_ = p.Ask(ctx, "m", "0.1.2")
reserved = false
e, _ := kv.Get(ctx, key)
e, _ = kv.Get(ctx, key)
p.Try(ctx, kv, e, decodeItem(t, e.Value()))
if len(kv.vals) != 0 {
t.Errorf("a dropped reservation left its item %v", kv.vals)
@@ -33,8 +33,8 @@ func (s awaitedQueue) Bytes(ctx context.Context) (uint64, error) {
// entryVia reads a catalog graph entry by its fingerprint through the seat mesh-catalog's verb entry.
func entryVia(call toolCaller) func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) {
return func(_ context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) {
raw, err := call("seat:"+meshcatalog.Seat+"."+meshcatalog.VerbEntry, meshcatalog.EntryInput{Fingerprint: fingerprint})
return func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) {
raw, err := callWithin(ctx, call, "seat:"+meshcatalog.Seat+"."+meshcatalog.VerbEntry, meshcatalog.EntryInput{Fingerprint: fingerprint})
if err != nil {
return meshcatalog.CatalogEntry{}, err
}
@@ -50,8 +50,8 @@ func entryVia(call toolCaller) func(ctx context.Context, fingerprint string) (me
// artifact store, addressed by digest. The four seats' publish take one wire form; an answer that lacks parts, or a fault of
// the call, is the holder's refusal, said in its words (not stored when the holder lacks the bytes).
func sendVia(call toolCaller) func(ctx context.Context, p versions.Publish) error {
return func(_ context.Context, p versions.Publish) error {
raw, err := call("seat:"+p.Seat+"."+versions.VerbPublish, publishBody(p))
return func(ctx context.Context, p versions.Publish) error {
raw, err := callWithin(ctx, call, "seat:"+p.Seat+"."+versions.VerbPublish, publishBody(p))
if err != nil {
return fmt.Errorf("%s's publish of %s %s: %w", p.Seat, p.Name, p.Version, err)
}
@@ -66,6 +66,29 @@ func sendVia(call toolCaller) func(ctx context.Context, p versions.Publish) erro
}
}
// callWithin is call within ctx's bound (design 47 H H-A-290 (4)): its answer, or, once the bound ends first, that it was not
// answered within it. The call itself ends by its own bound (the SDK's call timeout), its answer then dropped.
func callWithin(ctx context.Context, call toolCaller, key string, body any) (json.RawMessage, error) {
if err := ctx.Err(); err != nil {
return nil, fmt.Errorf("%s is not called: %w", key, context.Cause(ctx))
}
type answer struct {
raw json.RawMessage
err error
}
done := make(chan answer, 1)
go func() {
raw, err := call(key, body)
done <- answer{raw, err}
}()
select {
case a := <-done:
return a.raw, a.err
case <-ctx.Done():
return nil, fmt.Errorf("%s is not answered within its bound: %w", key, context.Cause(ctx))
}
}
// publishBody is the seat's publish input for p: the digest, its kind and name and version, no parts.
func publishBody(p versions.Publish) gopackageregistry.PublishInput {
return gopackageregistry.PublishInput{Commit: p.Commit, Digest: p.Digest, Fingerprint: p.Fingerprint, Kind: p.Kind, Name: p.Name, Transfer: p.Transfer, Version: p.Version}
@@ -57,6 +57,26 @@ func TestRepublishSend(t *testing.T) {
t.Errorf("%s: asked %q, error %v, want %q", name, asked, err, c.want)
}
}
// A send and its entry read honour the bound they are given (design 47 H H-A-290 (4)): a call still unanswered when it ends
// is said as not answered within it, and the send goes on.
release := make(chan struct{})
defer close(release)
called := make(chan struct{}, 2)
slow := func(string, any) (json.RawMessage, error) {
called <- struct{}{}
<-release
return json.RawMessage(`{}`), nil
}
for name, call := range map[string]func(context.Context) error{
"the publish": func(ctx context.Context) error { return sendVia(slow)(ctx, p) },
"the entry": func(ctx context.Context) error { _, err := entryVia(slow)(ctx, "fp"); return err },
} {
ctx, cancel := context.WithCancel(context.Background())
go func() { <-called; cancel() }()
if err := call(ctx); err == nil || !strings.Contains(err.Error(), "within its bound") {
t.Errorf("%s past its bound answered %v, want it not answered within its bound", name, err)
}
}
if _, _, err := build.NewRepublish("", "0.1.3", "why", "me", versionsNow()); err == nil {
t.Error("a version with no module is accepted")
}
@@ -38,6 +38,15 @@ const (
// (H-A-272).
VerbPublish = "publish"
VerbVersions = "versions"
// PublishClassLive is the class of a publish whose build ended in this run, which a pipeline's build step waits on;
// PublishClassStart that of a publish asked again at a start, of a build that ended before it. A holder's queue sends every
// live one before any of the start, each class oldest first (design 47 H H-A-290 (1)).
PublishClassLive = "build-ended-this-run"
PublishClassStart = "asked-again-at-start"
// PublishSendsAtOnce bounds the sends of one holder's queue that run at once (H-A-290 (3)).
PublishSendsAtOnce = 4
// PublishQueueListCap caps the queued versions and the publish items builder_publish_queue lists (H-A-290 (5)).
PublishQueueListCap = 50
)
// RepublishTransfer is a publish again's transfer name, the builder's own (the registry part's publish: the caller names its
@@ -78,6 +87,26 @@ type QueuedPublish struct {
Artifacts map[string]string `json:"artifacts"`
Queued time.Time `json:"queued"`
Begun bool `json:"begun,omitempty"`
// Class is PublishClassLive or PublishClassStart (H-A-290 (1)); a value written before it had one is of the start.
Class string `json:"class,omitempty"`
}
// rank is the place of q's class in a holder's queue: the live first.
func (q QueuedPublish) rank() int {
if q.Class == PublishClassLive {
return 0
}
return 1
}
// sortQueued sorts a holder's queued versions in the order they are sent: the live class first, each class oldest first.
func sortQueued(qs []QueuedPublish) {
sort.SliceStable(qs, func(i, j int) bool {
if a, b := qs[i].rank(), qs[j].rank(); a != b {
return a < b
}
return qs[i].Queued.Before(qs[j].Queued)
})
}
// QueueFilter is the filter of every queued version: a queue key has three tokens, which AllFilter (two) never lists on the bus.
@@ -121,6 +150,12 @@ type Republisher struct {
// Holders answers a registry seat's holders: the catalog's SeatHolders, the part's one holder function that its FollowSeat
// of the assignments takes too, read from the builder's start (design 47 G A-416, A-416a); nil answers none.
Holders func(seat string) []string
// Started is when this run of the builder started: a version whose build ended at or after it is of the live class, one
// that ended before it of the start's (H-A-290 (1)). Zero makes every version live.
Started time.Time
// Ended ends the publish item of a version that never publishes, its reservation withdrawn or taken over (H-A-290 (2));
// nil ends none.
Ended func(ctx context.Context, module, version string)
mu sync.Mutex
busy map[string]bool
@@ -155,6 +190,50 @@ func (r *Republisher) logf(f string, a ...any) {
}
}
// classOf is the class of v's publish: live when its build ended in this run (H-A-290 (1)).
func (r *Republisher) classOf(v ReservedVersion) string {
if v.BuiltAt.Before(r.Started) {
return PublishClassStart
}
return PublishClassLive
}
// NotStanding says why the reservation of a version, as Of answers it (found), never publishes: it stands no more at the
// version (released, dropped or taken over by a newer build), or it was withdrawn (design 47 H H-A-225, H-A-279). Empty when it
// stands (H-A-290 (2)).
func NotStanding(v ReservedVersion, found bool) string {
switch {
case !found:
return "its reservation stands no more at the version (released, dropped or taken over)"
case hasKey(v.Withdrawn, v.Version):
return "its reservation was withdrawn"
}
return ""
}
// drop deletes every queued key of module's version and ends its publish item: it never publishes (H-A-290 (2)), said with why.
func (r *Republisher) drop(ctx context.Context, module, version, why string) {
ctx = context.WithoutCancel(ctx)
keys, err := r.Queue.KeysOf(ctx, "*."+EscapeToken(module)+"."+EscapeToken(version))
if err != nil {
r.logf("publish again: %s %s never publishes (%s), and its queued keys cannot be listed: %v", module, version, why, err)
}
for _, k := range keys {
if err := r.Queue.Delete(ctx, k); err != nil {
r.logf("%s cannot be deleted from %s: %v", k, PublishQueueState, err)
}
}
r.end(ctx, module, version, why)
}
// end ends module's version's publish item, said with why.
func (r *Republisher) end(ctx context.Context, module, version, why string) {
r.logf("publish again: %s %s is not published: %s; its queued key is dropped and its publish item ended", module, version, why)
if r.Ended != nil {
r.Ended(context.WithoutCancel(ctx), module, version)
}
}
// ActedKey is the acted key of a subject.
func ActedKey(subject string) string { return EscapeToken(subject) }
@@ -189,6 +268,10 @@ func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf
if max > 0 && len(queued) >= max {
break
}
if why := NotStanding(v, true); why != "" {
r.drop(ctx, v.Module, v.Version, why)
continue
}
if v.Fingerprint == "" || v.BuiltAt.IsZero() {
r.logf("publish again: %s %s is not queued: it holds no fingerprint or its build has not ended with every digest", v.Module, v.Version)
continue
@@ -251,7 +334,7 @@ func (r *Republisher) enqueue(ctx context.Context, v ReservedVersion, holder str
r.logf("publish again: %s %s is queued at %s already", v.Module, v.Version, holder)
return false, nil
}
raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: holder, Module: v.Module, Version: v.Version, Artifacts: arts, Queued: r.queuedAt()})
raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: holder, Module: v.Module, Version: v.Version, Artifacts: arts, Queued: r.queuedAt(), Class: r.classOf(v)})
used, err := r.Queue.Bytes(ctx)
if err != nil {
return false, fmt.Errorf("the size of the state %s cannot be read: %w", PublishQueueState, err)
@@ -268,7 +351,7 @@ func (r *Republisher) enqueue(ctx context.Context, v ReservedVersion, holder str
}
return false, fmt.Errorf("%s cannot be queued: %w", key, err)
}
r.logf("publish again: %s %s is queued to publish at %s: %d artifacts", v.Module, v.Version, holder, len(arts))
r.logf("publish again: %s %s is queued to publish at %s: %d artifacts, class %s", v.Module, v.Version, holder, len(arts), r.classOf(v))
return true, nil
}
@@ -305,13 +388,31 @@ func (r *Republisher) Republish(ctx context.Context, module, version string) ([]
return r.queue(ctx, vs, r.holdersFn, 0)
}
// Queued are the queued versions, per holder in the order they are sent, and the acted keys with their sequences:
// builder_publish_queue.
// Queued is builder_publish_queue's answer, small at any size of the queue (design 47 H H-A-290 (5)): the queued versions'
// count per holder and class with the oldest and newest queue time, the first PublishQueueListCap of them in the order they are
// sent (each with its class), the acted keys with their sequences, and the publish items, counted and capped alike.
type Queued struct {
Queue []QueuedPublish `json:"queue"`
Acted map[string]uint64 `json:"acted"`
// Items are the publish items (design 47 G A-457a), each with its tries and its next try; an item that ended says why.
Items []PublishItem `json:"items"`
Total int `json:"total"`
Counts []QueueCount `json:"counts"`
// Queue are the first queued versions, per holder in the order they are sent (the live class first, each class oldest first);
// with a module named, only that module's.
Queue []QueuedPublish `json:"queue"`
// Listed says how many of Total the answer lists.
Listed int `json:"listed"`
Acted map[string]uint64 `json:"acted"`
// ItemCount is how many publish items stand; Items the first of them by their next try (design 47 G A-457a), each with its
// tries and its next try; an item that ended says why.
ItemCount int `json:"item-count"`
Items []PublishItem `json:"items"`
}
// QueueCount is how many versions one holder's queue holds of one class, and when the oldest and the newest were queued.
type QueueCount struct {
Holder string `json:"holder"`
Class string `json:"class"`
Count int `json:"count"`
Oldest time.Time `json:"oldest"`
Newest time.Time `json:"newest"`
}
// PublishItemKeyPrefix opens the key of a publish item in the builder's re-ask state.
@@ -335,69 +436,109 @@ func PublishItemKey(module, version string) string {
return PublishItemKeyPrefix + EscapeToken(module) + "." + EscapeToken(version)
}
// AnswerQueue is builder_publish_queue over the two sections as the tools bundle reads them.
func AnswerQueue(queue, acted, reasks Reader) (Queued, error) {
out := Queued{Queue: []QueuedPublish{}, Acted: map[string]uint64{}, Items: []PublishItem{}}
keys, err := queue.Keys()
// AnswerQueue is builder_publish_queue over the three states as the tools bundle reads them (H-A-290 (5)): each state read in
// bulk (Reader.All), the answer counted and capped at PublishQueueListCap whatever the queue holds; module (and version) narrow the
// listed queue to one module's (one version's), so an item's class can be read.
func AnswerQueue(queue, acted, reasks Reader, module, version string) (Queued, error) {
out := Queued{Counts: []QueueCount{}, Queue: []QueuedPublish{}, Acted: map[string]uint64{}, Items: []PublishItem{}}
vals, err := queue.values()
if err != nil {
return out, fmt.Errorf("the state %s cannot be listed: %w", PublishQueueState, err)
return out, fmt.Errorf("the state %s cannot be read: %w", PublishQueueState, err)
}
for _, k := range keys {
raw, ok, err := queue.Get(k)
if err != nil || !ok {
counts := map[[2]string]*QueueCount{}
var listed []QueuedPublish
for _, raw := range vals {
var q QueuedPublish
if json.Unmarshal(raw, &q) != nil {
continue
}
var q QueuedPublish
if json.Unmarshal(raw, &q) == nil {
if q.Class == "" {
q.Class = PublishClassStart
}
out.Total++
c := counts[[2]string{q.Holder, q.Class}]
if c == nil {
c = &QueueCount{Holder: q.Holder, Class: q.Class, Oldest: q.Queued, Newest: q.Queued}
counts[[2]string{q.Holder, q.Class}] = c
}
c.Count++
if q.Queued.Before(c.Oldest) {
c.Oldest = q.Queued
}
if q.Queued.After(c.Newest) {
c.Newest = q.Queued
}
if (module == "" || q.Module == module) && (version == "" || q.Version == version) {
listed = append(listed, q)
}
}
for _, c := range counts {
out.Counts = append(out.Counts, *c)
}
sort.Slice(out.Counts, func(i, j int) bool {
if out.Counts[i].Holder != out.Counts[j].Holder {
return out.Counts[i].Holder < out.Counts[j].Holder
}
return out.Counts[i].Class < out.Counts[j].Class
})
sort.SliceStable(listed, func(i, j int) bool { return listed[i].Holder < listed[j].Holder })
byHolder := map[string][]QueuedPublish{}
var holders []string
for _, q := range listed {
if byHolder[q.Holder] == nil {
holders = append(holders, q.Holder)
}
byHolder[q.Holder] = append(byHolder[q.Holder], q)
}
for _, h := range holders {
qs := byHolder[h]
sortQueued(qs)
for _, q := range qs {
if len(out.Queue) == PublishQueueListCap {
break
}
out.Queue = append(out.Queue, q)
}
}
sort.SliceStable(out.Queue, func(i, j int) bool {
if out.Queue[i].Holder != out.Queue[j].Holder {
return out.Queue[i].Holder < out.Queue[j].Holder
}
return out.Queue[i].Queued.Before(out.Queue[j].Queued)
})
akeys, err := acted.Keys()
out.Listed = len(out.Queue)
avals, err := acted.values()
if err != nil {
return out, fmt.Errorf("the state %s cannot be listed: %w", ActedState, err)
return out, fmt.Errorf("the state %s cannot be read: %w", ActedState, err)
}
for _, k := range akeys {
raw, ok, err := acted.Get(k)
if err != nil || !ok {
continue
}
for k, raw := range avals {
var a Acted
if json.Unmarshal(raw, &a) == nil {
out.Acted[k] = a.Sequence
}
}
rkeys, err := reasks.Keys()
rvals, err := reasks.values()
if err != nil {
return out, fmt.Errorf("the re-ask state cannot be listed: %w", err)
return out, fmt.Errorf("the re-ask state cannot be read: %w", err)
}
for _, k := range rkeys {
for k, raw := range rvals {
if !strings.HasPrefix(k, PublishItemKeyPrefix) {
continue
}
raw, ok, err := reasks.Get(k)
if err != nil || !ok {
continue
}
var it PublishItem
if json.Unmarshal(raw, &it) == nil {
out.Items = append(out.Items, it)
}
}
out.ItemCount = len(out.Items)
sort.SliceStable(out.Items, func(i, j int) bool { return out.Items[i].NextTry.Before(out.Items[j].NextTry) })
if len(out.Items) > PublishQueueListCap {
out.Items = out.Items[:PublishQueueListCap]
}
return out, nil
}
// PublishQueueDescription is builder_publish_queue's description.
const PublishQueueDescription = "The versions the builder will publish again (its state " + PublishQueueState + ", design 47 G section 588): per registry " +
"seat holder, in the order they are sent, each with its artifacts' digests, when it was queued and whether its publish has begun; and the " +
"keys of the state " + ActedState + ", the last stream sequence the builder acted on for each subject; and the publish items (design 47 G " +
"A-457a), each version's tries and next try, and why one ended after its tries. Read-only."
const PublishQueueDescription = "The versions the builder will publish again (its state " + PublishQueueState + ", design 47 G section 588, H H-A-290): " +
"how many each registry seat holder's queue holds per class (build-ended-this-run, sent first, or asked-again-at-start) with the oldest and newest " +
"queue time; the first 50 in the order they are sent, each with its class, its artifacts' digests, when it was queued and whether its publish has " +
"begun (module, and version, list only that module's, so one item's class can be read); the keys of the state " + ActedState + ", the last stream " +
"sequence the builder acted on for each subject; and the publish items (design 47 G A-457a), counted, the first 50 by next try, each version's " +
"tries and next try, and why one ended after its tries. Read-only."
// PublishAgainDescription is builder_publish_again's description.
const PublishAgainDescription = "Publish reserved versions again (design 47 G section 588), which builds nothing: with module and version, that reserved " +
@@ -406,22 +547,49 @@ const PublishAgainDescription = "Publish reserved versions again (design 47 G se
"with the versions it queued; each outcome is said as package-published, or left in version-unreleased with the holder's words. A hand act, " +
"recorded in the hand-act log."
// Resume takes up, at the builder's start, every queued version again, a begun one sent anew (§588).
// Resume takes up, at the builder's start, every queued version again, a begun one sent anew (§588): a version whose reservation
// was withdrawn or taken over is dropped with its publish item ended (H-A-290 (2)), and one of the live class whose build ended
// before this run is of the start's now (H-A-290 (1)). It reads the reservations once.
func (r *Republisher) Resume(ctx context.Context) error {
keys, err := r.Queue.KeysOf(ctx, QueueFilter)
if err != nil {
return fmt.Errorf("the state %s cannot be listed: %w", PublishQueueState, err)
}
all, err := r.Reserved.All(ctx)
if err != nil {
return fmt.Errorf("the queued publishes are not taken up: %w", err)
}
reserved := make(map[string]ReservedVersion, len(all))
for _, v := range all {
reserved[v.Module+" "+v.Version] = v
}
holders := map[string]bool{}
for _, k := range keys {
raw, _, found, err := r.Queue.Get(ctx, k)
raw, rev, found, err := r.Queue.Get(ctx, k)
if err != nil || !found {
continue
}
var q QueuedPublish
if json.Unmarshal(raw, &q) == nil {
holders[q.Holder] = true
if json.Unmarshal(raw, &q) != nil {
continue
}
v, ok := reserved[q.Module+" "+q.Version]
if why := NotStanding(v, ok); why != "" {
if err := r.Queue.Delete(ctx, k); err != nil {
r.logf("%s cannot be deleted from %s: %v", k, PublishQueueState, err)
}
r.end(ctx, q.Module, q.Version, why)
continue
}
if class := r.classOf(v); q.Class != class {
q.Class = class
if raw, err := json.Marshal(q); err == nil {
if _, err := r.Queue.Update(ctx, k, raw, rev); err != nil {
r.logf("%s keeps its class %s at the start: %v", k, q.Class, err)
}
}
}
holders[q.Holder] = true
}
for h := range holders {
r.drainLater(ctx, h)
@@ -451,8 +619,9 @@ func (r *Republisher) drainLater(ctx context.Context, holder string) {
go run()
}
// Drain sends holder's queued versions one at a time, oldest first, each bounded by UnreleasedBound; a key is deleted when
// its outcome is said.
// Drain sends holder's queued versions in passes (H-A-290 (3)): a pass lists the holder's keys once, reads each once, and sends
// them in their order (the live class first, each class oldest first), PublishSendsAtOnce at a time; the keys are listed again
// only when the pass is spent. Each send is bounded by UnreleasedBound; a key is deleted when its outcome is said.
func (r *Republisher) Drain(ctx context.Context, holder string) {
for ctx.Err() == nil {
keys, err := r.Queue.KeysOf(ctx, EscapeToken(holder)+".*.*")
@@ -460,7 +629,7 @@ func (r *Republisher) Drain(ctx context.Context, holder string) {
r.logf("the queue of %s cannot be listed: %v", holder, err)
return
}
var items []QueuedPublish
items := make([]QueuedPublish, 0, len(keys))
for _, k := range keys {
if raw, _, found, err := r.Queue.Get(ctx, k); err == nil && found {
var q QueuedPublish
@@ -472,12 +641,27 @@ func (r *Republisher) Drain(ctx context.Context, holder string) {
if len(items) == 0 {
return
}
sort.SliceStable(items, func(i, j int) bool { return items[i].Queued.Before(items[j].Queued) })
r.send(ctx, items[0])
sortQueued(items)
sem := make(chan struct{}, PublishSendsAtOnce)
var wg sync.WaitGroup
for _, q := range items {
if ctx.Err() != nil {
break
}
sem <- struct{}{}
wg.Add(1)
go func() {
defer func() { <-sem; wg.Done() }()
r.send(ctx, q)
}()
}
wg.Wait()
}
}
// send publishes one queued version's artifacts and says the outcome, then deletes its key.
// send publishes one queued version's artifacts and says the outcome, then deletes its key. Every call it makes honours the one
// bound it is given, UnreleasedBound (a context's bound, no timer, ADR 0437 §8), and a send that returns without sending says why
// (H-A-290 (4)).
func (r *Republisher) send(ctx context.Context, q QueuedPublish) {
key := QueueKey(q.Holder, q.Module, q.Version)
defer func() {
@@ -485,18 +669,25 @@ func (r *Republisher) send(ctx context.Context, q QueuedPublish) {
r.logf("%s cannot be deleted from %s: %v", key, PublishQueueState, err)
}
}()
bound, cancel := context.WithTimeout(ctx, UnreleasedBound)
defer cancel()
if !q.Begun {
q.Begun = true
raw, _ := json.Marshal(q)
if _, rev, found, err := r.Queue.Get(ctx, key); err == nil && found {
_, _ = r.Queue.Update(ctx, key, raw, rev)
if _, rev, found, err := r.Queue.Get(bound, key); err == nil && found {
_, _ = r.Queue.Update(bound, key, raw, rev)
}
}
v, found, err := r.Reserved.Of(ctx, q.Module, q.Version)
if err != nil || !found {
return // released meanwhile, or dropped
v, found, err := r.Reserved.Of(bound, q.Module, q.Version)
if err != nil {
r.logf("publish again: %s %s is not sent to %s: its reservation cannot be read: %v; its publish item asks it again", q.Module, q.Version, q.Holder, err)
return
}
e, err := r.Entry(ctx, v.Fingerprint)
if why := NotStanding(v, found); why != "" {
r.end(ctx, q.Module, q.Version, why)
return
}
e, err := r.Entry(bound, v.Fingerprint)
if err != nil {
r.say(ctx, v, fmt.Sprintf("the catalog entry %s cannot be read: %v", v.Fingerprint, err))
return
@@ -512,10 +703,8 @@ func (r *Republisher) send(ctx context.Context, q QueuedPublish) {
r.say(ctx, v, ref+": the catalog entry names no registry seat for it")
return
}
bound, cancel := context.WithTimeout(ctx, UnreleasedBound) // a context's bound, no timer (ADR 0437 §8)
err := r.Send(bound, Publish{Seat: seat, Kind: kind, Name: name, Version: v.Version, Digest: q.Artifacts[ref], Fingerprint: v.Fingerprint,
Commit: v.Commit, Transfer: RepublishTransfer(q.Module, v.Version, strings.TrimPrefix(ref, q.Module+":"))})
cancel()
if err == nil {
r.logf("publish again: %s %s: %s sent to %s as %s %s, digest %s; the package-published is said when the seat says published", q.Module, q.Version, ref, seat, name, v.Version, q.Artifacts[ref])
}
@@ -523,7 +712,7 @@ func (r *Republisher) send(ctx context.Context, q QueuedPublish) {
r.logf("publish again: %s %s: %s refused by %s: %v", q.Module, q.Version, ref, seat, err)
words := err.Error()
if strings.Contains(words, strings.TrimSuffix(NotStoredWords, ": ")) {
if ok, merr := r.Reserved.MarkUnstored(ctx, q.Module, q.Version, ref, words); merr != nil {
if ok, merr := r.Reserved.MarkUnstored(bound, q.Module, q.Version, ref, words); merr != nil {
r.logf("%s %s: the unstored %s is not kept: %v", q.Module, q.Version, ref, merr)
} else if ok {
r.say(ctx, v, ref+" "+NotStoredWords+words)
@@ -5,8 +5,13 @@ import (
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"strings"
"sync"
"sync/atomic"
"testing"
"testing/synctest"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
@@ -15,10 +20,19 @@ import (
type queueMem struct {
*reserveMem
bytes uint64
// listings counts the listings of one holder's keys (a filter `<holder>.*.*`), what a drain's pass lists.
listings atomic.Int32
}
func (q *queueMem) Bytes(context.Context) (uint64, error) { return q.bytes, nil }
func (q *queueMem) KeysOf(ctx context.Context, filter string) ([]string, error) {
if strings.HasSuffix(filter, ".*.*") && filter != QueueFilter {
q.listings.Add(1)
}
return q.reserveMem.KeysOf(ctx, filter)
}
// republishBed is a Republisher over memory states: reservations of module m (a bundle by the oci seat and a Go package by
// the Go seat), one holder module `oci` that claims the oci seat, and a Send and a Unreleased that record.
type republishBed struct {
@@ -26,8 +40,10 @@ type republishBed struct {
reserved *reserveMem
queue *queueMem
now time.Time
mu sync.Mutex // sent and said, which a holder's sends at once write
sent []Publish
said []string
ended []string
refuse string // the words Send fails with, empty for none
// holders are the registry seats' holders the seat-holders view names (design 47 G A-416).
holders map[string][]string
@@ -48,6 +64,8 @@ func newRepublishBed(t *testing.T) *republishBed {
{Ref: m + ":lib", Kind: "package", Language: "go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/" + m + "/go"}}}}, nil
},
Send: func(_ context.Context, p Publish) error {
b.mu.Lock()
defer b.mu.Unlock()
b.sent = append(b.sent, p)
if b.refuse != "" {
return errors.New(b.refuse)
@@ -55,8 +73,15 @@ func newRepublishBed(t *testing.T) *republishBed {
return nil
},
Unreleased: func(_ context.Context, v ReservedVersion, why string) {
b.mu.Lock()
defer b.mu.Unlock()
b.said = append(b.said, v.Module+" "+v.Version+": "+why)
},
Ended: func(_ context.Context, module, version string) {
b.mu.Lock()
defer b.mu.Unlock()
b.ended = append(b.ended, module+" "+version)
},
Logf: t.Logf}
b.Holders = func(seat string) []string { return b.holders[seat] }
return b
@@ -190,7 +215,7 @@ func TestRepublish(t *testing.T) {
return json.RawMessage(`{"mode":"publish","module":"m","version":"0.1.0","tries":3,"next-try":"2026-11-17T01:00:00Z","why":"w"}`), true, nil
}}
none := Reader{Keys: func() ([]string, error) { return nil, nil }}
answer, err := AnswerQueue(none, none, items)
answer, err := AnswerQueue(none, none, items, "", "")
if err != nil || len(answer.Items) != 1 || answer.Items[0].Tries != 3 || answer.Items[0].Version != "0.1.0" {
t.Errorf("builder_publish_queue items = %+v (%v), want the one publish item with its tries", answer.Items, err)
}
@@ -235,3 +260,185 @@ func TestTheRegistriesRecordSaysWhetherAVersionIsHeld(t *testing.T) {
t.Errorf("a version with no missing artifact is judged %+v (%v), want all held without a read", j, err)
}
}
// queueAt writes a queued version of module m at holder oci directly, as a start finds it.
func (b *republishBed) queueAt(t *testing.T, v ReservedVersion, class string, at time.Time) {
t.Helper()
raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: v.Module, Version: v.Version, Artifacts: map[string]string{v.Module + ":prog": "sha256:aa"}, Queued: at, Class: class})
if _, err := b.queue.Create(context.Background(), QueueKey("oci", v.Module, v.Version), raw); err != nil {
t.Fatal(err)
}
}
// TestAHoldersQueueSendsLiveFirstInPasses: design 47 H H-A-290 (1)-(3) — a publish whose build ended in this run is sent before
// any of a build that ended before the start, each class oldest first; a pass lists the holder's keys once and sends them, at most
// four at a time; a build whose reservation was withdrawn or taken over never publishes, its key dropped and its publish item ended,
// at enqueue, at the pass and at the start.
func TestAHoldersQueueSendsLiveFirstInPasses(t *testing.T) {
ctx := context.Background()
t.Run("the live class first, at most four sends at once", func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
b := newRepublishBed(t)
var old []ReservedVersion
for _, m := range []string{"o1", "o2", "o3", "o4", "o5"} {
old = append(old, b.reserve(t, m))
}
b.Started = b.now
b.reserve(t, "live")
for i, v := range old {
b.queueAt(t, v, PublishClassStart, b.now.Add(time.Duration(i)*time.Second))
}
b.now = b.now.Add(time.Hour) // the live one is queued last, after every old one
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
release := make(chan struct{})
var arrived []string
var inflight, most int
send := b.Send
b.Send = func(ctx context.Context, p Publish) error {
b.mu.Lock()
arrived = append(arrived, p.Name)
inflight++
most = max(most, inflight)
b.mu.Unlock()
<-release
b.mu.Lock()
inflight--
b.mu.Unlock()
return send(ctx, p)
}
b.Go = func(f func()) { go f() }
if q, err := b.Republish(ctx, "live", ""); err != nil || len(q) != 1 {
t.Fatalf("queued %v (%v), want the live version", q, err)
}
synctest.Wait() // every send the bound lets run waits on release
b.mu.Lock()
first := slices.Clone(arrived)
b.mu.Unlock()
if len(first) != PublishSendsAtOnce || !slices.Contains(first, "live/prog") {
t.Errorf("the sends running at once are %v: want %d, the live one among them", first, PublishSendsAtOnce)
}
close(release)
synctest.Wait()
if len(b.sent) != 6 || most != PublishSendsAtOnce {
t.Errorf("sent %d, at most %d at once: want all six, %d at once", len(b.sent), most, PublishSendsAtOnce)
}
})
})
t.Run("a pass lists the keys once for all its sends", func(t *testing.T) {
b := newRepublishBed(t)
var vs []ReservedVersion
for i := range 6 {
vs = append(vs, b.reserve(t, fmt.Sprintf("m%d", i)))
}
for i, v := range vs {
b.queueAt(t, v, PublishClassStart, b.now.Add(time.Duration(i)*time.Second))
}
b.Drain(ctx, "oci")
if n := b.queue.listings.Load(); len(b.sent) != 6 || n != 2 {
t.Errorf("sent %d with %d listings, want six sends from one pass and one listing that finds it spent", len(b.sent), n)
}
})
t.Run("a start makes a live key of a build before it a start key", func(t *testing.T) {
b := newRepublishBed(t)
v := b.reserve(t, "m")
b.Started = b.now
b.queueAt(t, v, PublishClassLive, b.now)
b.Go = func(func()) {} // the start's drain is not run here
if err := b.Resume(ctx); err != nil {
t.Fatal(err)
}
raw, _, _, _ := b.queue.Get(ctx, QueueKey("oci", "m", v.Version))
var q QueuedPublish
if json.Unmarshal(raw, &q) != nil || q.Class != PublishClassStart {
t.Errorf("the key after the start is %s, want the class %s: its build did not end in this run", raw, PublishClassStart)
}
})
for _, at := range []string{"enqueue", "pass", "start"} {
t.Run("a withdrawn build never publishes, at the "+at, func(t *testing.T) {
b := newRepublishBed(t)
v := b.reserve(t, "m")
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
if _, ok, err := b.Reserved.Withdraw(ctx, "m", v.Version); err != nil || !ok {
t.Fatalf("withdrawn %v (%v)", ok, err)
}
switch at {
case "enqueue":
b.queueAt(t, v, PublishClassLive, b.now)
b.Go = func(func()) {} // a key queued before the withdrawal is dropped by the enqueue itself
if q, _ := b.Republish(ctx, "m", v.Version); len(q) != 0 {
t.Errorf("queued %v", q)
}
case "pass":
b.queueAt(t, v, PublishClassLive, b.now)
b.Drain(ctx, "oci")
case "start":
b.queueAt(t, v, PublishClassLive, b.now)
b.Go = func(func()) {}
if err := b.Resume(ctx); err != nil {
t.Fatal(err)
}
}
keys, _ := b.queue.KeysOf(ctx, QueueFilter)
if len(b.sent) != 0 || len(keys) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
t.Errorf("sent %v, keys %v, items ended %v: want nothing sent, the key dropped and m %s's item ended", b.sent, keys, b.ended, v.Version)
}
})
}
t.Run("a version taken over is not sent, and its item ended", func(t *testing.T) {
b := newRepublishBed(t)
v := b.reserve(t, "m")
b.queueAt(t, v, PublishClassStart, b.now)
if _, err := b.Reserved.Reserve(ctx, "m", "0.1", []string{"m:prog"}, Source{Fingerprint: "src2:other"}); err != nil {
t.Fatal(err)
}
b.Drain(ctx, "oci")
if len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
t.Errorf("sent %v, ended %v: want nothing sent and the item ended", b.sent, b.ended)
}
})
}
// TestThePublishQueueToolAnswersAtAnySize: design 47 H H-A-290 (5) — builder_publish_queue over two thousand queued versions answers
// within a second and small: counts per holder and class with the oldest and newest queue time, a list capped at fifty in the
// order they are sent, and one module's item with its class; each state read in bulk where the reader can.
func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) {
at := time.Date(2026, 11, 20, 17, 0, 0, 0, time.UTC)
vals := map[string]json.RawMessage{}
for i := range 2000 {
holder, class := "oci", PublishClassStart
if i%2 == 1 {
holder = "go"
}
if i >= 1990 {
class = PublishClassLive
}
q := QueuedPublish{V: 1, Holder: holder, Module: fmt.Sprintf("m%d", i), Version: "0.1.0", Artifacts: map[string]string{"a": "sha256:aa"}, Queued: at.Add(time.Duration(i) * time.Second), Class: class}
vals[QueueKey(holder, q.Module, q.Version)], _ = json.Marshal(q)
}
gets := 0
queue := Reader{All: func() (map[string]json.RawMessage, error) { return vals, nil },
Keys: func() ([]string, error) { return slices.Collect(maps.Keys(vals)), nil },
Get: func(k string) (json.RawMessage, bool, error) { gets++; v, ok := vals[k]; return v, ok, nil }}
none := Reader{Keys: func() ([]string, error) { return nil, nil }}
start := time.Now()
a, err := AnswerQueue(queue, none, none, "", "")
took := time.Since(start)
raw, _ := json.Marshal(a)
if err != nil || took > time.Second || a.Total != 2000 || len(a.Queue) != PublishQueueListCap || a.Listed != PublishQueueListCap || len(raw) > 64<<10 || gets != 0 {
t.Fatalf("answered %d of %d in %s, %d bytes, %d single reads (%v): want all counted, fifty listed, within a second, small, in bulk", len(a.Queue), a.Total, took, len(raw), gets, err)
}
if fmt.Sprintln(a.Counts[1].Holder, a.Counts[1].Class, a.Counts[1].Count, a.Counts[1].Oldest.Equal(at.Add(1991*time.Second)), a.Counts[1].Newest.Equal(at.Add(1999*time.Second))) != "go "+PublishClassLive+" 5 true true\n" || len(a.Counts) != 4 {
t.Errorf("counts %+v, want per holder and class with their oldest and newest", a.Counts)
}
if a.Queue[0].Class != PublishClassLive || a.Queue[0].Holder != "go" {
t.Errorf("the list opens with %+v, want the go holder's live class, sent first", a.Queue[0])
}
one, err := AnswerQueue(queue, none, none, "m1995", "")
if err != nil || len(one.Queue) != 1 || one.Queue[0].Class != PublishClassLive || one.Total != 2000 {
t.Errorf("one module's answer %+v (%v), want its item with its class", one.Queue, err)
}
queue.All = func() (map[string]json.RawMessage, error) { return nil, errors.ErrUnsupported }
if b, err := AnswerQueue(queue, none, none, "", ""); err != nil || b.Total != 2000 || gets != 2000 {
t.Errorf("a reader with no bulk read answered %d (%v) with %d single reads, want every key read one by one", b.Total, err, gets)
}
}
@@ -21,10 +21,36 @@ const (
OldFormClosedEvent = "version.old-form-closed"
)
// Reader reads the state as the tools bundle may: every key, and one.
// Reader reads the state as the tools bundle may: every key, and one; and, where All is set, every key's value in one bulk read
// (the state's pages), which a state of thousands of keys needs to be answered at all (design 47 H H-A-290 (5)).
type Reader struct {
Keys func() ([]string, error)
Get func(key string) (json.RawMessage, bool, error)
// All is every key's value read in bulk; nil, or an error satisfying errors.Is(err, errors.ErrUnsupported), reads key by key.
All func() (map[string]json.RawMessage, error)
}
// values is every key's value: in bulk where the reader can, else key by key.
func (r Reader) values() (map[string]json.RawMessage, error) {
if r.All != nil {
out, err := r.All()
if !errors.Is(err, errors.ErrUnsupported) {
return out, err
}
}
keys, err := r.Keys()
if err != nil {
return nil, err
}
out := make(map[string]json.RawMessage, len(keys))
for _, k := range keys {
raw, ok, err := r.Get(k)
if err != nil || !ok {
continue
}
out[k] = raw
}
return out, nil
}
// readStore is a Store over a Reader that refuses every write: the tools bundle only reads versions.