H-A-291: a start publishes only each module's newest done build; a live enqueue cuts a start list short

A start asked a publish again for every unreleased done build, older ones whose digests a newer reservation or release
of the same module made dead included, and a live publish queued during a long start pass waited for its end. A done
build a newer reservation or release (the builder's own record) supersedes is now retired at the start and at the pass:
its key dropped, its publish item ended, said in the journal; the start asks only each module's newest. A live enqueue
marks its holder, and a pass about to send a start-class item stops there, lets the sends in flight end and lists again,
so the live one goes next; a start-class enqueue never does.

Builds: design 47 H H-A-291 (1), (2), H-A-290, H-A-275
This commit is contained in:
2026-11-21 18:59:58 +01:00
parent bda61ae64c
commit 77fb6c4ab1
5 changed files with 231 additions and 11 deletions
@@ -304,10 +304,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return republish.Held(ctx, v, registryVersionsVia(calls.Call))
},
markHeld: reserved.markHeld, moveElsewhere: reserved.movedElsewhere,
allDone: func(ctx context.Context) ([]versions.ReservedVersion, error) {
all, err := reserved.r.All(ctx)
return builtOf(all), err
}}
allDone: func(ctx context.Context) ([]versions.ReservedVersion, error) { return reserved.r.All(ctx) },
retired: reserved.r.Retired} // only each module's newest done build is asked at the start (H-A-291 (1))
// A version that never publishes (withdrawn or taken over) has its publish item ended by the queue too (H-A-290 (2)).
republish.Ended = items.Done
reasksKV.Ready(func(jetstream.KeyValue) { items.AskAllDone(ctx) })
@@ -46,7 +46,11 @@ type publishItems struct {
reserved func(ctx context.Context, module, version string) (versions.ReservedVersion, bool, error)
republish func(ctx context.Context, module, version string) ([]string, error)
endedWith func(ctx context.Context, v versions.ReservedVersion, why string) // raises the finding version-unreleased
allDone func(ctx context.Context) ([]versions.ReservedVersion, error)
// allDone are the reservations, done or not: those whose build is done are asked at the start (builtOf).
allDone func(ctx context.Context) ([]versions.ReservedVersion, error)
// retired are the done builds a newer reservation or release of the same module supersedes, with why (H-A-291 (1)); nil
// retires none.
retired func(ctx context.Context, all []versions.ReservedVersion) (map[string]string, error)
// unheard is how many messages the builder's durable on the events stream holds pending and unacknowledged, read before each
// try (design 47 H H-A-269 (2), H-A-272); nil reads none.
unheard func(ctx context.Context) (int, error)
@@ -98,12 +102,23 @@ func (p *publishItems) Ask(ctx context.Context, module, version string) error {
// AskAllDone asks the publish item of every reservation whose build is done and that has no release: at the builder's start, so
// what stood before this change moves too, whatever its artifacts.
func (p *publishItems) AskAllDone(ctx context.Context) {
vs, err := p.allDone(ctx)
all, err := p.allDone(ctx)
if err != nil {
p.logf("the reservations of done builds are not asked to publish at the start: %v", err)
return
}
for _, v := range vs {
var retired map[string]string
if p.retired != nil {
if retired, err = p.retired(ctx, all); err != nil {
p.logf("the superseded builds are not retired at the start: %v", err)
}
}
for _, v := range builtOf(all) {
if why := retired[v.Module+" "+v.Version]; why != "" { // only each module's newest is asked (H-A-291 (1), H-A-275)
p.logf("%s %s is not asked to publish at the start: %s; its publish item is ended", v.Module, v.Version, why)
p.Done(ctx, v.Module, v.Version)
continue
}
if why := versions.NotStanding(v, true); why != "" { // withdrawn: it never publishes (H-A-290 (2))
p.logf("%s %s is not asked to publish at the start: %s; its publish item is ended", v.Module, v.Version, why)
p.Done(ctx, v.Module, v.Version)
@@ -4,8 +4,10 @@ import (
"context"
"encoding/json"
"errors"
"maps"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
@@ -148,6 +150,22 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) {
if _, ok := kv.vals[versions.PublishItemKey("w", "0.4.0")]; len(kv.vals) != 2 || ok {
t.Errorf("the start left %d items (withdrawn's kept: %v), want one per done unreleased reservation and none for a withdrawn one (H-A-290 (2))", len(kv.vals), ok)
}
// A start with three unreleased done versions of one module asks only the newest and retires the two (H-A-291 (1), H-A-275).
kv.vals, kv.revs = map[string][]byte{}, map[string]uint64{}
_ = p.Ask(ctx, "s", "0.1.3") // asked in an earlier run
p.allDone = func(context.Context) ([]versions.ReservedVersion, error) {
var out []versions.ReservedVersion
for _, ver := range []string{"0.1.3", "0.10.0", "0.2.1"} {
out = append(out, versions.ReservedVersion{Module: "s", Version: ver, Artifacts: map[string]string{"s:a": "sha256:a"}, BuiltAt: now})
}
return out, nil
}
p.retired = (&versions.Reserver{}).Retired
p.AskAllDone(ctx)
if _, ok := kv.vals[versions.PublishItemKey("s", "0.10.0")]; len(kv.vals) != 1 || !ok {
t.Errorf("the start left the items %v, want only the newest, s 0.10.0, asked and the two older retired", slices.Collect(maps.Keys(kv.vals)))
}
kv.vals, kv.revs = map[string][]byte{}, map[string]uint64{}
// A released version's item is deleted; a version no longer reserved has it deleted at its next try.
p.Done(ctx, "m", "0.1.2")
p.Done(ctx, "n", "0.2.0")
@@ -7,6 +7,7 @@ import (
"fmt"
"slices"
"sort"
"strconv"
"strings"
"sync"
"time"
@@ -159,6 +160,9 @@ type Republisher struct {
mu sync.Mutex
busy map[string]bool
// relist marks a holder whose pass lists again before its next start-class send: a live publish was queued meanwhile
// (H-A-291 (2)).
relist map[string]bool
// lastQueued is the newest time a version was queued at: a clock that has not moved between two versions (a coarse clock,
// or a fixed one) must not make the queue's order the order of its keys, so each is queued after the last.
lastQueued time.Time
@@ -211,6 +215,68 @@ func NotStanding(v ReservedVersion, found bool) string {
return ""
}
// newerVersion says whether MAJOR.MINOR.PATCH a is above b, number by number.
func newerVersion(a, b string) bool {
pa, pb := strings.Split(a, "."), strings.Split(b, ".")
if len(pa) != 3 || len(pb) != 3 {
return false
}
for i := range 3 {
x, errx := strconv.Atoi(pa[i])
y, erry := strconv.Atoi(pb[i])
if errx != nil || erry != nil {
return false
}
if x != y {
return x > y
}
}
return false
}
// Retired are the done builds without a release among all that a newer reservation or a newer release of the same module
// supersedes, each with why (design 47 H H-A-291 (1); H-A-275: the older digest is dead): keyed `<module> <version>`. The
// releases are the builder's own record of them (Releases), read once per module.
func (r *Reserver) Retired(ctx context.Context, all []ReservedVersion) (map[string]string, error) {
newest := map[string]string{} // module -> its newest reserved version
for _, v := range all {
if n, ok := newest[v.Module]; !ok || newerVersion(v.Version, n) {
newest[v.Module] = v.Version
}
}
released := map[string]string{} // module -> its newest release the record keeps
out := map[string]string{}
for _, v := range all {
if v.BuiltAt.IsZero() {
continue
}
if n := newest[v.Module]; newerVersion(n, v.Version) {
out[v.Module+" "+v.Version] = "the newer reservation " + n + " of " + v.Module + " supersedes it"
continue
}
rel, read := released[v.Module]
if !read && r.Releases != nil {
raw, _, found, err := r.Releases.Get(ctx, ReleasedKey(v.Module))
if err != nil {
return out, fmt.Errorf("the releases of %s cannot be read: %w", v.Module, err)
}
var rec ReleasedRecord
if found && json.Unmarshal(raw, &rec) == nil {
for _, k := range rec.Kept {
if rel == "" || newerVersion(k.Version, rel) {
rel = k.Version
}
}
}
released[v.Module] = rel
}
if rel != "" && newerVersion(rel, v.Version) {
out[v.Module+" "+v.Version] = "the newer release " + rel + " of " + v.Module + " supersedes it"
}
}
return out, nil
}
// drop deletes every queued key of module's version and ends its publish item: it never publishes (H-A-290 (2)), said with why.
func (r *Republisher) drop(ctx context.Context, module, version, why string) {
ctx = context.WithoutCancel(ctx)
@@ -352,6 +418,14 @@ func (r *Republisher) enqueue(ctx context.Context, v ReservedVersion, holder str
return false, fmt.Errorf("%s cannot be queued: %w", key, err)
}
r.logf("publish again: %s %s is queued to publish at %s: %d artifacts, class %s", v.Module, v.Version, holder, len(arts), r.classOf(v))
if r.classOf(v) == PublishClassLive { // a pass sending a start list lists again, so this one goes next (H-A-291 (2))
r.mu.Lock()
if r.relist == nil {
r.relist = map[string]bool{}
}
r.relist[holder] = true
r.mu.Unlock()
}
return true, nil
}
@@ -563,6 +637,10 @@ func (r *Republisher) Resume(ctx context.Context) error {
for _, v := range all {
reserved[v.Module+" "+v.Version] = v
}
retired, err := r.Reserved.Retired(ctx, all)
if err != nil {
r.logf("publish again: the superseded builds are not retired at the start: %v", err)
}
holders := map[string]bool{}
for _, k := range keys {
raw, rev, found, err := r.Queue.Get(ctx, k)
@@ -574,7 +652,11 @@ func (r *Republisher) Resume(ctx context.Context) error {
continue
}
v, ok := reserved[q.Module+" "+q.Version]
if why := NotStanding(v, ok); why != "" {
why := NotStanding(v, ok)
if why == "" {
why = retired[q.Module+" "+q.Version]
}
if why != "" {
if err := r.Queue.Delete(ctx, k); err != nil {
r.logf("%s cannot be deleted from %s: %v", k, PublishQueueState, err)
}
@@ -621,9 +703,14 @@ func (r *Republisher) drainLater(ctx context.Context, holder string) {
// Drain sends holder's queued versions in passes (H-A-290 (3)): a pass lists the holder's keys once, reads each once, and sends
// them in their order (the live class first, each class oldest first), PublishSendsAtOnce at a time; the keys are listed again
// only when the pass is spent. Each send is bounded by UnreleasedBound; a key is deleted when its outcome is said.
// only when the pass is spent, or when a live publish was queued while it sends a start list: then the sends in flight end and
// the keys are listed again, so the live one goes next (H-A-291 (2)); a start-class one queued meanwhile waits for the pass's end.
// Each send is bounded by UnreleasedBound; a key is deleted when its outcome is said.
func (r *Republisher) Drain(ctx context.Context, holder string) {
for ctx.Err() == nil {
r.mu.Lock()
delete(r.relist, holder)
r.mu.Unlock()
keys, err := r.Queue.KeysOf(ctx, EscapeToken(holder)+".*.*")
if err != nil {
r.logf("the queue of %s cannot be listed: %v", holder, err)
@@ -649,6 +736,14 @@ func (r *Republisher) Drain(ctx context.Context, holder string) {
break
}
sem <- struct{}{}
r.mu.Lock()
again := q.Class != PublishClassLive && r.relist[holder]
r.mu.Unlock()
if again {
<-sem
r.logf("publish again: %s's pass lists again after its sends in flight: a live publish was queued", holder)
break
}
wg.Add(1)
go func() {
defer func() { <-sem; wg.Done() }()
@@ -678,12 +773,31 @@ func (r *Republisher) send(ctx context.Context, q QueuedPublish) {
_, _ = r.Queue.Update(bound, key, raw, rev)
}
}
v, found, err := r.Reserved.Of(bound, q.Module, q.Version)
all, err := r.Reserved.All(bound)
if err != nil {
r.logf("publish again: %s %s is not sent to %s: its reservation cannot be read: %v; its publish item asks it again", q.Module, q.Version, q.Holder, err)
return
}
if why := NotStanding(v, found); why != "" {
var v ReservedVersion
found := false
var mine []ReservedVersion
for _, w := range all {
if w.Module == q.Module {
mine = append(mine, w)
if w.Version == q.Version {
v, found = w, true
}
}
}
why := NotStanding(v, found)
if why == "" {
retired, err := r.Reserved.Retired(bound, mine)
if err != nil {
r.logf("publish again: %s %s: whether a newer version supersedes it cannot be read, so it is sent: %v", q.Module, q.Version, err)
}
why = retired[q.Module+" "+q.Version]
}
if why != "" {
r.end(ctx, q.Module, q.Version, why)
return
}
@@ -384,6 +384,81 @@ func TestAHoldersQueueSendsLiveFirstInPasses(t *testing.T) {
}
})
}
t.Run("a live publish queued mid-pass goes after the sends in flight, before the rest of the start list", func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
b := newRepublishBed(t)
var old []ReservedVersion
for _, m := range []string{"o1", "o2", "o3", "o4", "o5", "o6"} {
old = append(old, b.reserve(t, m))
}
b.Started = b.now
b.reserve(t, "live")
for i, v := range old {
b.queueAt(t, v, PublishClassStart, b.now.Add(time.Duration(i)*time.Second))
}
b.now = b.now.Add(time.Hour)
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
first, then := make(chan struct{}), make(chan struct{}) // the four in flight wait on first, every later send on then
var arrived []string
send := b.Send
b.Send = func(ctx context.Context, p Publish) error {
b.mu.Lock()
arrived = append(arrived, p.Name)
gate := map[bool]chan struct{}{true: first, false: then}[len(arrived) <= PublishSendsAtOnce]
b.mu.Unlock()
<-gate
return send(ctx, p)
}
b.Go = func(f func()) { go f() }
b.drainLater(ctx, "oci")
synctest.Wait() // four start sends in flight, the pass holding o5 and o6
if q, err := b.Republish(ctx, "live", ""); err != nil || len(q) != 1 {
t.Fatalf("queued %v (%v)", q, err)
}
close(first)
synctest.Wait() // the sends after the four: with the live one among them, or o5 and o6 alone when the pass goes on
b.mu.Lock()
next := slices.Clone(arrived[PublishSendsAtOnce:])
b.mu.Unlock()
if !slices.Contains(next, "live/prog") {
t.Errorf("after the four in flight the sends are %v, want the live one among them, the pass listed again", next)
}
close(then)
synctest.Wait()
if len(arrived) != 7 {
t.Errorf("sent %v, want all seven", arrived)
}
})
})
t.Run("a build a newer reservation or release supersedes is retired at the pass and at the start", func(t *testing.T) {
b := newRepublishBed(t)
v := b.reserve(t, "m")
if _, err := b.Reserved.Reserve(ctx, "m", "0.2", []string{"m:prog"}, Source{Fingerprint: "src2:m"}); err != nil {
t.Fatal(err)
}
b.queueAt(t, v, PublishClassStart, b.now)
b.Drain(ctx, "oci")
if len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
t.Errorf("at the pass: sent %v, ended %v, want m %s retired", b.sent, b.ended, v.Version)
}
b.ended = nil
b.queueAt(t, v, PublishClassStart, b.now)
b.Go = func(func()) {}
if err := b.Resume(ctx); err != nil {
t.Fatal(err)
}
if keys, _ := b.queue.KeysOf(ctx, QueueFilter); len(keys) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
t.Errorf("at the start: keys %v, ended %v, want m %s retired", keys, b.ended, v.Version)
}
rel := &reserveMem{mem: newMem()}
raw, _ := json.Marshal(ReleasedRecord{Kept: []ReleasedBuild{{Fingerprint: "f", Version: "0.1.9"}}})
_, _ = rel.Create(ctx, ReleasedKey("n"), raw)
r := &Reserver{Releases: rel}
got, err := r.Retired(ctx, []ReservedVersion{{Module: "n", Version: "0.1.4", BuiltAt: b.now}, {Module: "p", Version: "0.1.12", BuiltAt: b.now}})
if err != nil || len(got) != 1 || !strings.Contains(got["n 0.1.4"], "release 0.1.9") {
t.Errorf("retired %v (%v), want n 0.1.4 by the release 0.1.9 and p 0.1.12 kept", got, err)
}
})
t.Run("a version taken over is not sent, and its item ended", func(t *testing.T) {
b := newRepublishBed(t)
v := b.reserve(t, "m")