diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 4d9532ef1..2bfff2677 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -304,10 +304,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return republish.Held(ctx, v, registryVersionsVia(calls.Call)) }, markHeld: reserved.markHeld, moveElsewhere: reserved.movedElsewhere, - allDone: func(ctx context.Context) ([]versions.ReservedVersion, error) { - all, err := reserved.r.All(ctx) - return builtOf(all), err - }} + allDone: func(ctx context.Context) ([]versions.ReservedVersion, error) { return reserved.r.All(ctx) }, + retired: reserved.r.Retired} // only each module's newest done build is asked at the start (H-A-291 (1)) // A version that never publishes (withdrawn or taken over) has its publish item ended by the queue too (H-A-290 (2)). republish.Ended = items.Done reasksKV.Ready(func(jetstream.KeyValue) { items.AskAllDone(ctx) }) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go index 0770d19ea..652064769 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go @@ -46,7 +46,11 @@ type publishItems struct { reserved func(ctx context.Context, module, version string) (versions.ReservedVersion, bool, error) republish func(ctx context.Context, module, version string) ([]string, error) endedWith func(ctx context.Context, v versions.ReservedVersion, why string) // raises the finding version-unreleased - allDone func(ctx context.Context) ([]versions.ReservedVersion, error) + // allDone are the reservations, done or not: those whose build is done are asked at the start (builtOf). + allDone func(ctx context.Context) ([]versions.ReservedVersion, error) + // retired are the done builds a newer reservation or release of the same module supersedes, with why (H-A-291 (1)); nil + // retires none. + retired func(ctx context.Context, all []versions.ReservedVersion) (map[string]string, error) // unheard is how many messages the builder's durable on the events stream holds pending and unacknowledged, read before each // try (design 47 H H-A-269 (2), H-A-272); nil reads none. unheard func(ctx context.Context) (int, error) @@ -98,12 +102,23 @@ func (p *publishItems) Ask(ctx context.Context, module, version string) error { // AskAllDone asks the publish item of every reservation whose build is done and that has no release: at the builder's start, so // what stood before this change moves too, whatever its artifacts. func (p *publishItems) AskAllDone(ctx context.Context) { - vs, err := p.allDone(ctx) + all, err := p.allDone(ctx) if err != nil { p.logf("the reservations of done builds are not asked to publish at the start: %v", err) return } - for _, v := range vs { + var retired map[string]string + if p.retired != nil { + if retired, err = p.retired(ctx, all); err != nil { + p.logf("the superseded builds are not retired at the start: %v", err) + } + } + for _, v := range builtOf(all) { + if why := retired[v.Module+" "+v.Version]; why != "" { // only each module's newest is asked (H-A-291 (1), H-A-275) + p.logf("%s %s is not asked to publish at the start: %s; its publish item is ended", v.Module, v.Version, why) + p.Done(ctx, v.Module, v.Version) + continue + } if why := versions.NotStanding(v, true); why != "" { // withdrawn: it never publishes (H-A-290 (2)) p.logf("%s %s is not asked to publish at the start: %s; its publish item is ended", v.Module, v.Version, why) p.Done(ctx, v.Module, v.Version) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem_test.go index f96193ffe..8a70df53f 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem_test.go @@ -4,8 +4,10 @@ import ( "context" "encoding/json" "errors" + "maps" "os" "path/filepath" + "slices" "strings" "testing" "time" @@ -148,6 +150,22 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) { if _, ok := kv.vals[versions.PublishItemKey("w", "0.4.0")]; len(kv.vals) != 2 || ok { t.Errorf("the start left %d items (withdrawn's kept: %v), want one per done unreleased reservation and none for a withdrawn one (H-A-290 (2))", len(kv.vals), ok) } + // A start with three unreleased done versions of one module asks only the newest and retires the two (H-A-291 (1), H-A-275). + kv.vals, kv.revs = map[string][]byte{}, map[string]uint64{} + _ = p.Ask(ctx, "s", "0.1.3") // asked in an earlier run + p.allDone = func(context.Context) ([]versions.ReservedVersion, error) { + var out []versions.ReservedVersion + for _, ver := range []string{"0.1.3", "0.10.0", "0.2.1"} { + out = append(out, versions.ReservedVersion{Module: "s", Version: ver, Artifacts: map[string]string{"s:a": "sha256:a"}, BuiltAt: now}) + } + return out, nil + } + p.retired = (&versions.Reserver{}).Retired + p.AskAllDone(ctx) + if _, ok := kv.vals[versions.PublishItemKey("s", "0.10.0")]; len(kv.vals) != 1 || !ok { + t.Errorf("the start left the items %v, want only the newest, s 0.10.0, asked and the two older retired", slices.Collect(maps.Keys(kv.vals))) + } + kv.vals, kv.revs = map[string][]byte{}, map[string]uint64{} // A released version's item is deleted; a version no longer reserved has it deleted at its next try. p.Done(ctx, "m", "0.1.2") p.Done(ctx, "n", "0.2.0") diff --git a/modules/mesh-delivery-builder/internal/versions/republish.go b/modules/mesh-delivery-builder/internal/versions/republish.go index dea1f3967..922f26d84 100644 --- a/modules/mesh-delivery-builder/internal/versions/republish.go +++ b/modules/mesh-delivery-builder/internal/versions/republish.go @@ -7,6 +7,7 @@ import ( "fmt" "slices" "sort" + "strconv" "strings" "sync" "time" @@ -159,6 +160,9 @@ type Republisher struct { mu sync.Mutex busy map[string]bool + // relist marks a holder whose pass lists again before its next start-class send: a live publish was queued meanwhile + // (H-A-291 (2)). + relist map[string]bool // lastQueued is the newest time a version was queued at: a clock that has not moved between two versions (a coarse clock, // or a fixed one) must not make the queue's order the order of its keys, so each is queued after the last. lastQueued time.Time @@ -211,6 +215,68 @@ func NotStanding(v ReservedVersion, found bool) string { return "" } +// newerVersion says whether MAJOR.MINOR.PATCH a is above b, number by number. +func newerVersion(a, b string) bool { + pa, pb := strings.Split(a, "."), strings.Split(b, ".") + if len(pa) != 3 || len(pb) != 3 { + return false + } + for i := range 3 { + x, errx := strconv.Atoi(pa[i]) + y, erry := strconv.Atoi(pb[i]) + if errx != nil || erry != nil { + return false + } + if x != y { + return x > y + } + } + return false +} + +// Retired are the done builds without a release among all that a newer reservation or a newer release of the same module +// supersedes, each with why (design 47 H H-A-291 (1); H-A-275: the older digest is dead): keyed ` `. The +// releases are the builder's own record of them (Releases), read once per module. +func (r *Reserver) Retired(ctx context.Context, all []ReservedVersion) (map[string]string, error) { + newest := map[string]string{} // module -> its newest reserved version + for _, v := range all { + if n, ok := newest[v.Module]; !ok || newerVersion(v.Version, n) { + newest[v.Module] = v.Version + } + } + released := map[string]string{} // module -> its newest release the record keeps + out := map[string]string{} + for _, v := range all { + if v.BuiltAt.IsZero() { + continue + } + if n := newest[v.Module]; newerVersion(n, v.Version) { + out[v.Module+" "+v.Version] = "the newer reservation " + n + " of " + v.Module + " supersedes it" + continue + } + rel, read := released[v.Module] + if !read && r.Releases != nil { + raw, _, found, err := r.Releases.Get(ctx, ReleasedKey(v.Module)) + if err != nil { + return out, fmt.Errorf("the releases of %s cannot be read: %w", v.Module, err) + } + var rec ReleasedRecord + if found && json.Unmarshal(raw, &rec) == nil { + for _, k := range rec.Kept { + if rel == "" || newerVersion(k.Version, rel) { + rel = k.Version + } + } + } + released[v.Module] = rel + } + if rel != "" && newerVersion(rel, v.Version) { + out[v.Module+" "+v.Version] = "the newer release " + rel + " of " + v.Module + " supersedes it" + } + } + return out, nil +} + // drop deletes every queued key of module's version and ends its publish item: it never publishes (H-A-290 (2)), said with why. func (r *Republisher) drop(ctx context.Context, module, version, why string) { ctx = context.WithoutCancel(ctx) @@ -352,6 +418,14 @@ func (r *Republisher) enqueue(ctx context.Context, v ReservedVersion, holder str return false, fmt.Errorf("%s cannot be queued: %w", key, err) } r.logf("publish again: %s %s is queued to publish at %s: %d artifacts, class %s", v.Module, v.Version, holder, len(arts), r.classOf(v)) + if r.classOf(v) == PublishClassLive { // a pass sending a start list lists again, so this one goes next (H-A-291 (2)) + r.mu.Lock() + if r.relist == nil { + r.relist = map[string]bool{} + } + r.relist[holder] = true + r.mu.Unlock() + } return true, nil } @@ -563,6 +637,10 @@ func (r *Republisher) Resume(ctx context.Context) error { for _, v := range all { reserved[v.Module+" "+v.Version] = v } + retired, err := r.Reserved.Retired(ctx, all) + if err != nil { + r.logf("publish again: the superseded builds are not retired at the start: %v", err) + } holders := map[string]bool{} for _, k := range keys { raw, rev, found, err := r.Queue.Get(ctx, k) @@ -574,7 +652,11 @@ func (r *Republisher) Resume(ctx context.Context) error { continue } v, ok := reserved[q.Module+" "+q.Version] - if why := NotStanding(v, ok); why != "" { + why := NotStanding(v, ok) + if why == "" { + why = retired[q.Module+" "+q.Version] + } + if why != "" { if err := r.Queue.Delete(ctx, k); err != nil { r.logf("%s cannot be deleted from %s: %v", k, PublishQueueState, err) } @@ -621,9 +703,14 @@ func (r *Republisher) drainLater(ctx context.Context, holder string) { // Drain sends holder's queued versions in passes (H-A-290 (3)): a pass lists the holder's keys once, reads each once, and sends // them in their order (the live class first, each class oldest first), PublishSendsAtOnce at a time; the keys are listed again -// only when the pass is spent. Each send is bounded by UnreleasedBound; a key is deleted when its outcome is said. +// only when the pass is spent, or when a live publish was queued while it sends a start list: then the sends in flight end and +// the keys are listed again, so the live one goes next (H-A-291 (2)); a start-class one queued meanwhile waits for the pass's end. +// Each send is bounded by UnreleasedBound; a key is deleted when its outcome is said. func (r *Republisher) Drain(ctx context.Context, holder string) { for ctx.Err() == nil { + r.mu.Lock() + delete(r.relist, holder) + r.mu.Unlock() keys, err := r.Queue.KeysOf(ctx, EscapeToken(holder)+".*.*") if err != nil { r.logf("the queue of %s cannot be listed: %v", holder, err) @@ -649,6 +736,14 @@ func (r *Republisher) Drain(ctx context.Context, holder string) { break } sem <- struct{}{} + r.mu.Lock() + again := q.Class != PublishClassLive && r.relist[holder] + r.mu.Unlock() + if again { + <-sem + r.logf("publish again: %s's pass lists again after its sends in flight: a live publish was queued", holder) + break + } wg.Add(1) go func() { defer func() { <-sem; wg.Done() }() @@ -678,12 +773,31 @@ func (r *Republisher) send(ctx context.Context, q QueuedPublish) { _, _ = r.Queue.Update(bound, key, raw, rev) } } - v, found, err := r.Reserved.Of(bound, q.Module, q.Version) + all, err := r.Reserved.All(bound) if err != nil { r.logf("publish again: %s %s is not sent to %s: its reservation cannot be read: %v; its publish item asks it again", q.Module, q.Version, q.Holder, err) return } - if why := NotStanding(v, found); why != "" { + var v ReservedVersion + found := false + var mine []ReservedVersion + for _, w := range all { + if w.Module == q.Module { + mine = append(mine, w) + if w.Version == q.Version { + v, found = w, true + } + } + } + why := NotStanding(v, found) + if why == "" { + retired, err := r.Reserved.Retired(bound, mine) + if err != nil { + r.logf("publish again: %s %s: whether a newer version supersedes it cannot be read, so it is sent: %v", q.Module, q.Version, err) + } + why = retired[q.Module+" "+q.Version] + } + if why != "" { r.end(ctx, q.Module, q.Version, why) return } diff --git a/modules/mesh-delivery-builder/internal/versions/republish_test.go b/modules/mesh-delivery-builder/internal/versions/republish_test.go index 37d0f9f7e..6966e2a74 100644 --- a/modules/mesh-delivery-builder/internal/versions/republish_test.go +++ b/modules/mesh-delivery-builder/internal/versions/republish_test.go @@ -384,6 +384,81 @@ func TestAHoldersQueueSendsLiveFirstInPasses(t *testing.T) { } }) } + t.Run("a live publish queued mid-pass goes after the sends in flight, before the rest of the start list", func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + b := newRepublishBed(t) + var old []ReservedVersion + for _, m := range []string{"o1", "o2", "o3", "o4", "o5", "o6"} { + old = append(old, b.reserve(t, m)) + } + b.Started = b.now + b.reserve(t, "live") + for i, v := range old { + b.queueAt(t, v, PublishClassStart, b.now.Add(time.Duration(i)*time.Second)) + } + b.now = b.now.Add(time.Hour) + b.holders = map[string][]string{SeatArtifacts: {"oci"}} + first, then := make(chan struct{}), make(chan struct{}) // the four in flight wait on first, every later send on then + var arrived []string + send := b.Send + b.Send = func(ctx context.Context, p Publish) error { + b.mu.Lock() + arrived = append(arrived, p.Name) + gate := map[bool]chan struct{}{true: first, false: then}[len(arrived) <= PublishSendsAtOnce] + b.mu.Unlock() + <-gate + return send(ctx, p) + } + b.Go = func(f func()) { go f() } + b.drainLater(ctx, "oci") + synctest.Wait() // four start sends in flight, the pass holding o5 and o6 + if q, err := b.Republish(ctx, "live", ""); err != nil || len(q) != 1 { + t.Fatalf("queued %v (%v)", q, err) + } + close(first) + synctest.Wait() // the sends after the four: with the live one among them, or o5 and o6 alone when the pass goes on + b.mu.Lock() + next := slices.Clone(arrived[PublishSendsAtOnce:]) + b.mu.Unlock() + if !slices.Contains(next, "live/prog") { + t.Errorf("after the four in flight the sends are %v, want the live one among them, the pass listed again", next) + } + close(then) + synctest.Wait() + if len(arrived) != 7 { + t.Errorf("sent %v, want all seven", arrived) + } + }) + }) + t.Run("a build a newer reservation or release supersedes is retired at the pass and at the start", func(t *testing.T) { + b := newRepublishBed(t) + v := b.reserve(t, "m") + if _, err := b.Reserved.Reserve(ctx, "m", "0.2", []string{"m:prog"}, Source{Fingerprint: "src2:m"}); err != nil { + t.Fatal(err) + } + b.queueAt(t, v, PublishClassStart, b.now) + b.Drain(ctx, "oci") + if len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" { + t.Errorf("at the pass: sent %v, ended %v, want m %s retired", b.sent, b.ended, v.Version) + } + b.ended = nil + b.queueAt(t, v, PublishClassStart, b.now) + b.Go = func(func()) {} + if err := b.Resume(ctx); err != nil { + t.Fatal(err) + } + if keys, _ := b.queue.KeysOf(ctx, QueueFilter); len(keys) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" { + t.Errorf("at the start: keys %v, ended %v, want m %s retired", keys, b.ended, v.Version) + } + rel := &reserveMem{mem: newMem()} + raw, _ := json.Marshal(ReleasedRecord{Kept: []ReleasedBuild{{Fingerprint: "f", Version: "0.1.9"}}}) + _, _ = rel.Create(ctx, ReleasedKey("n"), raw) + r := &Reserver{Releases: rel} + got, err := r.Retired(ctx, []ReservedVersion{{Module: "n", Version: "0.1.4", BuiltAt: b.now}, {Module: "p", Version: "0.1.12", BuiltAt: b.now}}) + if err != nil || len(got) != 1 || !strings.Contains(got["n 0.1.4"], "release 0.1.9") { + t.Errorf("retired %v (%v), want n 0.1.4 by the release 0.1.9 and p 0.1.12 kept", got, err) + } + }) t.Run("a version taken over is not sent, and its item ended", func(t *testing.T) { b := newRepublishBed(t) v := b.reserve(t, "m")