Builder: review round 2 of the publish items and the release record (A-456b)
QueueDone removed; an item is deleted only with its release, drop or takeover; releases kept one key per module (released-kept, 16); a plan that cannot be read ends the step could not run; the wiring of the items to the reservations and the re-ask watch is tested. Builds: design 47 G A-456b, A-457a, A-456a
This commit is contained in:
@@ -248,6 +248,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
releasedKept, err := numberFile(getenv("MESH_BUILDER_RELEASED_KEPT_FILE"), versions.DefaultReleasedKept)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reasksKV := process.AwaitBucket(ctx, js, conn, builds.StateBucket(cred.Module, build.ReAsksState), "builder_ask_again is not heard; an ask that can no longer be answered is still asked again by itself", logger.Printf)
|
||||
// A version's publish is the builder's own work until its release is written (design 47 G A-457a): an item of the re-ask state.
|
||||
items := &publishItems{reasks: reasksKV.KV, set: publishSet, now: time.Now, logf: logger.Printf,
|
||||
@@ -259,15 +263,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
all, err := reserved.r.All(ctx)
|
||||
var out []versions.ReservedVersion
|
||||
for _, v := range all {
|
||||
if !v.BuiltAt.IsZero() && len(v.Missing()) > 0 {
|
||||
if !v.BuiltAt.IsZero() {
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
return out, err
|
||||
}}
|
||||
reserved.built, reserved.released = items.Ask, items.Done
|
||||
reasksKV.Ready(func(jetstream.KeyValue) { items.AskAllDone(ctx) })
|
||||
reserved.r.Releases = awaitedReserved{awaitedVersions{b: actedKV, bucket: own.acted}} // the record of the releases it saw (design 47 G A-456a)
|
||||
wirePublishItems(reserved, items, awaitedReserved{awaitedVersions{b: actedKV, bucket: own.acted}}, releasedKept)
|
||||
acts.Republish = republish.Republish
|
||||
// builder_reserve_done: a done trunk build that reserved none is given a version, and its publish queued as publish again does.
|
||||
acts.ReserveDone = reserveDoneAct(b.ReserveDone, republish.Republish, logger.Printf)
|
||||
|
||||
@@ -86,7 +86,7 @@ func (p *publishItems) Ask(ctx context.Context, module, version string) error {
|
||||
}
|
||||
|
||||
// AskAllDone asks the publish item of every reservation whose build is done and that has no release: at the builder's start, so
|
||||
// what stood before this change moves too.
|
||||
// what stood before this change moves too, whatever its artifacts.
|
||||
func (p *publishItems) AskAllDone(ctx context.Context) {
|
||||
vs, err := p.allDone(ctx)
|
||||
if err != nil {
|
||||
@@ -112,8 +112,8 @@ func (p *publishItems) Done(ctx context.Context, module, version string) {
|
||||
}
|
||||
|
||||
// Try waits for the item's next try, then, while the version is still reserved, queues its publish and writes the item again with
|
||||
// one more try and the pause after it; once the tries are spent it ends the item loudly. A version no longer reserved (released or
|
||||
// dropped) has its item deleted.
|
||||
// one more try and the pause after it; once the tries are spent it ends the item loudly. A version no longer reserved (released,
|
||||
// dropped or taken over) has its item deleted; one whose artifacts are published keeps it until its release is written.
|
||||
func (p *publishItems) Try(ctx context.Context, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk) {
|
||||
if !p.wait(ctx, r.NextTry.Sub(p.now())) {
|
||||
return
|
||||
@@ -127,7 +127,7 @@ func (p *publishItems) Try(ctx context.Context, kv jetstream.KeyValue, e jetstre
|
||||
p.logf("%s %s: its reservation cannot be read, the publish item waits: %v", r.Module, r.Version, err)
|
||||
return
|
||||
}
|
||||
if !held || len(v.Missing()) == 0 {
|
||||
if !held { // dropped, taken over or released: never because its artifacts are published (A-456b)
|
||||
_ = kv.Delete(ctx, e.Key())
|
||||
return
|
||||
}
|
||||
@@ -195,3 +195,10 @@ func publishSettingsOf(getenv func(string) string) (publishSettings, error) {
|
||||
}
|
||||
return publishSettings{Pause: time.Duration(pause) * time.Second, PauseMax: time.Duration(pmax) * time.Minute, Tries: tries}, nil
|
||||
}
|
||||
|
||||
// wirePublishItems joins the reservations to the publish items (design 47 G A-457a, A-456a, A-456b): a done build asks its item, a
|
||||
// release, a drop and a takeover end it, and a release is recorded in releases, the last kept of each module.
|
||||
func wirePublishItems(rv *reservations, items *publishItems, releases versions.ReserveStore, kept int) {
|
||||
rv.built, rv.released = items.Ask, items.Done
|
||||
rv.r.ReleasedKept, rv.r.Releases = kept, releases
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -11,6 +12,9 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
@@ -166,3 +170,80 @@ func TestThePublishSettingsAreReadFromTheirFiles(t *testing.T) {
|
||||
t.Errorf("a content that is no number: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestThePublishItemsAreWiredToTheReservationsAndTheReAskWatch: design 47 G A-457a, A-456b — a mode-publish value of the re-ask state
|
||||
// reaches the item's try through reAsked; a done build gets an item; the catalog's version-released, a drop and a takeover delete it
|
||||
// and the release is kept by module.
|
||||
func TestThePublishItemsAreWiredToTheReservationsAndTheReAskWatch(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}}
|
||||
st := &memFindings{vals: map[string][]byte{}}
|
||||
found := &findings{module: builds.BuilderModule, w: st, logf: t.Logf, now: func() time.Time { return time.Unix(0, 0) }}
|
||||
call := func(string, any) (json.RawMessage, error) {
|
||||
return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"})
|
||||
}
|
||||
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, found, nil, t.Logf)
|
||||
rv.filled.Store(true)
|
||||
tried := make(chan string, 4)
|
||||
items := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: defaultPublishSettings, now: time.Now, logf: t.Logf,
|
||||
reserved: func(ctx context.Context, m, v string) (versions.ReservedVersion, bool, error) {
|
||||
return rv.r.Of(ctx, m, v)
|
||||
},
|
||||
republish: func(_ context.Context, m, v string) ([]string, error) { tried <- m + " " + v; return nil, nil }}
|
||||
releases := &memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}
|
||||
wirePublishItems(rv, items, releases, 2)
|
||||
done := func(fp string) versions.ReservedVersion {
|
||||
v, err := rv.Reserve(ctx, "m", "0.1", []string{"m:a"}, versions.Source{Fingerprint: fp})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rv.Built(ctx, "m", v.Version, map[string]string{"m:a": "sha256:a"}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
first := done("src2:one")
|
||||
key := versions.PublishItemKey("m", first.Version)
|
||||
if _, ok := kv.vals[key]; !ok {
|
||||
t.Fatalf("a done build has no publish item: %v", kv.vals)
|
||||
}
|
||||
// The watch hands the item to its try, which queues the publish.
|
||||
e, _ := kv.Get(ctx, key)
|
||||
reAsked(ctx, nil, versions.Acts{}, items, kv, e, t.Logf)
|
||||
select {
|
||||
case got := <-tried:
|
||||
if got != "m "+first.Version {
|
||||
t.Errorf("tried %q", got)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("a mode-publish value was not handed to the item's try")
|
||||
}
|
||||
// A takeover by another build deletes the older's item and asks the newer's.
|
||||
second := done("src2:two")
|
||||
if _, ok := kv.vals[key]; ok {
|
||||
t.Errorf("the taken-over reservation kept its item")
|
||||
}
|
||||
secondKey := versions.PublishItemKey("m", second.Version)
|
||||
if _, ok := kv.vals[secondKey]; !ok {
|
||||
t.Fatalf("the newer build has no item: %v", kv.vals)
|
||||
}
|
||||
// version-released deletes the item and keeps the release.
|
||||
msg := &wakeMsg{subject: meshcatalog.SubjectVersionReleased, data: []byte(`{"v":1,"module":"m","version":"` + second.Version + `","source":1,"artifacts":[]}`)}
|
||||
if !rv.heard(ctx, msg) || !msg.acked {
|
||||
t.Fatal("version-released was not taken in")
|
||||
}
|
||||
if _, ok := kv.vals[secondKey]; ok {
|
||||
t.Errorf("a released version kept its publish item")
|
||||
}
|
||||
if ok, _ := rv.ReleasedFor(ctx, "m", "src2:two"); !ok {
|
||||
t.Errorf("the release is not kept by fingerprint")
|
||||
}
|
||||
// A drop deletes the item too.
|
||||
third := done("src2:three")
|
||||
if _, err := rv.dropped(ctx, "m", "0.1", "op", "never", time.Unix(0, 0)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := kv.vals[versions.PublishItemKey("m", third.Version)]; ok {
|
||||
t.Errorf("a dropped reservation kept its item")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,7 +82,14 @@ func newReservations(store versions.ReserveStore, call toolCaller, bounds func()
|
||||
// took is a build's takeover of the reservation of an older build never released (design 47 G A-455): the older version is
|
||||
// dropped, its bound ended and its finding cleared (the newer build's reservation has its own), and the builder's log says so.
|
||||
func (rv *reservations) took(ctx context.Context, older, newer versions.ReservedVersion) {
|
||||
rv.logf("%s: the reservation of %s (build of %s, never released) is dropped: the build of %s takes the series over and reserves %s", newer.Module, older.Version, older.Fingerprint, newer.Fingerprint, newer.Version)
|
||||
if older.Fingerprint == newer.Fingerprint {
|
||||
rv.logf("%s: the build of %s reserves again: %s takes the place of %s", newer.Module, newer.Fingerprint, newer.Version, older.Version)
|
||||
} else {
|
||||
rv.logf("%s: the reservation of %s (build of %s, never released) is dropped: the build of %s takes the series over and reserves %s", newer.Module, older.Version, older.Fingerprint, newer.Fingerprint, newer.Version)
|
||||
}
|
||||
if rv.released != nil {
|
||||
rv.released(ctx, older.Module, older.Version) // its publish item goes with its reservation (A-456b)
|
||||
}
|
||||
rv.dropBound(ctx, versions.ReservedKey(older.Module, older.Series))
|
||||
rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, older.Module)
|
||||
}
|
||||
@@ -122,7 +129,7 @@ func (rv *reservations) Built(ctx context.Context, module, version string, diges
|
||||
return err
|
||||
}
|
||||
rv.arm(ctx, cur)
|
||||
if rv.built != nil && len(cur.Missing()) > 0 {
|
||||
if rv.built != nil {
|
||||
if err := rv.built(ctx, module, version); err != nil {
|
||||
rv.logf("%v", err)
|
||||
}
|
||||
@@ -283,10 +290,14 @@ func (rv *reservations) sayPublished(ctx context.Context, m versions.Marked, dig
|
||||
|
||||
// dropped is builder_reserved_drop: the reservation is deleted and its bound ended.
|
||||
func (rv *reservations) dropped(ctx context.Context, module, series, by, why string, at time.Time) (string, error) {
|
||||
cur, held, _ := rv.r.At(ctx, versions.ReservedKey(module, series))
|
||||
ok, err := rv.r.Drop(ctx, module, series)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if ok && held && rv.released != nil {
|
||||
rv.released(ctx, module, cur.Version) // its publish item goes with its reservation (A-456b)
|
||||
}
|
||||
if !ok {
|
||||
return "", fmt.Errorf("%s holds no reservation of %s", module, series)
|
||||
}
|
||||
|
||||
@@ -104,7 +104,10 @@ func (b *Builder) passedWithItsVersion(ctx context.Context, entry meshcatalog.Ca
|
||||
return meshbuilds.StepEndedOutcomePassed, ""
|
||||
}
|
||||
plan, planErr, _ := b.planOf(ctx, meshcatalog.BareManifestDigest(entry.ManifestDigest))
|
||||
if planErr != nil || len(plan.Tasks) == 0 || strings.Count(plan.Series, ".") != 1 {
|
||||
switch {
|
||||
case planErr != nil:
|
||||
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("%s at %s: the plan cannot be read to judge its version: %v", entry.Module, entry.Fp, planErr)
|
||||
case len(plan.Tasks) == 0 || strings.Count(plan.Series, ".") != 1:
|
||||
return meshbuilds.StepEndedOutcomePassed, ""
|
||||
}
|
||||
_, held, err := b.Reserved.ReservedFor(ctx, entry.Module, entry.Fp)
|
||||
|
||||
@@ -156,23 +156,6 @@ func (r *Republisher) logf(f string, a ...any) {
|
||||
// ActedKey is the acted key of a subject.
|
||||
func ActedKey(subject string) string { return EscapeToken(subject) }
|
||||
|
||||
// QueueDone queues every reserved version whose build is done and that has an artifact unpublished (design 47 G A-457a): asked at
|
||||
// the builder's start, so what stood before the change moves too, and when a build ends. At most Max versions at once, newest
|
||||
// first; a version queued already is not queued again. It answers the versions queued.
|
||||
func (r *Republisher) QueueDone(ctx context.Context) ([]string, error) {
|
||||
all, err := r.Reserved.All(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var vs []ReservedVersion
|
||||
for _, v := range all {
|
||||
if !v.BuiltAt.IsZero() && len(v.Missing()) > 0 {
|
||||
vs = append(vs, v)
|
||||
}
|
||||
}
|
||||
return r.queue(ctx, vs, r.holdersFn, r.Max)
|
||||
}
|
||||
|
||||
// holdersFn answers a seat's holders from Holders.
|
||||
func (r *Republisher) holdersFn(seat string) []string {
|
||||
if r.Holders == nil {
|
||||
@@ -429,7 +412,7 @@ const PublishQueueDescription = "The versions the builder will publish again (it
|
||||
// PublishAgainDescription is builder_publish_again's description.
|
||||
const PublishAgainDescription = "Publish reserved versions again (design 47 G section 588), which builds nothing: with module and version, that reserved " +
|
||||
"version's missing artifacts; with module alone, each reserved version of that module whose artifacts are not all published; with neither, every " +
|
||||
"version the finding version-unreleased names, newest first. It queues them as the builder does on a registry holder's running, and answers at once " +
|
||||
"version the finding version-unreleased names, newest first. It queues them as the builder's publish items do, and answers at once " +
|
||||
"with the versions it queued; each outcome is said as package-published, or left in version-unreleased with the holder's words. A hand act, " +
|
||||
"recorded in the hand-act log."
|
||||
|
||||
|
||||
@@ -92,7 +92,7 @@ func TestRepublish(t *testing.T) {
|
||||
if v.Fingerprint != "src2:m" {
|
||||
t.Fatalf("the reservation keeps the entry's fingerprint: %q", v.Fingerprint)
|
||||
}
|
||||
if q, err := b.QueueDone(ctx); err != nil || len(b.sent) != 1 || fmt.Sprint(q) != "[m "+v.Version+"]" {
|
||||
if q, err := b.Republish(ctx, "", ""); err != nil || len(b.sent) != 1 || fmt.Sprint(q) != "[m "+v.Version+"]" {
|
||||
t.Fatalf("queued %v sent %d (%v), want 1", q, len(b.sent), err)
|
||||
}
|
||||
p := b.sent[0]
|
||||
@@ -104,20 +104,6 @@ func TestRepublish(t *testing.T) {
|
||||
t.Errorf("a key stays in the queue after its outcome: %v", keys)
|
||||
}
|
||||
})
|
||||
t.Run("the setting limits an event, newest first", func(t *testing.T) {
|
||||
b := newRepublishBed(t, 2)
|
||||
var versions []string
|
||||
for _, m := range []string{"m1", "m2", "m3"} {
|
||||
versions = append(versions, b.reserve(t, m).Module)
|
||||
}
|
||||
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
|
||||
if _, err := b.QueueDone(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(b.sent) != 2 || !strings.HasPrefix(b.sent[0].Name, "m3/") || !strings.HasPrefix(b.sent[1].Name, "m2/") {
|
||||
t.Errorf("sent %+v, want the two newest, m3 then m2", b.sent)
|
||||
}
|
||||
})
|
||||
t.Run("an entry that cannot be read stops neither the other versions nor the drain", func(t *testing.T) {
|
||||
b := newRepublishBed(t, 50)
|
||||
b.reserve(t, "m1")
|
||||
@@ -130,7 +116,7 @@ func TestRepublish(t *testing.T) {
|
||||
return entry(ctx, fp)
|
||||
}
|
||||
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
|
||||
if _, err := b.QueueDone(ctx); err == nil || !strings.Contains(err.Error(), "src2:m1") {
|
||||
if _, err := b.Republish(ctx, "", ""); err == nil || !strings.Contains(err.Error(), "src2:m1") {
|
||||
t.Errorf("the unreadable entry is not said: %v", err)
|
||||
}
|
||||
if len(b.sent) != 1 || !strings.HasPrefix(b.sent[0].Name, "m2/") {
|
||||
@@ -142,8 +128,8 @@ func TestRepublish(t *testing.T) {
|
||||
b.reserve(t, "m")
|
||||
b.refuse = "oci-artifact-registry's publish of m/prog: not stored: no blob sha256:aa"
|
||||
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
|
||||
_, _ = b.QueueDone(ctx)
|
||||
_, _ = b.QueueDone(ctx)
|
||||
_, _ = b.Republish(ctx, "", "")
|
||||
_, _ = b.Republish(ctx, "", "")
|
||||
if len(b.sent) != 1 || len(b.said) != 1 || !strings.Contains(b.said[0], "not stored: ") {
|
||||
t.Errorf("sent %d, said %v: want one try and one saying of not stored", len(b.sent), b.said)
|
||||
}
|
||||
@@ -153,8 +139,8 @@ func TestRepublish(t *testing.T) {
|
||||
b.reserve(t, "m")
|
||||
b.refuse = "the holder is full"
|
||||
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
|
||||
_, _ = b.QueueDone(ctx)
|
||||
_, _ = b.QueueDone(ctx)
|
||||
_, _ = b.Republish(ctx, "", "")
|
||||
_, _ = b.Republish(ctx, "", "")
|
||||
if len(b.sent) != 2 || len(b.said) != 2 {
|
||||
t.Errorf("sent %d, said %v: want both events tried and said", len(b.sent), b.said)
|
||||
}
|
||||
@@ -164,7 +150,7 @@ func TestRepublish(t *testing.T) {
|
||||
b.reserve(t, "m")
|
||||
b.queue.bytes = PublishQueueMiB << 20
|
||||
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
|
||||
_, _ = b.QueueDone(ctx)
|
||||
_, _ = b.Republish(ctx, "", "")
|
||||
if len(b.sent) != 0 || len(b.said) != 1 || !strings.HasSuffix(b.said[0], QueueFull) {
|
||||
t.Errorf("sent %d, said %v, want nothing sent and queue full said", len(b.sent), b.said)
|
||||
}
|
||||
@@ -198,7 +184,7 @@ func TestRepublish(t *testing.T) {
|
||||
t.Error("a version no reservation holds is not refused")
|
||||
}
|
||||
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
|
||||
_, _ = b.QueueDone(ctx)
|
||||
_, _ = b.Republish(ctx, "", "")
|
||||
items := Reader{Keys: func() ([]string, error) { return []string{"publish.m.0.1.0", "reserve-done.m.x"}, nil },
|
||||
Get: func(k string) (json.RawMessage, bool, error) {
|
||||
return json.RawMessage(`{"mode":"publish","module":"m","version":"0.1.0","tries":3,"next-try":"2026-11-17T01:00:00Z","why":"w"}`), true, nil
|
||||
|
||||
@@ -118,7 +118,10 @@ type Reserver struct {
|
||||
// Releases is the builder's own record of the versions it saw released, by module and fingerprint (design 47 G A-456a): the
|
||||
// reservation is deleted at release, so a rerun of a released build's step reads it here, never from the catalog. Nil keeps none.
|
||||
Releases ReserveStore
|
||||
Now func() time.Time
|
||||
// ReleasedKept is how many releases per module the record keeps, the oldest dropped (the setting released-kept; A-456b); 0 is
|
||||
// DefaultReleasedKept.
|
||||
ReleasedKept int
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
func (r *Reserver) now() time.Time {
|
||||
@@ -251,18 +254,80 @@ func allSet(m map[string]string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// ReleasedKey is the key of the record that the build of fingerprint was released (design 47 G A-456a), in the state acted.
|
||||
func ReleasedKey(module, fingerprint string) string {
|
||||
return "released." + EscapeToken(module) + "." + EscapeToken(fingerprint)
|
||||
// DefaultReleasedKept is the setting released-kept unless set (design 47 G A-456b).
|
||||
const DefaultReleasedKept = 16
|
||||
|
||||
// ReleasedKey is the key of a module's record of its last releases (design 47 G A-456a, A-456b), in the state acted.
|
||||
func ReleasedKey(module string) string { return "released." + EscapeToken(module) }
|
||||
|
||||
// ReleasedRecord is that key's value: the module's last releases, oldest first.
|
||||
type ReleasedRecord struct {
|
||||
Kept []ReleasedBuild `json:"kept"`
|
||||
}
|
||||
|
||||
// ReleasedFor says whether the builder saw a version released for the build of fingerprint.
|
||||
// ReleasedBuild is one release: the fingerprint of the build and its version.
|
||||
type ReleasedBuild struct {
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
// ReleasedFor says whether the builder saw a version released for the build of fingerprint among the module's last releases.
|
||||
func (r *Reserver) ReleasedFor(ctx context.Context, module, fingerprint string) (bool, error) {
|
||||
if r.Releases == nil {
|
||||
return false, nil
|
||||
}
|
||||
_, _, found, err := r.Releases.Get(ctx, ReleasedKey(module, fingerprint))
|
||||
return found, err
|
||||
raw, _, found, err := r.Releases.Get(ctx, ReleasedKey(module))
|
||||
if err != nil || !found {
|
||||
return false, err
|
||||
}
|
||||
var rec ReleasedRecord
|
||||
if json.Unmarshal(raw, &rec) != nil {
|
||||
return false, nil
|
||||
}
|
||||
for _, k := range rec.Kept {
|
||||
if k.Fingerprint == fingerprint {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// keepRelease adds a release to the module's record, dropping the oldest past ReleasedKept.
|
||||
func (r *Reserver) keepRelease(ctx context.Context, module, fingerprint, version string) error {
|
||||
n := r.ReleasedKept
|
||||
if n < 1 {
|
||||
n = DefaultReleasedKept
|
||||
}
|
||||
for {
|
||||
raw, rev, found, err := r.Releases.Get(ctx, ReleasedKey(module))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var rec ReleasedRecord
|
||||
if found {
|
||||
_ = json.Unmarshal(raw, &rec)
|
||||
}
|
||||
kept := rec.Kept[:0:0]
|
||||
for _, k := range rec.Kept {
|
||||
if k.Fingerprint != fingerprint {
|
||||
kept = append(kept, k)
|
||||
}
|
||||
}
|
||||
kept = append(kept, ReleasedBuild{Fingerprint: fingerprint, Version: version})
|
||||
if len(kept) > n {
|
||||
kept = kept[len(kept)-n:]
|
||||
}
|
||||
out, _ := json.Marshal(ReleasedRecord{Kept: kept})
|
||||
if found {
|
||||
_, err = r.Releases.Update(ctx, ReleasedKey(module), out, rev)
|
||||
} else {
|
||||
_, err = r.Releases.Create(ctx, ReleasedKey(module), out)
|
||||
}
|
||||
if errors.Is(err, ErrConflict) {
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Released takes the catalog's version-released in: the key of that version is deleted, and every key of module's other
|
||||
@@ -287,7 +352,7 @@ func (r *Reserver) Released(ctx context.Context, module, version string) ([]stri
|
||||
continue
|
||||
}
|
||||
if cur.Version == version && cur.Fingerprint != "" && r.Releases != nil {
|
||||
if _, err := r.Releases.Create(ctx, ReleasedKey(module, cur.Fingerprint), []byte(`{"version":"`+version+`"}`)); err != nil && !errors.Is(err, ErrConflict) {
|
||||
if err := r.keepRelease(ctx, module, cur.Fingerprint, version); err != nil {
|
||||
return gone, fmt.Errorf("the release of %s %s cannot be recorded: %w", module, version, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -179,7 +179,7 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) {
|
||||
func TestAReleaseIsRecordedByFingerprintBeforeItsReservationGoes(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
rel := &reserveMem{mem: newMem()}
|
||||
r := &Reserver{Store: &reserveMem{mem: newMem()}, Releases: rel, Filled: func() bool { return true },
|
||||
r := &Reserver{Store: &reserveMem{mem: newMem()}, Releases: rel, ReleasedKept: 2, Filled: func() bool { return true },
|
||||
ReleasedOf: func(context.Context, string) ([]string, error) { return nil, nil }}
|
||||
v, err := r.Reserve(ctx, "m", "0.1", []string{"m:a"}, Source{Fingerprint: "src2:aa"})
|
||||
if err != nil {
|
||||
@@ -200,4 +200,23 @@ func TestAReleaseIsRecordedByFingerprintBeforeItsReservationGoes(t *testing.T) {
|
||||
if all, _ := r.All(ctx); len(all) != 0 {
|
||||
t.Errorf("reservations left: %v", all)
|
||||
}
|
||||
// One key per module holds the last releases, the oldest dropped (A-456b).
|
||||
for _, fp := range []string{"src2:b1", "src2:b2"} {
|
||||
n, err := r.Reserve(ctx, "m", "0.1", []string{"m:a"}, Source{Fingerprint: fp})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := r.Released(ctx, "m", n.Version); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if keys, _ := rel.KeysOf(ctx, AllFilter); len(keys) != 1 {
|
||||
t.Errorf("release keys %v, want one per module", keys)
|
||||
}
|
||||
if old, _ := r.ReleasedFor(ctx, "m", "src2:aa"); old {
|
||||
t.Error("the oldest release is kept past released-kept")
|
||||
}
|
||||
if ok, _ := r.ReleasedFor(ctx, "m", "src2:b2"); !ok {
|
||||
t.Error("the newest release is not kept")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -185,7 +185,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
m.Streams[2].Name != "mesh-delivery-builder-findings" || !reflect.DeepEqual(m.Streams[2].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}) {
|
||||
t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days, kept last per subject: design 47 G A-181) and the findings' stream", m.Streams)
|
||||
}
|
||||
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 5 || m.Config["republish-max-versions"].Default != float64(50) ||
|
||||
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 6 || m.Config["republish-max-versions"].Default != float64(50) ||
|
||||
m.Config["publish-pause"].Default != float64(30) || m.Config["publish-pause-max"].Default != float64(30) || m.Config["publish-tries"].Default != float64(8) {
|
||||
t.Errorf("config = %+v, want toolchain-wait-minutes of 60, republish-max-versions of 50 and the publish settings 30 s, 30 min and 8 tries (design 47 G §588, §633, A-457a)", m.Config)
|
||||
}
|
||||
@@ -240,7 +240,7 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
|
||||
t.Errorf("process = %v", p)
|
||||
}
|
||||
env, _ := p["env"].(map[string]any)
|
||||
if len(env) != 6 || env["MESH_BUILDER_PUBLISH_TRIES_FILE"] != "${dir:state}/publish-tries" || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_REPUBLISH_MAX_FILE"] != "${dir:state}/republish-max-versions" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" {
|
||||
if len(env) != 7 || env["MESH_BUILDER_PUBLISH_TRIES_FILE"] != "${dir:state}/publish-tries" || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_REPUBLISH_MAX_FILE"] != "${dir:state}/republish-max-versions" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" {
|
||||
t.Errorf("env = %v, want the broker file and the path of the toolchain wait's file, never the setting itself (design 47 G §672, A-71)", p["env"])
|
||||
}
|
||||
for k, v := range env {
|
||||
|
||||
@@ -149,7 +149,7 @@
|
||||
},
|
||||
"republish-max-versions": {
|
||||
"type": "number",
|
||||
"description": "the most versions one start or one builder_publish_again queues to publish at once, newest first (design 47 G section 588); the rest wait for the next try or builder_publish_again.",
|
||||
"description": "kept for the module's setting file; no longer read: the publish is tried per version by the builder's publish items (design 47 G section 588, A-457a).",
|
||||
"minimum": 1,
|
||||
"maximum": 10000,
|
||||
"trusted": false,
|
||||
@@ -171,6 +171,14 @@
|
||||
"trusted": false,
|
||||
"default": 30
|
||||
},
|
||||
"released-kept": {
|
||||
"type": "number",
|
||||
"description": "how many of a module's last releases the builder keeps, by fingerprint, to judge a rerun of a released build's step (design 47 G A-456b); the oldest is dropped.",
|
||||
"minimum": 1,
|
||||
"maximum": 1000,
|
||||
"trusted": false,
|
||||
"default": 16
|
||||
},
|
||||
"publish-tries": {
|
||||
"type": "number",
|
||||
"description": "how many times the builder tries a done build's publish before it ends loudly (log and condition) and leaves it readable in builder_publish_queue (design 47 G A-457a).",
|
||||
@@ -247,6 +255,14 @@
|
||||
"mode": "0600",
|
||||
"content": "${config:publish-pause-max}"
|
||||
},
|
||||
{
|
||||
"id": "released-kept",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/released-kept",
|
||||
"owner": "mesh-delivery-builder",
|
||||
"mode": "0600",
|
||||
"content": "${config:released-kept}"
|
||||
},
|
||||
{
|
||||
"id": "publish-tries",
|
||||
"type": "file",
|
||||
@@ -272,7 +288,8 @@
|
||||
"MESH_BUILDER_REPUBLISH_MAX_FILE": "${dir:state}/republish-max-versions",
|
||||
"MESH_BUILDER_PUBLISH_PAUSE_FILE": "${dir:state}/publish-pause",
|
||||
"MESH_BUILDER_PUBLISH_PAUSE_MAX_FILE": "${dir:state}/publish-pause-max",
|
||||
"MESH_BUILDER_PUBLISH_TRIES_FILE": "${dir:state}/publish-tries"
|
||||
"MESH_BUILDER_PUBLISH_TRIES_FILE": "${dir:state}/publish-tries",
|
||||
"MESH_BUILDER_RELEASED_KEPT_FILE": "${dir:state}/released-kept"
|
||||
},
|
||||
"user": "mesh-delivery-builder",
|
||||
"health": {
|
||||
|
||||
Reference in New Issue
Block a user