builder: keep a published heard before the build's digests and say package-published when they are recorded
A trunk build's registry publishes during the build, before the builder records the digests on the reservation; such a published was acknowledged and lost, so package-published was never said. It is now kept on the reservation (name, version, digest) and marks its artifact when the digests are recorded; one that never matches is logged. Builds: design 47 G §588, §52, §334, A-73, A-415
This commit is contained in:
@@ -96,6 +96,17 @@ func (rv *reservations) Built(ctx context.Context, module, version string, diges
|
||||
return err
|
||||
}
|
||||
rv.arm(ctx, cur)
|
||||
// A published heard before the digests were recorded marks its artifact now, said as one heard after (design 47 G A-415).
|
||||
marked, unmatched, err := rv.r.Kept(ctx, module, version)
|
||||
if err != nil {
|
||||
rv.logf("the published kept on %s %s are not taken in: %v", module, version, err)
|
||||
}
|
||||
for _, h := range unmatched {
|
||||
rv.logf("a published of %s@%s kept on %s %s marks no artifact: its digest %s is none of the built digests %v", h.Name, h.Version, module, version, h.Digest, cur.Artifacts)
|
||||
}
|
||||
for _, m := range marked {
|
||||
rv.sayPublished(ctx, m, m.Reserved.Artifacts[m.Artifact])
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -206,24 +217,31 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
|
||||
// published marks the artifacts of the reservation a registry seat's published names (its version and digest).
|
||||
func (rv *reservations) published(ctx context.Context, o versions.Outcome) {
|
||||
marked, err := rv.r.Published(ctx, o.Version, o.Digest)
|
||||
marked, kept, err := rv.r.Published(ctx, o.Name, o.Version, o.Digest)
|
||||
if err != nil {
|
||||
rv.logf("a published of %s@%s is not recorded in the reservations: %v", o.Name, o.Version, err)
|
||||
}
|
||||
if len(marked) == 0 {
|
||||
if len(kept) > 0 {
|
||||
rv.logf("the published of %s@%s (digest %s, holder %s) is kept on %s until the build records its digests", o.Name, o.Version, o.Digest, o.Holder, strings.Join(kept, ", "))
|
||||
} else if len(marked) == 0 {
|
||||
rv.logf("package-published not said: the published of %s@%s (digest %s, holder %s) marks no artifact of a reservation: %s", o.Name, o.Version, o.Digest, o.Holder, rv.noMark(ctx, o))
|
||||
}
|
||||
// Each artifact the holder's published newly marked is said as the seat mesh-builds' package-published (design 47 G §52, §334).
|
||||
for _, m := range marked {
|
||||
if rv.say == nil {
|
||||
rv.logf("package-published of %s at %s not said: no seat to say it on", m.Artifact, m.Reserved.Version)
|
||||
continue
|
||||
}
|
||||
if err := rv.say(ctx, m.PackagePublished(o.Digest, time.Now())); err != nil {
|
||||
rv.logf("the package-published of %s at %s is not said: %v", m.Artifact, m.Reserved.Version, err)
|
||||
} else {
|
||||
rv.logf("package-published said: %s at %s, digest %s", m.Artifact, m.Reserved.Version, o.Digest)
|
||||
}
|
||||
rv.sayPublished(ctx, m, o.Digest)
|
||||
}
|
||||
}
|
||||
|
||||
// sayPublished says the seat mesh-builds' package-published of an artifact a published marked.
|
||||
func (rv *reservations) sayPublished(ctx context.Context, m versions.Marked, digest string) {
|
||||
if rv.say == nil {
|
||||
rv.logf("package-published of %s at %s not said: no seat to say it on", m.Artifact, m.Reserved.Version)
|
||||
return
|
||||
}
|
||||
if err := rv.say(ctx, m.PackagePublished(digest, time.Now())); err != nil {
|
||||
rv.logf("the package-published of %s at %s is not said: %v", m.Artifact, m.Reserved.Version, err)
|
||||
} else {
|
||||
rv.logf("package-published said: %s at %s, digest %s", m.Artifact, m.Reserved.Version, digest)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -283,7 +301,7 @@ func (rv *reservations) noMark(ctx context.Context, o versions.Outcome) string {
|
||||
if v.Version != version {
|
||||
continue
|
||||
}
|
||||
out = append(out, fmt.Sprintf("%s holds %d built digests %v, published %v (a published heard before the build recorded its digests marks nothing)", v.Module, len(v.Artifacts), v.Artifacts, v.Published))
|
||||
out = append(out, fmt.Sprintf("%s holds %d built digests %v, published %v (a published heard before the build recorded its digests is kept until it does)", v.Module, len(v.Artifacts), v.Artifacts, v.Published))
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return "no reservation holds version " + version
|
||||
|
||||
@@ -203,9 +203,10 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A published heard before the build recorded its digests marks nothing and says why, so a package-published never said is
|
||||
// readable in the journal.
|
||||
func TestAPublishedBeforeTheDigestsIsSaidAsMarkingNothing(t *testing.T) {
|
||||
// Design 47 G A-415: a published heard before the build recorded its digests is kept on its reservation, and when the build
|
||||
// records them it marks its artifact and package-published is said then; a kept one whose digest is none of the built digests
|
||||
// is said in the log.
|
||||
func TestAPublishedHeardBeforeTheDigestsIsSaidWhenTheBuildRecordsThem(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
var logs []string
|
||||
logf := func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }
|
||||
@@ -214,12 +215,30 @@ func TestAPublishedBeforeTheDigestsIsSaidAsMarkingNothing(t *testing.T) {
|
||||
}
|
||||
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, none, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} },
|
||||
&findings{module: builds.BuilderModule, w: &memFindings{vals: map[string][]byte{}}, logf: t.Logf, now: time.Now}, nil, logf)
|
||||
var said []meshbuilds.PackagePublished
|
||||
rv.say = func(_ context.Context, p meshbuilds.PackagePublished) error { said = append(said, p); return nil }
|
||||
rv.filled.Store(true)
|
||||
if _, err := rv.Reserve(ctx, "m", "1.4", []string{"m:a"}, versions.Source{}); err != nil {
|
||||
res, err := rv.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, versions.Source{Branch: "main", Revision: 41})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rv.published(ctx, versions.Outcome{Name: "m-a", Version: "1.4.1", Digest: "sha256:a"})
|
||||
if !strings.Contains(strings.Join(logs, "\n"), "package-published not said") || !strings.Contains(strings.Join(logs, "\n"), "marks no artifact") {
|
||||
t.Fatalf("log = %q, want package-published not said", logs)
|
||||
rv.published(ctx, versions.Outcome{Name: "m-a", Version: res.Version, Digest: "sha256:a"})
|
||||
rv.published(ctx, versions.Outcome{Name: "m-x", Version: res.Version, Digest: "sha256:x"})
|
||||
if len(said) != 0 {
|
||||
t.Fatalf("package-published said before the digests are recorded: %+v", said)
|
||||
}
|
||||
if err := rv.Built(ctx, "m", res.Version, map[string]string{"m:a": "sha256:a", "m:b": "sha256:b"}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(said) != 1 || said[0].Artifact != "m:a" || said[0].Version != res.Version || said[0].Digest != "sha256:a" || said[0].Branch != "main" || said[0].Source != 41 {
|
||||
t.Fatalf("package-published said %+v, want one of m:a at %s for main at source 41", said, res.Version)
|
||||
}
|
||||
if log := strings.Join(logs, "\n"); !strings.Contains(log, "is kept on") || !strings.Contains(log, "m-x@"+res.Version+" kept on m "+res.Version+" marks no artifact") {
|
||||
t.Fatalf("log = %q, want the published kept and the one never matching said", logs)
|
||||
}
|
||||
// Taken in once: a published of m:a heard again says nothing more.
|
||||
rv.published(ctx, versions.Outcome{Name: "m-a", Version: res.Version, Digest: "sha256:a"})
|
||||
if len(said) != 1 {
|
||||
t.Fatalf("package-published said again: %+v", said)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -66,6 +66,16 @@ type ReservedVersion struct {
|
||||
// Dependencies are, by artifact reference, what the step's result says each artifact resolved (kind 1) and was built with
|
||||
// (kind 3), kept for `package-published` (design 47 G §466, §673, A-57).
|
||||
Dependencies map[string][]meshbuilds.Dependency `json:"dependencies,omitempty"`
|
||||
// Kept are the registry seats' published heard while the build had not yet recorded every digest and whose digest marked
|
||||
// no artifact then: kept as heard, they mark their artifact when the build records its digests (design 47 G A-415).
|
||||
Kept []KeptPublished `json:"kept-published,omitempty"`
|
||||
}
|
||||
|
||||
// KeptPublished is a registry seat's published kept on a reservation, as heard (design 47 G A-415).
|
||||
type KeptPublished struct {
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
|
||||
// Source is what a build reserving a version is made from (design 47 G §370): the repository and commit the git tag is
|
||||
@@ -345,30 +355,38 @@ func (m Marked) PackagePublished(digest string, at time.Time) meshbuilds.Package
|
||||
|
||||
// Published takes a registry seat's published in: the artifacts of a reservation at version (a leading v is the Go form)
|
||||
// whose built digest it names are marked published. It answers each artifact it newly marked, with the reservation as marked.
|
||||
func (r *Reserver) Published(ctx context.Context, version, digest string) ([]Marked, error) {
|
||||
// A published that marks no artifact of a reservation at version whose build has not recorded every digest is kept on that
|
||||
// reservation as heard (name, version, digest), and the keys it is kept on are answered too (design 47 G A-415): Kept marks
|
||||
// it when the build records its digests.
|
||||
func (r *Reserver) Published(ctx context.Context, name, version, digest string) ([]Marked, []string, error) {
|
||||
version = strings.TrimPrefix(version, "v")
|
||||
if version == "" || digest == "" {
|
||||
return nil, nil
|
||||
return nil, nil, nil
|
||||
}
|
||||
keys, err := r.Store.KeysOf(ctx, AllFilter)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the reservations cannot be listed: %w", err)
|
||||
return nil, nil, fmt.Errorf("the reservations cannot be listed: %w", err)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var marked []Marked
|
||||
var kept []string
|
||||
for _, k := range keys {
|
||||
again:
|
||||
cur, rev, found, err := r.get(ctx, k)
|
||||
if err != nil {
|
||||
return marked, err
|
||||
return marked, kept, err
|
||||
}
|
||||
if !found || cur.Version != version {
|
||||
continue
|
||||
}
|
||||
changed := false
|
||||
changed, matched, keep := false, false, false
|
||||
var arts []string
|
||||
for a, d := range cur.Artifacts {
|
||||
if d == digest && !cur.Published[a] {
|
||||
if d != digest {
|
||||
continue
|
||||
}
|
||||
matched = true
|
||||
if !cur.Published[a] {
|
||||
if cur.Published == nil {
|
||||
cur.Published = map[string]bool{}
|
||||
}
|
||||
@@ -377,6 +395,10 @@ func (r *Reserver) Published(ctx context.Context, version, digest string) ([]Mar
|
||||
}
|
||||
}
|
||||
sort.Strings(arts)
|
||||
if !matched && cur.BuiltAt.IsZero() && !cur.keeps(name, digest) {
|
||||
cur.Kept = append(cur.Kept, KeptPublished{Name: name, Version: version, Digest: digest})
|
||||
changed, keep = true, true
|
||||
}
|
||||
if !changed {
|
||||
continue
|
||||
}
|
||||
@@ -384,13 +406,76 @@ func (r *Reserver) Published(ctx context.Context, version, digest string) ([]Mar
|
||||
if _, err := r.Store.Update(ctx, k, raw, rev); errors.Is(err, ErrConflict) {
|
||||
goto again
|
||||
} else if err != nil {
|
||||
return marked, fmt.Errorf("%s cannot be written: %w", k, err)
|
||||
return marked, kept, fmt.Errorf("%s cannot be written: %w", k, err)
|
||||
}
|
||||
for _, a := range arts {
|
||||
marked = append(marked, Marked{Reserved: cur, Artifact: a})
|
||||
}
|
||||
if keep {
|
||||
kept = append(kept, k)
|
||||
}
|
||||
}
|
||||
return marked, kept, nil
|
||||
}
|
||||
|
||||
func (v ReservedVersion) keeps(name, digest string) bool {
|
||||
for _, h := range v.Kept {
|
||||
if h.Name == name && h.Digest == digest {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Kept takes the published kept on module's reservation of version in once its build recorded every digest (design 47 G
|
||||
// A-415): each kept published whose digest is an artifact's marks that artifact, as one heard after would, and every kept one
|
||||
// leaves the reservation. It answers the artifacts newly marked and the kept published whose digest matched no artifact.
|
||||
func (r *Reserver) Kept(ctx context.Context, module, version string) ([]Marked, []KeptPublished, error) {
|
||||
series, _, err := SplitVersion(version)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
key := ReservedKey(module, series)
|
||||
for {
|
||||
cur, rev, found, err := r.get(ctx, key)
|
||||
if err != nil || !found || cur.Version != version || cur.BuiltAt.IsZero() || len(cur.Kept) == 0 {
|
||||
return nil, nil, err
|
||||
}
|
||||
var arts []string
|
||||
var unmatched []KeptPublished
|
||||
for _, h := range cur.Kept {
|
||||
matched := false
|
||||
for a, d := range cur.Artifacts {
|
||||
if d != h.Digest {
|
||||
continue
|
||||
}
|
||||
matched = true
|
||||
if !cur.Published[a] {
|
||||
if cur.Published == nil {
|
||||
cur.Published = map[string]bool{}
|
||||
}
|
||||
cur.Published[a] = true
|
||||
arts = append(arts, a)
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
unmatched = append(unmatched, h)
|
||||
}
|
||||
}
|
||||
cur.Kept = nil
|
||||
sort.Strings(arts)
|
||||
raw, _ := json.Marshal(cur)
|
||||
if _, err := r.Store.Update(ctx, key, raw, rev); errors.Is(err, ErrConflict) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return nil, nil, fmt.Errorf("%s cannot be written: %w", key, err)
|
||||
}
|
||||
var marked []Marked
|
||||
for _, a := range arts {
|
||||
marked = append(marked, Marked{Reserved: cur, Artifact: a})
|
||||
}
|
||||
return marked, unmatched, nil
|
||||
}
|
||||
return marked, nil
|
||||
}
|
||||
|
||||
// Of is module's reservation of series, and whether there is one; with an empty series the module's highest.
|
||||
|
||||
@@ -143,7 +143,7 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) {
|
||||
if got, err := AnswerReserved(rd, "other"); err != nil || len(got["reserved"].([]ReservedVersion)) != 0 {
|
||||
t.Errorf("builder_reserved of another module = %v (%v), want none", got, err)
|
||||
}
|
||||
marked, err := r.Published(ctx, "v1.4.3", "sha256:aa")
|
||||
marked, _, err := r.Published(ctx, "m-a", "v1.4.3", "sha256:aa")
|
||||
if err != nil || len(marked) != 1 {
|
||||
t.Fatalf("a published of 1.4.3 with a built digest marked %v (%v)", marked, err)
|
||||
}
|
||||
@@ -155,7 +155,7 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) {
|
||||
if wantSaid := (meshbuilds.PackagePublished{V: 1, Artifact: "m:a", At: &atUTC, Branch: "main", Source: 41, Module: "m", Version: "1.4.3", Digest: "sha256:aa", Dependencies: deps}); !reflect.DeepEqual(said, wantSaid) {
|
||||
t.Errorf("package-published = %+v, want %+v", said, wantSaid)
|
||||
}
|
||||
if marked, _ := r.Published(ctx, "1.4.3", "sha256:other"); len(marked) != 0 {
|
||||
if marked, _, _ := r.Published(ctx, "m-a", "1.4.3", "sha256:other"); len(marked) != 0 {
|
||||
t.Errorf("a published with another digest marked %v", marked)
|
||||
}
|
||||
cur, _, _ := r.At(ctx, ReservedKey("m", "1.4"))
|
||||
|
||||
Reference in New Issue
Block a user