builder: keep a published heard before the build's digests and say package-published when they are recorded

A trunk build's registry publishes during the build, before the builder records the digests on the reservation;
such a published was acknowledged and lost, so package-published was never said. It is now kept on the reservation
(name, version, digest) and marks its artifact when the digests are recorded; one that never matches is logged.

Builds: design 47 G §588, §52, §334, A-73, A-415
This commit is contained in:
2026-11-15 02:07:47 +01:00
parent 1506507f21
commit f89f34ce7b
4 changed files with 151 additions and 29 deletions
@@ -96,6 +96,17 @@ func (rv *reservations) Built(ctx context.Context, module, version string, diges
return err
}
rv.arm(ctx, cur)
// A published heard before the digests were recorded marks its artifact now, said as one heard after (design 47 G A-415).
marked, unmatched, err := rv.r.Kept(ctx, module, version)
if err != nil {
rv.logf("the published kept on %s %s are not taken in: %v", module, version, err)
}
for _, h := range unmatched {
rv.logf("a published of %s@%s kept on %s %s marks no artifact: its digest %s is none of the built digests %v", h.Name, h.Version, module, version, h.Digest, cur.Artifacts)
}
for _, m := range marked {
rv.sayPublished(ctx, m, m.Reserved.Artifacts[m.Artifact])
}
return nil
}
@@ -206,24 +217,31 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
// published marks the artifacts of the reservation a registry seat's published names (its version and digest).
func (rv *reservations) published(ctx context.Context, o versions.Outcome) {
marked, err := rv.r.Published(ctx, o.Version, o.Digest)
marked, kept, err := rv.r.Published(ctx, o.Name, o.Version, o.Digest)
if err != nil {
rv.logf("a published of %s@%s is not recorded in the reservations: %v", o.Name, o.Version, err)
}
if len(marked) == 0 {
if len(kept) > 0 {
rv.logf("the published of %s@%s (digest %s, holder %s) is kept on %s until the build records its digests", o.Name, o.Version, o.Digest, o.Holder, strings.Join(kept, ", "))
} else if len(marked) == 0 {
rv.logf("package-published not said: the published of %s@%s (digest %s, holder %s) marks no artifact of a reservation: %s", o.Name, o.Version, o.Digest, o.Holder, rv.noMark(ctx, o))
}
// Each artifact the holder's published newly marked is said as the seat mesh-builds' package-published (design 47 G §52, §334).
for _, m := range marked {
if rv.say == nil {
rv.logf("package-published of %s at %s not said: no seat to say it on", m.Artifact, m.Reserved.Version)
continue
}
if err := rv.say(ctx, m.PackagePublished(o.Digest, time.Now())); err != nil {
rv.logf("the package-published of %s at %s is not said: %v", m.Artifact, m.Reserved.Version, err)
} else {
rv.logf("package-published said: %s at %s, digest %s", m.Artifact, m.Reserved.Version, o.Digest)
}
rv.sayPublished(ctx, m, o.Digest)
}
}
// sayPublished says the seat mesh-builds' package-published of an artifact a published marked.
func (rv *reservations) sayPublished(ctx context.Context, m versions.Marked, digest string) {
if rv.say == nil {
rv.logf("package-published of %s at %s not said: no seat to say it on", m.Artifact, m.Reserved.Version)
return
}
if err := rv.say(ctx, m.PackagePublished(digest, time.Now())); err != nil {
rv.logf("the package-published of %s at %s is not said: %v", m.Artifact, m.Reserved.Version, err)
} else {
rv.logf("package-published said: %s at %s, digest %s", m.Artifact, m.Reserved.Version, digest)
}
}
@@ -283,7 +301,7 @@ func (rv *reservations) noMark(ctx context.Context, o versions.Outcome) string {
if v.Version != version {
continue
}
out = append(out, fmt.Sprintf("%s holds %d built digests %v, published %v (a published heard before the build recorded its digests marks nothing)", v.Module, len(v.Artifacts), v.Artifacts, v.Published))
out = append(out, fmt.Sprintf("%s holds %d built digests %v, published %v (a published heard before the build recorded its digests is kept until it does)", v.Module, len(v.Artifacts), v.Artifacts, v.Published))
}
if len(out) == 0 {
return "no reservation holds version " + version
@@ -203,9 +203,10 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) {
}
}
// A published heard before the build recorded its digests marks nothing and says why, so a package-published never said is
// readable in the journal.
func TestAPublishedBeforeTheDigestsIsSaidAsMarkingNothing(t *testing.T) {
// Design 47 G A-415: a published heard before the build recorded its digests is kept on its reservation, and when the build
// records them it marks its artifact and package-published is said then; a kept one whose digest is none of the built digests
// is said in the log.
func TestAPublishedHeardBeforeTheDigestsIsSaidWhenTheBuildRecordsThem(t *testing.T) {
ctx := context.Background()
var logs []string
logf := func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }
@@ -214,12 +215,30 @@ func TestAPublishedBeforeTheDigestsIsSaidAsMarkingNothing(t *testing.T) {
}
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, none, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} },
&findings{module: builds.BuilderModule, w: &memFindings{vals: map[string][]byte{}}, logf: t.Logf, now: time.Now}, nil, logf)
var said []meshbuilds.PackagePublished
rv.say = func(_ context.Context, p meshbuilds.PackagePublished) error { said = append(said, p); return nil }
rv.filled.Store(true)
if _, err := rv.Reserve(ctx, "m", "1.4", []string{"m:a"}, versions.Source{}); err != nil {
res, err := rv.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, versions.Source{Branch: "main", Revision: 41})
if err != nil {
t.Fatal(err)
}
rv.published(ctx, versions.Outcome{Name: "m-a", Version: "1.4.1", Digest: "sha256:a"})
if !strings.Contains(strings.Join(logs, "\n"), "package-published not said") || !strings.Contains(strings.Join(logs, "\n"), "marks no artifact") {
t.Fatalf("log = %q, want package-published not said", logs)
rv.published(ctx, versions.Outcome{Name: "m-a", Version: res.Version, Digest: "sha256:a"})
rv.published(ctx, versions.Outcome{Name: "m-x", Version: res.Version, Digest: "sha256:x"})
if len(said) != 0 {
t.Fatalf("package-published said before the digests are recorded: %+v", said)
}
if err := rv.Built(ctx, "m", res.Version, map[string]string{"m:a": "sha256:a", "m:b": "sha256:b"}, nil); err != nil {
t.Fatal(err)
}
if len(said) != 1 || said[0].Artifact != "m:a" || said[0].Version != res.Version || said[0].Digest != "sha256:a" || said[0].Branch != "main" || said[0].Source != 41 {
t.Fatalf("package-published said %+v, want one of m:a at %s for main at source 41", said, res.Version)
}
if log := strings.Join(logs, "\n"); !strings.Contains(log, "is kept on") || !strings.Contains(log, "m-x@"+res.Version+" kept on m "+res.Version+" marks no artifact") {
t.Fatalf("log = %q, want the published kept and the one never matching said", logs)
}
// Taken in once: a published of m:a heard again says nothing more.
rv.published(ctx, versions.Outcome{Name: "m-a", Version: res.Version, Digest: "sha256:a"})
if len(said) != 1 {
t.Fatalf("package-published said again: %+v", said)
}
}
@@ -66,6 +66,16 @@ type ReservedVersion struct {
// Dependencies are, by artifact reference, what the step's result says each artifact resolved (kind 1) and was built with
// (kind 3), kept for `package-published` (design 47 G §466, §673, A-57).
Dependencies map[string][]meshbuilds.Dependency `json:"dependencies,omitempty"`
// Kept are the registry seats' published heard while the build had not yet recorded every digest and whose digest marked
// no artifact then: kept as heard, they mark their artifact when the build records its digests (design 47 G A-415).
Kept []KeptPublished `json:"kept-published,omitempty"`
}
// KeptPublished is a registry seat's published kept on a reservation, as heard (design 47 G A-415).
type KeptPublished struct {
Name string `json:"name"`
Version string `json:"version"`
Digest string `json:"digest"`
}
// Source is what a build reserving a version is made from (design 47 G §370): the repository and commit the git tag is
@@ -345,30 +355,38 @@ func (m Marked) PackagePublished(digest string, at time.Time) meshbuilds.Package
// Published takes a registry seat's published in: the artifacts of a reservation at version (a leading v is the Go form)
// whose built digest it names are marked published. It answers each artifact it newly marked, with the reservation as marked.
func (r *Reserver) Published(ctx context.Context, version, digest string) ([]Marked, error) {
// A published that marks no artifact of a reservation at version whose build has not recorded every digest is kept on that
// reservation as heard (name, version, digest), and the keys it is kept on are answered too (design 47 G A-415): Kept marks
// it when the build records its digests.
func (r *Reserver) Published(ctx context.Context, name, version, digest string) ([]Marked, []string, error) {
version = strings.TrimPrefix(version, "v")
if version == "" || digest == "" {
return nil, nil
return nil, nil, nil
}
keys, err := r.Store.KeysOf(ctx, AllFilter)
if err != nil {
return nil, fmt.Errorf("the reservations cannot be listed: %w", err)
return nil, nil, fmt.Errorf("the reservations cannot be listed: %w", err)
}
sort.Strings(keys)
var marked []Marked
var kept []string
for _, k := range keys {
again:
cur, rev, found, err := r.get(ctx, k)
if err != nil {
return marked, err
return marked, kept, err
}
if !found || cur.Version != version {
continue
}
changed := false
changed, matched, keep := false, false, false
var arts []string
for a, d := range cur.Artifacts {
if d == digest && !cur.Published[a] {
if d != digest {
continue
}
matched = true
if !cur.Published[a] {
if cur.Published == nil {
cur.Published = map[string]bool{}
}
@@ -377,6 +395,10 @@ func (r *Reserver) Published(ctx context.Context, version, digest string) ([]Mar
}
}
sort.Strings(arts)
if !matched && cur.BuiltAt.IsZero() && !cur.keeps(name, digest) {
cur.Kept = append(cur.Kept, KeptPublished{Name: name, Version: version, Digest: digest})
changed, keep = true, true
}
if !changed {
continue
}
@@ -384,13 +406,76 @@ func (r *Reserver) Published(ctx context.Context, version, digest string) ([]Mar
if _, err := r.Store.Update(ctx, k, raw, rev); errors.Is(err, ErrConflict) {
goto again
} else if err != nil {
return marked, fmt.Errorf("%s cannot be written: %w", k, err)
return marked, kept, fmt.Errorf("%s cannot be written: %w", k, err)
}
for _, a := range arts {
marked = append(marked, Marked{Reserved: cur, Artifact: a})
}
if keep {
kept = append(kept, k)
}
}
return marked, kept, nil
}
func (v ReservedVersion) keeps(name, digest string) bool {
for _, h := range v.Kept {
if h.Name == name && h.Digest == digest {
return true
}
}
return false
}
// Kept takes the published kept on module's reservation of version in once its build recorded every digest (design 47 G
// A-415): each kept published whose digest is an artifact's marks that artifact, as one heard after would, and every kept one
// leaves the reservation. It answers the artifacts newly marked and the kept published whose digest matched no artifact.
func (r *Reserver) Kept(ctx context.Context, module, version string) ([]Marked, []KeptPublished, error) {
series, _, err := SplitVersion(version)
if err != nil {
return nil, nil, err
}
key := ReservedKey(module, series)
for {
cur, rev, found, err := r.get(ctx, key)
if err != nil || !found || cur.Version != version || cur.BuiltAt.IsZero() || len(cur.Kept) == 0 {
return nil, nil, err
}
var arts []string
var unmatched []KeptPublished
for _, h := range cur.Kept {
matched := false
for a, d := range cur.Artifacts {
if d != h.Digest {
continue
}
matched = true
if !cur.Published[a] {
if cur.Published == nil {
cur.Published = map[string]bool{}
}
cur.Published[a] = true
arts = append(arts, a)
}
}
if !matched {
unmatched = append(unmatched, h)
}
}
cur.Kept = nil
sort.Strings(arts)
raw, _ := json.Marshal(cur)
if _, err := r.Store.Update(ctx, key, raw, rev); errors.Is(err, ErrConflict) {
continue
} else if err != nil {
return nil, nil, fmt.Errorf("%s cannot be written: %w", key, err)
}
var marked []Marked
for _, a := range arts {
marked = append(marked, Marked{Reserved: cur, Artifact: a})
}
return marked, unmatched, nil
}
return marked, nil
}
// Of is module's reservation of series, and whether there is one; with an empty series the module's highest.
@@ -143,7 +143,7 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) {
if got, err := AnswerReserved(rd, "other"); err != nil || len(got["reserved"].([]ReservedVersion)) != 0 {
t.Errorf("builder_reserved of another module = %v (%v), want none", got, err)
}
marked, err := r.Published(ctx, "v1.4.3", "sha256:aa")
marked, _, err := r.Published(ctx, "m-a", "v1.4.3", "sha256:aa")
if err != nil || len(marked) != 1 {
t.Fatalf("a published of 1.4.3 with a built digest marked %v (%v)", marked, err)
}
@@ -155,7 +155,7 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) {
if wantSaid := (meshbuilds.PackagePublished{V: 1, Artifact: "m:a", At: &atUTC, Branch: "main", Source: 41, Module: "m", Version: "1.4.3", Digest: "sha256:aa", Dependencies: deps}); !reflect.DeepEqual(said, wantSaid) {
t.Errorf("package-published = %+v, want %+v", said, wantSaid)
}
if marked, _ := r.Published(ctx, "1.4.3", "sha256:other"); len(marked) != 0 {
if marked, _, _ := r.Published(ctx, "m-a", "1.4.3", "sha256:other"); len(marked) != 0 {
t.Errorf("a published with another digest marked %v", marked)
}
cur, _, _ := r.At(ctx, ReservedKey("m", "1.4"))