From f89f34ce7b65eabe2d47475938ceeb888ffa1b70 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 15 Nov 2026 02:07:47 +0100 Subject: [PATCH] builder: keep a published heard before the build's digests and say package-published when they are recorded MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A trunk build's registry publishes during the build, before the builder records the digests on the reservation; such a published was acknowledged and lost, so package-published was never said. It is now kept on the reservation (name, version, digest) and marks its artifact when the digests are recorded; one that never matches is logged. Builds: design 47 G §588, §52, §334, A-73, A-415 --- .../cmd/mesh-delivery-builder/reserve.go | 42 +++++--- .../cmd/mesh-delivery-builder/reserve_test.go | 33 ++++-- .../internal/versions/reserve.go | 101 ++++++++++++++++-- .../internal/versions/reserve_test.go | 4 +- 4 files changed, 151 insertions(+), 29 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go index 78776941..c1d63ee5 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go @@ -96,6 +96,17 @@ func (rv *reservations) Built(ctx context.Context, module, version string, diges return err } rv.arm(ctx, cur) + // A published heard before the digests were recorded marks its artifact now, said as one heard after (design 47 G A-415). + marked, unmatched, err := rv.r.Kept(ctx, module, version) + if err != nil { + rv.logf("the published kept on %s %s are not taken in: %v", module, version, err) + } + for _, h := range unmatched { + rv.logf("a published of %s@%s kept on %s %s marks no artifact: its digest %s is none of the built digests %v", h.Name, h.Version, module, version, h.Digest, cur.Artifacts) + } + for _, m := range marked { + rv.sayPublished(ctx, m, m.Reserved.Artifacts[m.Artifact]) + } return nil } @@ -206,24 +217,31 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool { // published marks the artifacts of the reservation a registry seat's published names (its version and digest). func (rv *reservations) published(ctx context.Context, o versions.Outcome) { - marked, err := rv.r.Published(ctx, o.Version, o.Digest) + marked, kept, err := rv.r.Published(ctx, o.Name, o.Version, o.Digest) if err != nil { rv.logf("a published of %s@%s is not recorded in the reservations: %v", o.Name, o.Version, err) } - if len(marked) == 0 { + if len(kept) > 0 { + rv.logf("the published of %s@%s (digest %s, holder %s) is kept on %s until the build records its digests", o.Name, o.Version, o.Digest, o.Holder, strings.Join(kept, ", ")) + } else if len(marked) == 0 { rv.logf("package-published not said: the published of %s@%s (digest %s, holder %s) marks no artifact of a reservation: %s", o.Name, o.Version, o.Digest, o.Holder, rv.noMark(ctx, o)) } // Each artifact the holder's published newly marked is said as the seat mesh-builds' package-published (design 47 G §52, §334). for _, m := range marked { - if rv.say == nil { - rv.logf("package-published of %s at %s not said: no seat to say it on", m.Artifact, m.Reserved.Version) - continue - } - if err := rv.say(ctx, m.PackagePublished(o.Digest, time.Now())); err != nil { - rv.logf("the package-published of %s at %s is not said: %v", m.Artifact, m.Reserved.Version, err) - } else { - rv.logf("package-published said: %s at %s, digest %s", m.Artifact, m.Reserved.Version, o.Digest) - } + rv.sayPublished(ctx, m, o.Digest) + } +} + +// sayPublished says the seat mesh-builds' package-published of an artifact a published marked. +func (rv *reservations) sayPublished(ctx context.Context, m versions.Marked, digest string) { + if rv.say == nil { + rv.logf("package-published of %s at %s not said: no seat to say it on", m.Artifact, m.Reserved.Version) + return + } + if err := rv.say(ctx, m.PackagePublished(digest, time.Now())); err != nil { + rv.logf("the package-published of %s at %s is not said: %v", m.Artifact, m.Reserved.Version, err) + } else { + rv.logf("package-published said: %s at %s, digest %s", m.Artifact, m.Reserved.Version, digest) } } @@ -283,7 +301,7 @@ func (rv *reservations) noMark(ctx context.Context, o versions.Outcome) string { if v.Version != version { continue } - out = append(out, fmt.Sprintf("%s holds %d built digests %v, published %v (a published heard before the build recorded its digests marks nothing)", v.Module, len(v.Artifacts), v.Artifacts, v.Published)) + out = append(out, fmt.Sprintf("%s holds %d built digests %v, published %v (a published heard before the build recorded its digests is kept until it does)", v.Module, len(v.Artifacts), v.Artifacts, v.Published)) } if len(out) == 0 { return "no reservation holds version " + version diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go index ab2122da..bc4c71b4 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go @@ -203,9 +203,10 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) { } } -// A published heard before the build recorded its digests marks nothing and says why, so a package-published never said is -// readable in the journal. -func TestAPublishedBeforeTheDigestsIsSaidAsMarkingNothing(t *testing.T) { +// Design 47 G A-415: a published heard before the build recorded its digests is kept on its reservation, and when the build +// records them it marks its artifact and package-published is said then; a kept one whose digest is none of the built digests +// is said in the log. +func TestAPublishedHeardBeforeTheDigestsIsSaidWhenTheBuildRecordsThem(t *testing.T) { ctx := context.Background() var logs []string logf := func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) } @@ -214,12 +215,30 @@ func TestAPublishedBeforeTheDigestsIsSaidAsMarkingNothing(t *testing.T) { } rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, none, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, &findings{module: builds.BuilderModule, w: &memFindings{vals: map[string][]byte{}}, logf: t.Logf, now: time.Now}, nil, logf) + var said []meshbuilds.PackagePublished + rv.say = func(_ context.Context, p meshbuilds.PackagePublished) error { said = append(said, p); return nil } rv.filled.Store(true) - if _, err := rv.Reserve(ctx, "m", "1.4", []string{"m:a"}, versions.Source{}); err != nil { + res, err := rv.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, versions.Source{Branch: "main", Revision: 41}) + if err != nil { t.Fatal(err) } - rv.published(ctx, versions.Outcome{Name: "m-a", Version: "1.4.1", Digest: "sha256:a"}) - if !strings.Contains(strings.Join(logs, "\n"), "package-published not said") || !strings.Contains(strings.Join(logs, "\n"), "marks no artifact") { - t.Fatalf("log = %q, want package-published not said", logs) + rv.published(ctx, versions.Outcome{Name: "m-a", Version: res.Version, Digest: "sha256:a"}) + rv.published(ctx, versions.Outcome{Name: "m-x", Version: res.Version, Digest: "sha256:x"}) + if len(said) != 0 { + t.Fatalf("package-published said before the digests are recorded: %+v", said) + } + if err := rv.Built(ctx, "m", res.Version, map[string]string{"m:a": "sha256:a", "m:b": "sha256:b"}, nil); err != nil { + t.Fatal(err) + } + if len(said) != 1 || said[0].Artifact != "m:a" || said[0].Version != res.Version || said[0].Digest != "sha256:a" || said[0].Branch != "main" || said[0].Source != 41 { + t.Fatalf("package-published said %+v, want one of m:a at %s for main at source 41", said, res.Version) + } + if log := strings.Join(logs, "\n"); !strings.Contains(log, "is kept on") || !strings.Contains(log, "m-x@"+res.Version+" kept on m "+res.Version+" marks no artifact") { + t.Fatalf("log = %q, want the published kept and the one never matching said", logs) + } + // Taken in once: a published of m:a heard again says nothing more. + rv.published(ctx, versions.Outcome{Name: "m-a", Version: res.Version, Digest: "sha256:a"}) + if len(said) != 1 { + t.Fatalf("package-published said again: %+v", said) } } diff --git a/modules/mesh-delivery-builder/internal/versions/reserve.go b/modules/mesh-delivery-builder/internal/versions/reserve.go index eef5f337..717fc94a 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve.go @@ -66,6 +66,16 @@ type ReservedVersion struct { // Dependencies are, by artifact reference, what the step's result says each artifact resolved (kind 1) and was built with // (kind 3), kept for `package-published` (design 47 G §466, §673, A-57). Dependencies map[string][]meshbuilds.Dependency `json:"dependencies,omitempty"` + // Kept are the registry seats' published heard while the build had not yet recorded every digest and whose digest marked + // no artifact then: kept as heard, they mark their artifact when the build records its digests (design 47 G A-415). + Kept []KeptPublished `json:"kept-published,omitempty"` +} + +// KeptPublished is a registry seat's published kept on a reservation, as heard (design 47 G A-415). +type KeptPublished struct { + Name string `json:"name"` + Version string `json:"version"` + Digest string `json:"digest"` } // Source is what a build reserving a version is made from (design 47 G §370): the repository and commit the git tag is @@ -345,30 +355,38 @@ func (m Marked) PackagePublished(digest string, at time.Time) meshbuilds.Package // Published takes a registry seat's published in: the artifacts of a reservation at version (a leading v is the Go form) // whose built digest it names are marked published. It answers each artifact it newly marked, with the reservation as marked. -func (r *Reserver) Published(ctx context.Context, version, digest string) ([]Marked, error) { +// A published that marks no artifact of a reservation at version whose build has not recorded every digest is kept on that +// reservation as heard (name, version, digest), and the keys it is kept on are answered too (design 47 G A-415): Kept marks +// it when the build records its digests. +func (r *Reserver) Published(ctx context.Context, name, version, digest string) ([]Marked, []string, error) { version = strings.TrimPrefix(version, "v") if version == "" || digest == "" { - return nil, nil + return nil, nil, nil } keys, err := r.Store.KeysOf(ctx, AllFilter) if err != nil { - return nil, fmt.Errorf("the reservations cannot be listed: %w", err) + return nil, nil, fmt.Errorf("the reservations cannot be listed: %w", err) } sort.Strings(keys) var marked []Marked + var kept []string for _, k := range keys { again: cur, rev, found, err := r.get(ctx, k) if err != nil { - return marked, err + return marked, kept, err } if !found || cur.Version != version { continue } - changed := false + changed, matched, keep := false, false, false var arts []string for a, d := range cur.Artifacts { - if d == digest && !cur.Published[a] { + if d != digest { + continue + } + matched = true + if !cur.Published[a] { if cur.Published == nil { cur.Published = map[string]bool{} } @@ -377,6 +395,10 @@ func (r *Reserver) Published(ctx context.Context, version, digest string) ([]Mar } } sort.Strings(arts) + if !matched && cur.BuiltAt.IsZero() && !cur.keeps(name, digest) { + cur.Kept = append(cur.Kept, KeptPublished{Name: name, Version: version, Digest: digest}) + changed, keep = true, true + } if !changed { continue } @@ -384,13 +406,76 @@ func (r *Reserver) Published(ctx context.Context, version, digest string) ([]Mar if _, err := r.Store.Update(ctx, k, raw, rev); errors.Is(err, ErrConflict) { goto again } else if err != nil { - return marked, fmt.Errorf("%s cannot be written: %w", k, err) + return marked, kept, fmt.Errorf("%s cannot be written: %w", k, err) } for _, a := range arts { marked = append(marked, Marked{Reserved: cur, Artifact: a}) } + if keep { + kept = append(kept, k) + } + } + return marked, kept, nil +} + +func (v ReservedVersion) keeps(name, digest string) bool { + for _, h := range v.Kept { + if h.Name == name && h.Digest == digest { + return true + } + } + return false +} + +// Kept takes the published kept on module's reservation of version in once its build recorded every digest (design 47 G +// A-415): each kept published whose digest is an artifact's marks that artifact, as one heard after would, and every kept one +// leaves the reservation. It answers the artifacts newly marked and the kept published whose digest matched no artifact. +func (r *Reserver) Kept(ctx context.Context, module, version string) ([]Marked, []KeptPublished, error) { + series, _, err := SplitVersion(version) + if err != nil { + return nil, nil, err + } + key := ReservedKey(module, series) + for { + cur, rev, found, err := r.get(ctx, key) + if err != nil || !found || cur.Version != version || cur.BuiltAt.IsZero() || len(cur.Kept) == 0 { + return nil, nil, err + } + var arts []string + var unmatched []KeptPublished + for _, h := range cur.Kept { + matched := false + for a, d := range cur.Artifacts { + if d != h.Digest { + continue + } + matched = true + if !cur.Published[a] { + if cur.Published == nil { + cur.Published = map[string]bool{} + } + cur.Published[a] = true + arts = append(arts, a) + } + } + if !matched { + unmatched = append(unmatched, h) + } + } + cur.Kept = nil + sort.Strings(arts) + raw, _ := json.Marshal(cur) + if _, err := r.Store.Update(ctx, key, raw, rev); errors.Is(err, ErrConflict) { + continue + } else if err != nil { + return nil, nil, fmt.Errorf("%s cannot be written: %w", key, err) + } + var marked []Marked + for _, a := range arts { + marked = append(marked, Marked{Reserved: cur, Artifact: a}) + } + return marked, unmatched, nil } - return marked, nil } // Of is module's reservation of series, and whether there is one; with an empty series the module's highest. diff --git a/modules/mesh-delivery-builder/internal/versions/reserve_test.go b/modules/mesh-delivery-builder/internal/versions/reserve_test.go index 4fd7936f..2ba984d3 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve_test.go @@ -143,7 +143,7 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) { if got, err := AnswerReserved(rd, "other"); err != nil || len(got["reserved"].([]ReservedVersion)) != 0 { t.Errorf("builder_reserved of another module = %v (%v), want none", got, err) } - marked, err := r.Published(ctx, "v1.4.3", "sha256:aa") + marked, _, err := r.Published(ctx, "m-a", "v1.4.3", "sha256:aa") if err != nil || len(marked) != 1 { t.Fatalf("a published of 1.4.3 with a built digest marked %v (%v)", marked, err) } @@ -155,7 +155,7 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) { if wantSaid := (meshbuilds.PackagePublished{V: 1, Artifact: "m:a", At: &atUTC, Branch: "main", Source: 41, Module: "m", Version: "1.4.3", Digest: "sha256:aa", Dependencies: deps}); !reflect.DeepEqual(said, wantSaid) { t.Errorf("package-published = %+v, want %+v", said, wantSaid) } - if marked, _ := r.Published(ctx, "1.4.3", "sha256:other"); len(marked) != 0 { + if marked, _, _ := r.Published(ctx, "m-a", "1.4.3", "sha256:other"); len(marked) != 0 { t.Errorf("a published with another digest marked %v", marked) } cur, _, _ := r.At(ctx, ReservedKey("m", "1.4"))