H-A-486: an upstream artifact needs no publish to release

Design 51 copies an upstream image by its pinned digest, and distribution's
mirror says published with no version, so nats, hello-web, mailu and showcase
could never be released. The reservation keeps its upstream refs (Upstream,
filled at the reserve from the plan's upstream tasks) and Missing never awaits
them; the indexer releases a version once every other artifact is published,
naming each upstream artifact with the pin its manifest gives, and refuses the
release loudly when an upstream artifact names no pin.

Builds: design 47 H2 H-A-486, H-A-486 (addendum)
This commit is contained in:
2026-11-22 19:26:22 +01:00
parent 18136ce063
commit 564fd5e0bb
6 changed files with 117 additions and 3 deletions
@@ -199,6 +199,9 @@ func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.Modu
}
src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2, Branch: o.Branch, Revision: o.Source, CatalogRevision: o.Revision, Rebuild: rebuild}
for _, t := range plan.Tasks {
if t.Kind == catalogue.ArtifactUpstream {
src.Upstream = append(src.Upstream, module+":"+t.Artifact) // copied by its pin, never awaited (H-A-486)
}
if t.Kind == catalogue.ArtifactPackage && t.Language == "go" && !src.GoPackage {
src.GoPackage, src.GoArtifact = true, t.Artifact // its module path names its tag's directory (design 47 H2 §796)
}
@@ -118,6 +118,13 @@ func TestABuildReservesItsVersionAtItsStartOnceTheCatalogFilled(t *testing.T) {
if res.built["m:tool"] != digest || len(res.built) != 2 {
t.Errorf("recorded %v, want the digest of m:tool under the reserved version", res.built)
}
// H-A-486: an upstream artifact is reserved for the record and named as upstream, so its publish is never awaited.
up := &fakeReservations{filled: true}
plan := Plan{Series: "0.1", Tasks: []Task{{Artifact: "server", Kind: catalogue.ArtifactUpstream}, {Artifact: "tools", Kind: catalogue.ArtifactBundle}}}
if _, ok := (&Builder{Reserved: up}).reserve(context.Background(), "nats", meshcatalog.ModulesByModuleValue{Fp2: "f"}, plan, Origin{}, false); !ok ||
!slices.Equal(up.reserved[0], []string{"nats:server", "nats:tools"}) || !slices.Equal(up.sources[0].Upstream, []string{"nats:server"}) {
t.Errorf("an upstream artifact reserved %v from %+v, want both refs reserved and nats:server named upstream", up.reserved, up.sources)
}
r.consider(t, "modules/m")
if len(res.reserved) != 1 {
t.Errorf("a done key reserved again: %v", res.reserved)
@@ -55,6 +55,10 @@ type ReservedVersion struct {
Version string `json:"version"` // MAJOR.MINOR.PATCH
// Artifacts are the module's artifacts at the version: the digest once built, empty before.
Artifacts map[string]string `json:"artifacts"`
// Upstream are the artifacts of kind upstream, by reference (design 47 H2 H-A-486): kept in Artifacts with their digest for the
// record, never awaited, as design 51 copies an upstream image by its pinned digest and no registry seat says it published at
// a version. A reservation written before carries none.
Upstream []string `json:"upstream,omitempty"`
// BuiltAt is when the build ended with every digest known; the publish's bound counts from it.
BuiltAt time.Time `json:"built-at,omitzero"`
// Published are the artifacts a registry seat said published at this version with the built digest.
@@ -126,6 +130,8 @@ type Source struct {
Revision int64
// CatalogRevision is the catalog revision number the build's step ask named (H-A-218), 0 when none did.
CatalogRevision int64
// Upstream are the artifacts of kind upstream among those reserved, by reference (H-A-486), kept on the reservation.
Upstream []string
// Rebuild marks a build again of a done key whose version is not released: the standing reservation of Fingerprint is replaced
// and is no floor, so the number is the next above the catalog's highest alone (design 47 G A-490b).
Rebuild bool
@@ -377,6 +383,7 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
for _, a := range artifacts {
out.Artifacts[a] = ""
}
out.Upstream = slices.Clone(src.Upstream)
if found {
out.Withdrawn = maps.Clone(cur.Withdrawn)
if cur.Fingerprint != src.Fingerprint {
@@ -1134,11 +1141,11 @@ const (
ReservedVerb = meshbuilds.VerbReserved
)
// Missing are the artifacts of v no registry seat said published, sorted.
// Missing are the artifacts of v no registry seat said published, sorted; an upstream one is never awaited (H-A-486).
func (v ReservedVersion) Missing() []string {
var out []string
for a := range v.Artifacts {
if !v.Published[a] {
if !v.Published[a] && !slices.Contains(v.Upstream, a) {
out = append(out, a)
}
}
@@ -171,6 +171,23 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) {
if got := cur.Missing(); !reflect.DeepEqual(got, []string{"m:b"}) {
t.Errorf("missing = %v, want m:b", got)
}
// H-A-486: an upstream artifact is reserved and recorded with its digest, but never awaited: once the others are published
// nothing is missing.
up, err := r.Reserve(ctx, "u", "0.1", []string{"u:server", "u:tools"}, Source{Upstream: []string{"u:server"}})
if err != nil || !reflect.DeepEqual(up.Upstream, []string{"u:server"}) {
t.Fatalf("reserved %+v (%v), want u:server kept as upstream", up, err)
}
for _, a := range []string{"u:server", "u:tools"} {
if _, err := r.Built(ctx, "u", up.Version, a, "sha256:"+a, nil); err != nil {
t.Fatal(err)
}
}
if marked, _, err := r.Published(ctx, "u-tools", up.Version, "sha256:u:tools"); err != nil || len(marked) != 1 {
t.Fatalf("the published of u:tools marked %v (%v)", marked, err)
}
if cur, _, _ := r.At(ctx, ReservedKey("u", "0.1")); len(cur.Missing()) != 0 || cur.Artifacts["u:server"] != "sha256:u:server" {
t.Errorf("missing = %v with artifacts %v, want none missing and the upstream digest kept", cur.Missing(), cur.Artifacts)
}
if key, ok := ReservedBoundOf(ReservedBoundKey("m", "1.4")); !ok || key != ReservedKey("m", "1.4") || ReservedBoundFor(key) != ReservedBoundKey("m", "1.4") {
t.Errorf("the bound key does not name the reservation back: %q %v", key, ok)
}
@@ -140,7 +140,9 @@ func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.Packag
}
var want []string
for _, a := range entry.Artifacts {
want = append(want, a.Ref)
if a.Kind != catalogue.ArtifactUpstream { // an upstream copy is released by its pinned digest, never published (H-A-486)
want = append(want, a.Ref)
}
}
key := PendingPrefix + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Version)
var have published
@@ -211,7 +213,15 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli
late := len(kept) > 0 && compareVersions(kept[0].Version, p.Version) > 0
at := ix.now().UTC()
v := meshcatalog.VersionEntry{V: int64(keys.Version), Module: p.Module, Version: p.Version, Source: have.Source, At: &at, Late: late}
pins, err := upstreamPins(entry)
if err != nil {
return err
}
for _, a := range entry.Artifacts {
if a.Kind == catalogue.ArtifactUpstream {
v.Artifacts = append(v.Artifacts, meshcatalog.VersionArtifact{Artifact: a.Ref, Digest: pins[a.Ref]})
continue
}
v.Artifacts = append(v.Artifacts, have.Artifacts[a.Ref])
}
slices.SortFunc(v.Artifacts, func(a, b meshcatalog.VersionArtifact) int { return strings.Compare(a.Artifact, b.Artifact) })
@@ -255,6 +265,46 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli
return ix.forgetPublished(ctx, pendingKey)
}
// upstreamPins are the pinned digests of the entry's upstream artifacts, by reference, read from its manifest's from
// (`<image>@sha256:<digest>`): design 51 copies an upstream image by that digest, so a release names it (H-A-486). An upstream
// artifact whose manifest names no pin refuses the release, naming it.
func upstreamPins(entry meshcatalog.CatalogEntry) (map[string]string, error) {
out := map[string]string{}
if !slices.ContainsFunc(entry.Artifacts, func(a meshcatalog.Artifact) bool { return a.Kind == catalogue.ArtifactUpstream }) {
return out, nil
}
raw, err := json.Marshal(entry.Manifest)
if err != nil {
return nil, err
}
var m struct {
Build struct {
Artifacts []struct {
Name string `json:"name"`
Kind string `json:"kind"`
From string `json:"from"`
} `json:"artifacts"`
} `json:"build"`
}
if err := json.Unmarshal(raw, &m); err != nil {
return nil, fmt.Errorf("the manifest of %s's entry does not read, so its upstream artifacts' pins are not known: %w", entry.Module, err)
}
for _, a := range m.Build.Artifacts {
if a.Kind != catalogue.ArtifactUpstream {
continue
}
if _, d, ok := strings.Cut(a.From, "@"); ok && strings.HasPrefix(d, "sha256:") {
out[ArtifactRefOf(entry.Module, a.Name)] = d
}
}
for _, a := range entry.Artifacts {
if _, ok := out[a.Ref]; a.Kind == catalogue.ArtifactUpstream && !ok {
return nil, fmt.Errorf("%s is an upstream artifact whose manifest names no pinned digest (from <image>@sha256:<digest>), so %s releases no version", a.Ref, entry.Module)
}
}
return out, nil
}
// trimVersions deletes the module's oldest versions beyond the kept 100, never its highest of any MAJOR.MINOR.
func (ix *Indexer) trimVersions(ctx context.Context, module string) error {
kept, err := ModuleVersions(ctx, ix.Versions, module) // newest first
@@ -83,6 +83,36 @@ func TestAVersionIsReleasedOnceEveryArtifactIsPublished(t *testing.T) {
if compareVersions(oldest, "0.1.5") < 0 {
t.Errorf("the oldest kept is %s: the oldest were not the ones deleted", oldest)
}
// H-A-486: an upstream artifact is copied by its pinned digest and never published, so the version is released once every
// other artifact is, the upstream one named with the pin its manifest gives; one whose manifest names no pin releases none.
pin := "sha256:" + strings.Repeat("ab", 32)
nats := func(from string) map[string]meshcatalog.CatalogEntry {
return map[string]meshcatalog.CatalogEntry{"nats": {Module: "nats", Manifest: map[string]any{"build": map[string]any{"artifacts": []any{
map[string]any{"name": "server", "kind": "upstream", "from": from},
map[string]any{"name": "tools", "kind": "bundle", "from": "."}}}},
Artifacts: []meshcatalog.Artifact{{Ref: "nats:server", Kind: "upstream"}, {Ref: "nats:tools", Kind: "bundle", Language: "go"}}}}
}
for i, c := range []struct {
from, version string
released bool
}{{"nats:2.12-alpine@" + pin, "0.1.1", true}, {"nats:2.12-alpine", "0.1.2", false}} {
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: int64(4 + i), Kind: "trunk"}, nats(c.from))
if err != nil {
t.Fatal(err)
}
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
t.Fatal(err)
}
err = ix.HearPackagePublished(ctx, meshbuilds.PackagePublished{Module: "nats", Artifact: "nats:tools", Digest: "sha256:tools", Version: c.version, Source: 9, Branch: "main"})
got, _ := ModuleVersions(ctx, ix.Versions, "nats")
switch {
case c.released && (err != nil || len(got) != 1 || len(got[0].Artifacts) != 2 || got[0].Artifacts[0] != (meshcatalog.VersionArtifact{Artifact: "nats:server", Digest: pin})):
t.Fatalf("pinned %q: released %+v (%v); want the server named with its pin", c.from, got, err)
case !c.released && (err == nil || !strings.Contains(err.Error(), "no pinned digest") || len(got) != 1):
t.Fatalf("unpinned %q: released %+v (%v); want a refusal naming the pin", c.from, got, err)
}
}
}
// Design 47 G §372, A-350: the first fill records, once, each module's highest version the builder gave a number, numbered or