H-A-486: an upstream artifact needs no publish to release
Design 51 copies an upstream image by its pinned digest, and distribution's mirror says published with no version, so nats, hello-web, mailu and showcase could never be released. The reservation keeps its upstream refs (Upstream, filled at the reserve from the plan's upstream tasks) and Missing never awaits them; the indexer releases a version once every other artifact is published, naming each upstream artifact with the pin its manifest gives, and refuses the release loudly when an upstream artifact names no pin. Builds: design 47 H2 H-A-486, H-A-486 (addendum)
This commit is contained in:
@@ -199,6 +199,9 @@ func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.Modu
|
||||
}
|
||||
src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2, Branch: o.Branch, Revision: o.Source, CatalogRevision: o.Revision, Rebuild: rebuild}
|
||||
for _, t := range plan.Tasks {
|
||||
if t.Kind == catalogue.ArtifactUpstream {
|
||||
src.Upstream = append(src.Upstream, module+":"+t.Artifact) // copied by its pin, never awaited (H-A-486)
|
||||
}
|
||||
if t.Kind == catalogue.ArtifactPackage && t.Language == "go" && !src.GoPackage {
|
||||
src.GoPackage, src.GoArtifact = true, t.Artifact // its module path names its tag's directory (design 47 H2 §796)
|
||||
}
|
||||
|
||||
@@ -118,6 +118,13 @@ func TestABuildReservesItsVersionAtItsStartOnceTheCatalogFilled(t *testing.T) {
|
||||
if res.built["m:tool"] != digest || len(res.built) != 2 {
|
||||
t.Errorf("recorded %v, want the digest of m:tool under the reserved version", res.built)
|
||||
}
|
||||
// H-A-486: an upstream artifact is reserved for the record and named as upstream, so its publish is never awaited.
|
||||
up := &fakeReservations{filled: true}
|
||||
plan := Plan{Series: "0.1", Tasks: []Task{{Artifact: "server", Kind: catalogue.ArtifactUpstream}, {Artifact: "tools", Kind: catalogue.ArtifactBundle}}}
|
||||
if _, ok := (&Builder{Reserved: up}).reserve(context.Background(), "nats", meshcatalog.ModulesByModuleValue{Fp2: "f"}, plan, Origin{}, false); !ok ||
|
||||
!slices.Equal(up.reserved[0], []string{"nats:server", "nats:tools"}) || !slices.Equal(up.sources[0].Upstream, []string{"nats:server"}) {
|
||||
t.Errorf("an upstream artifact reserved %v from %+v, want both refs reserved and nats:server named upstream", up.reserved, up.sources)
|
||||
}
|
||||
r.consider(t, "modules/m")
|
||||
if len(res.reserved) != 1 {
|
||||
t.Errorf("a done key reserved again: %v", res.reserved)
|
||||
|
||||
@@ -55,6 +55,10 @@ type ReservedVersion struct {
|
||||
Version string `json:"version"` // MAJOR.MINOR.PATCH
|
||||
// Artifacts are the module's artifacts at the version: the digest once built, empty before.
|
||||
Artifacts map[string]string `json:"artifacts"`
|
||||
// Upstream are the artifacts of kind upstream, by reference (design 47 H2 H-A-486): kept in Artifacts with their digest for the
|
||||
// record, never awaited, as design 51 copies an upstream image by its pinned digest and no registry seat says it published at
|
||||
// a version. A reservation written before carries none.
|
||||
Upstream []string `json:"upstream,omitempty"`
|
||||
// BuiltAt is when the build ended with every digest known; the publish's bound counts from it.
|
||||
BuiltAt time.Time `json:"built-at,omitzero"`
|
||||
// Published are the artifacts a registry seat said published at this version with the built digest.
|
||||
@@ -126,6 +130,8 @@ type Source struct {
|
||||
Revision int64
|
||||
// CatalogRevision is the catalog revision number the build's step ask named (H-A-218), 0 when none did.
|
||||
CatalogRevision int64
|
||||
// Upstream are the artifacts of kind upstream among those reserved, by reference (H-A-486), kept on the reservation.
|
||||
Upstream []string
|
||||
// Rebuild marks a build again of a done key whose version is not released: the standing reservation of Fingerprint is replaced
|
||||
// and is no floor, so the number is the next above the catalog's highest alone (design 47 G A-490b).
|
||||
Rebuild bool
|
||||
@@ -377,6 +383,7 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
|
||||
for _, a := range artifacts {
|
||||
out.Artifacts[a] = ""
|
||||
}
|
||||
out.Upstream = slices.Clone(src.Upstream)
|
||||
if found {
|
||||
out.Withdrawn = maps.Clone(cur.Withdrawn)
|
||||
if cur.Fingerprint != src.Fingerprint {
|
||||
@@ -1134,11 +1141,11 @@ const (
|
||||
ReservedVerb = meshbuilds.VerbReserved
|
||||
)
|
||||
|
||||
// Missing are the artifacts of v no registry seat said published, sorted.
|
||||
// Missing are the artifacts of v no registry seat said published, sorted; an upstream one is never awaited (H-A-486).
|
||||
func (v ReservedVersion) Missing() []string {
|
||||
var out []string
|
||||
for a := range v.Artifacts {
|
||||
if !v.Published[a] {
|
||||
if !v.Published[a] && !slices.Contains(v.Upstream, a) {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -171,6 +171,23 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) {
|
||||
if got := cur.Missing(); !reflect.DeepEqual(got, []string{"m:b"}) {
|
||||
t.Errorf("missing = %v, want m:b", got)
|
||||
}
|
||||
// H-A-486: an upstream artifact is reserved and recorded with its digest, but never awaited: once the others are published
|
||||
// nothing is missing.
|
||||
up, err := r.Reserve(ctx, "u", "0.1", []string{"u:server", "u:tools"}, Source{Upstream: []string{"u:server"}})
|
||||
if err != nil || !reflect.DeepEqual(up.Upstream, []string{"u:server"}) {
|
||||
t.Fatalf("reserved %+v (%v), want u:server kept as upstream", up, err)
|
||||
}
|
||||
for _, a := range []string{"u:server", "u:tools"} {
|
||||
if _, err := r.Built(ctx, "u", up.Version, a, "sha256:"+a, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if marked, _, err := r.Published(ctx, "u-tools", up.Version, "sha256:u:tools"); err != nil || len(marked) != 1 {
|
||||
t.Fatalf("the published of u:tools marked %v (%v)", marked, err)
|
||||
}
|
||||
if cur, _, _ := r.At(ctx, ReservedKey("u", "0.1")); len(cur.Missing()) != 0 || cur.Artifacts["u:server"] != "sha256:u:server" {
|
||||
t.Errorf("missing = %v with artifacts %v, want none missing and the upstream digest kept", cur.Missing(), cur.Artifacts)
|
||||
}
|
||||
if key, ok := ReservedBoundOf(ReservedBoundKey("m", "1.4")); !ok || key != ReservedKey("m", "1.4") || ReservedBoundFor(key) != ReservedBoundKey("m", "1.4") {
|
||||
t.Errorf("the bound key does not name the reservation back: %q %v", key, ok)
|
||||
}
|
||||
|
||||
@@ -140,7 +140,9 @@ func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.Packag
|
||||
}
|
||||
var want []string
|
||||
for _, a := range entry.Artifacts {
|
||||
want = append(want, a.Ref)
|
||||
if a.Kind != catalogue.ArtifactUpstream { // an upstream copy is released by its pinned digest, never published (H-A-486)
|
||||
want = append(want, a.Ref)
|
||||
}
|
||||
}
|
||||
key := PendingPrefix + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Version)
|
||||
var have published
|
||||
@@ -211,7 +213,15 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli
|
||||
late := len(kept) > 0 && compareVersions(kept[0].Version, p.Version) > 0
|
||||
at := ix.now().UTC()
|
||||
v := meshcatalog.VersionEntry{V: int64(keys.Version), Module: p.Module, Version: p.Version, Source: have.Source, At: &at, Late: late}
|
||||
pins, err := upstreamPins(entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, a := range entry.Artifacts {
|
||||
if a.Kind == catalogue.ArtifactUpstream {
|
||||
v.Artifacts = append(v.Artifacts, meshcatalog.VersionArtifact{Artifact: a.Ref, Digest: pins[a.Ref]})
|
||||
continue
|
||||
}
|
||||
v.Artifacts = append(v.Artifacts, have.Artifacts[a.Ref])
|
||||
}
|
||||
slices.SortFunc(v.Artifacts, func(a, b meshcatalog.VersionArtifact) int { return strings.Compare(a.Artifact, b.Artifact) })
|
||||
@@ -255,6 +265,46 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli
|
||||
return ix.forgetPublished(ctx, pendingKey)
|
||||
}
|
||||
|
||||
// upstreamPins are the pinned digests of the entry's upstream artifacts, by reference, read from its manifest's from
|
||||
// (`<image>@sha256:<digest>`): design 51 copies an upstream image by that digest, so a release names it (H-A-486). An upstream
|
||||
// artifact whose manifest names no pin refuses the release, naming it.
|
||||
func upstreamPins(entry meshcatalog.CatalogEntry) (map[string]string, error) {
|
||||
out := map[string]string{}
|
||||
if !slices.ContainsFunc(entry.Artifacts, func(a meshcatalog.Artifact) bool { return a.Kind == catalogue.ArtifactUpstream }) {
|
||||
return out, nil
|
||||
}
|
||||
raw, err := json.Marshal(entry.Manifest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m struct {
|
||||
Build struct {
|
||||
Artifacts []struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
From string `json:"from"`
|
||||
} `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("the manifest of %s's entry does not read, so its upstream artifacts' pins are not known: %w", entry.Module, err)
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Kind != catalogue.ArtifactUpstream {
|
||||
continue
|
||||
}
|
||||
if _, d, ok := strings.Cut(a.From, "@"); ok && strings.HasPrefix(d, "sha256:") {
|
||||
out[ArtifactRefOf(entry.Module, a.Name)] = d
|
||||
}
|
||||
}
|
||||
for _, a := range entry.Artifacts {
|
||||
if _, ok := out[a.Ref]; a.Kind == catalogue.ArtifactUpstream && !ok {
|
||||
return nil, fmt.Errorf("%s is an upstream artifact whose manifest names no pinned digest (from <image>@sha256:<digest>), so %s releases no version", a.Ref, entry.Module)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// trimVersions deletes the module's oldest versions beyond the kept 100, never its highest of any MAJOR.MINOR.
|
||||
func (ix *Indexer) trimVersions(ctx context.Context, module string) error {
|
||||
kept, err := ModuleVersions(ctx, ix.Versions, module) // newest first
|
||||
|
||||
@@ -83,6 +83,36 @@ func TestAVersionIsReleasedOnceEveryArtifactIsPublished(t *testing.T) {
|
||||
if compareVersions(oldest, "0.1.5") < 0 {
|
||||
t.Errorf("the oldest kept is %s: the oldest were not the ones deleted", oldest)
|
||||
}
|
||||
|
||||
// H-A-486: an upstream artifact is copied by its pinned digest and never published, so the version is released once every
|
||||
// other artifact is, the upstream one named with the pin its manifest gives; one whose manifest names no pin releases none.
|
||||
pin := "sha256:" + strings.Repeat("ab", 32)
|
||||
nats := func(from string) map[string]meshcatalog.CatalogEntry {
|
||||
return map[string]meshcatalog.CatalogEntry{"nats": {Module: "nats", Manifest: map[string]any{"build": map[string]any{"artifacts": []any{
|
||||
map[string]any{"name": "server", "kind": "upstream", "from": from},
|
||||
map[string]any{"name": "tools", "kind": "bundle", "from": "."}}}},
|
||||
Artifacts: []meshcatalog.Artifact{{Ref: "nats:server", Kind: "upstream"}, {Ref: "nats:tools", Kind: "bundle", Language: "go"}}}}
|
||||
}
|
||||
for i, c := range []struct {
|
||||
from, version string
|
||||
released bool
|
||||
}{{"nats:2.12-alpine@" + pin, "0.1.1", true}, {"nats:2.12-alpine", "0.1.2", false}} {
|
||||
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: int64(4 + i), Kind: "trunk"}, nats(c.from))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = ix.HearPackagePublished(ctx, meshbuilds.PackagePublished{Module: "nats", Artifact: "nats:tools", Digest: "sha256:tools", Version: c.version, Source: 9, Branch: "main"})
|
||||
got, _ := ModuleVersions(ctx, ix.Versions, "nats")
|
||||
switch {
|
||||
case c.released && (err != nil || len(got) != 1 || len(got[0].Artifacts) != 2 || got[0].Artifacts[0] != (meshcatalog.VersionArtifact{Artifact: "nats:server", Digest: pin})):
|
||||
t.Fatalf("pinned %q: released %+v (%v); want the server named with its pin", c.from, got, err)
|
||||
case !c.released && (err == nil || !strings.Contains(err.Error(), "no pinned digest") || len(got) != 1):
|
||||
t.Fatalf("unpinned %q: released %+v (%v); want a refusal naming the pin", c.from, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §372, A-350: the first fill records, once, each module's highest version the builder gave a number, numbered or
|
||||
|
||||
Reference in New Issue
Block a user