diff --git a/modules/mesh-delivery-builder/internal/build/numbers.go b/modules/mesh-delivery-builder/internal/build/numbers.go index c34e611bc..b14cca58b 100644 --- a/modules/mesh-delivery-builder/internal/build/numbers.go +++ b/modules/mesh-delivery-builder/internal/build/numbers.go @@ -199,6 +199,9 @@ func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.Modu } src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2, Branch: o.Branch, Revision: o.Source, CatalogRevision: o.Revision, Rebuild: rebuild} for _, t := range plan.Tasks { + if t.Kind == catalogue.ArtifactUpstream { + src.Upstream = append(src.Upstream, module+":"+t.Artifact) // copied by its pin, never awaited (H-A-486) + } if t.Kind == catalogue.ArtifactPackage && t.Language == "go" && !src.GoPackage { src.GoPackage, src.GoArtifact = true, t.Artifact // its module path names its tag's directory (design 47 H2 §796) } diff --git a/modules/mesh-delivery-builder/internal/build/reserve_test.go b/modules/mesh-delivery-builder/internal/build/reserve_test.go index 37d11baa0..bcc214c9a 100644 --- a/modules/mesh-delivery-builder/internal/build/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/build/reserve_test.go @@ -118,6 +118,13 @@ func TestABuildReservesItsVersionAtItsStartOnceTheCatalogFilled(t *testing.T) { if res.built["m:tool"] != digest || len(res.built) != 2 { t.Errorf("recorded %v, want the digest of m:tool under the reserved version", res.built) } + // H-A-486: an upstream artifact is reserved for the record and named as upstream, so its publish is never awaited. + up := &fakeReservations{filled: true} + plan := Plan{Series: "0.1", Tasks: []Task{{Artifact: "server", Kind: catalogue.ArtifactUpstream}, {Artifact: "tools", Kind: catalogue.ArtifactBundle}}} + if _, ok := (&Builder{Reserved: up}).reserve(context.Background(), "nats", meshcatalog.ModulesByModuleValue{Fp2: "f"}, plan, Origin{}, false); !ok || + !slices.Equal(up.reserved[0], []string{"nats:server", "nats:tools"}) || !slices.Equal(up.sources[0].Upstream, []string{"nats:server"}) { + t.Errorf("an upstream artifact reserved %v from %+v, want both refs reserved and nats:server named upstream", up.reserved, up.sources) + } r.consider(t, "modules/m") if len(res.reserved) != 1 { t.Errorf("a done key reserved again: %v", res.reserved) diff --git a/modules/mesh-delivery-builder/internal/versions/reserve.go b/modules/mesh-delivery-builder/internal/versions/reserve.go index b76a1534c..724c5f282 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve.go @@ -55,6 +55,10 @@ type ReservedVersion struct { Version string `json:"version"` // MAJOR.MINOR.PATCH // Artifacts are the module's artifacts at the version: the digest once built, empty before. Artifacts map[string]string `json:"artifacts"` + // Upstream are the artifacts of kind upstream, by reference (design 47 H2 H-A-486): kept in Artifacts with their digest for the + // record, never awaited, as design 51 copies an upstream image by its pinned digest and no registry seat says it published at + // a version. A reservation written before carries none. + Upstream []string `json:"upstream,omitempty"` // BuiltAt is when the build ended with every digest known; the publish's bound counts from it. BuiltAt time.Time `json:"built-at,omitzero"` // Published are the artifacts a registry seat said published at this version with the built digest. @@ -126,6 +130,8 @@ type Source struct { Revision int64 // CatalogRevision is the catalog revision number the build's step ask named (H-A-218), 0 when none did. CatalogRevision int64 + // Upstream are the artifacts of kind upstream among those reserved, by reference (H-A-486), kept on the reservation. + Upstream []string // Rebuild marks a build again of a done key whose version is not released: the standing reservation of Fingerprint is replaced // and is no floor, so the number is the next above the catalog's highest alone (design 47 G A-490b). Rebuild bool @@ -377,6 +383,7 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts for _, a := range artifacts { out.Artifacts[a] = "" } + out.Upstream = slices.Clone(src.Upstream) if found { out.Withdrawn = maps.Clone(cur.Withdrawn) if cur.Fingerprint != src.Fingerprint { @@ -1134,11 +1141,11 @@ const ( ReservedVerb = meshbuilds.VerbReserved ) -// Missing are the artifacts of v no registry seat said published, sorted. +// Missing are the artifacts of v no registry seat said published, sorted; an upstream one is never awaited (H-A-486). func (v ReservedVersion) Missing() []string { var out []string for a := range v.Artifacts { - if !v.Published[a] { + if !v.Published[a] && !slices.Contains(v.Upstream, a) { out = append(out, a) } } diff --git a/modules/mesh-delivery-builder/internal/versions/reserve_test.go b/modules/mesh-delivery-builder/internal/versions/reserve_test.go index 3fadf9cd4..a0d96f966 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve_test.go @@ -171,6 +171,23 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) { if got := cur.Missing(); !reflect.DeepEqual(got, []string{"m:b"}) { t.Errorf("missing = %v, want m:b", got) } + // H-A-486: an upstream artifact is reserved and recorded with its digest, but never awaited: once the others are published + // nothing is missing. + up, err := r.Reserve(ctx, "u", "0.1", []string{"u:server", "u:tools"}, Source{Upstream: []string{"u:server"}}) + if err != nil || !reflect.DeepEqual(up.Upstream, []string{"u:server"}) { + t.Fatalf("reserved %+v (%v), want u:server kept as upstream", up, err) + } + for _, a := range []string{"u:server", "u:tools"} { + if _, err := r.Built(ctx, "u", up.Version, a, "sha256:"+a, nil); err != nil { + t.Fatal(err) + } + } + if marked, _, err := r.Published(ctx, "u-tools", up.Version, "sha256:u:tools"); err != nil || len(marked) != 1 { + t.Fatalf("the published of u:tools marked %v (%v)", marked, err) + } + if cur, _, _ := r.At(ctx, ReservedKey("u", "0.1")); len(cur.Missing()) != 0 || cur.Artifacts["u:server"] != "sha256:u:server" { + t.Errorf("missing = %v with artifacts %v, want none missing and the upstream digest kept", cur.Missing(), cur.Artifacts) + } if key, ok := ReservedBoundOf(ReservedBoundKey("m", "1.4")); !ok || key != ReservedKey("m", "1.4") || ReservedBoundFor(key) != ReservedBoundKey("m", "1.4") { t.Errorf("the bound key does not name the reservation back: %q %v", key, ok) } diff --git a/modules/mesh-delivery-indexer/internal/index/versions.go b/modules/mesh-delivery-indexer/internal/index/versions.go index 71336e1f3..ad9f2f8d4 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions.go +++ b/modules/mesh-delivery-indexer/internal/index/versions.go @@ -140,7 +140,9 @@ func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.Packag } var want []string for _, a := range entry.Artifacts { - want = append(want, a.Ref) + if a.Kind != catalogue.ArtifactUpstream { // an upstream copy is released by its pinned digest, never published (H-A-486) + want = append(want, a.Ref) + } } key := PendingPrefix + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Version) var have published @@ -211,7 +213,15 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli late := len(kept) > 0 && compareVersions(kept[0].Version, p.Version) > 0 at := ix.now().UTC() v := meshcatalog.VersionEntry{V: int64(keys.Version), Module: p.Module, Version: p.Version, Source: have.Source, At: &at, Late: late} + pins, err := upstreamPins(entry) + if err != nil { + return err + } for _, a := range entry.Artifacts { + if a.Kind == catalogue.ArtifactUpstream { + v.Artifacts = append(v.Artifacts, meshcatalog.VersionArtifact{Artifact: a.Ref, Digest: pins[a.Ref]}) + continue + } v.Artifacts = append(v.Artifacts, have.Artifacts[a.Ref]) } slices.SortFunc(v.Artifacts, func(a, b meshcatalog.VersionArtifact) int { return strings.Compare(a.Artifact, b.Artifact) }) @@ -255,6 +265,46 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli return ix.forgetPublished(ctx, pendingKey) } +// upstreamPins are the pinned digests of the entry's upstream artifacts, by reference, read from its manifest's from +// (`@sha256:`): design 51 copies an upstream image by that digest, so a release names it (H-A-486). An upstream +// artifact whose manifest names no pin refuses the release, naming it. +func upstreamPins(entry meshcatalog.CatalogEntry) (map[string]string, error) { + out := map[string]string{} + if !slices.ContainsFunc(entry.Artifacts, func(a meshcatalog.Artifact) bool { return a.Kind == catalogue.ArtifactUpstream }) { + return out, nil + } + raw, err := json.Marshal(entry.Manifest) + if err != nil { + return nil, err + } + var m struct { + Build struct { + Artifacts []struct { + Name string `json:"name"` + Kind string `json:"kind"` + From string `json:"from"` + } `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + return nil, fmt.Errorf("the manifest of %s's entry does not read, so its upstream artifacts' pins are not known: %w", entry.Module, err) + } + for _, a := range m.Build.Artifacts { + if a.Kind != catalogue.ArtifactUpstream { + continue + } + if _, d, ok := strings.Cut(a.From, "@"); ok && strings.HasPrefix(d, "sha256:") { + out[ArtifactRefOf(entry.Module, a.Name)] = d + } + } + for _, a := range entry.Artifacts { + if _, ok := out[a.Ref]; a.Kind == catalogue.ArtifactUpstream && !ok { + return nil, fmt.Errorf("%s is an upstream artifact whose manifest names no pinned digest (from @sha256:), so %s releases no version", a.Ref, entry.Module) + } + } + return out, nil +} + // trimVersions deletes the module's oldest versions beyond the kept 100, never its highest of any MAJOR.MINOR. func (ix *Indexer) trimVersions(ctx context.Context, module string) error { kept, err := ModuleVersions(ctx, ix.Versions, module) // newest first diff --git a/modules/mesh-delivery-indexer/internal/index/versions_test.go b/modules/mesh-delivery-indexer/internal/index/versions_test.go index b4c56076a..8e7e2e663 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_test.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_test.go @@ -83,6 +83,36 @@ func TestAVersionIsReleasedOnceEveryArtifactIsPublished(t *testing.T) { if compareVersions(oldest, "0.1.5") < 0 { t.Errorf("the oldest kept is %s: the oldest were not the ones deleted", oldest) } + + // H-A-486: an upstream artifact is copied by its pinned digest and never published, so the version is released once every + // other artifact is, the upstream one named with the pin its manifest gives; one whose manifest names no pin releases none. + pin := "sha256:" + strings.Repeat("ab", 32) + nats := func(from string) map[string]meshcatalog.CatalogEntry { + return map[string]meshcatalog.CatalogEntry{"nats": {Module: "nats", Manifest: map[string]any{"build": map[string]any{"artifacts": []any{ + map[string]any{"name": "server", "kind": "upstream", "from": from}, + map[string]any{"name": "tools", "kind": "bundle", "from": "."}}}}, + Artifacts: []meshcatalog.Artifact{{Ref: "nats:server", Kind: "upstream"}, {Ref: "nats:tools", Kind: "bundle", Language: "go"}}}} + } + for i, c := range []struct { + from, version string + released bool + }{{"nats:2.12-alpine@" + pin, "0.1.1", true}, {"nats:2.12-alpine", "0.1.2", false}} { + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: int64(4 + i), Kind: "trunk"}, nats(c.from)) + if err != nil { + t.Fatal(err) + } + if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { + t.Fatal(err) + } + err = ix.HearPackagePublished(ctx, meshbuilds.PackagePublished{Module: "nats", Artifact: "nats:tools", Digest: "sha256:tools", Version: c.version, Source: 9, Branch: "main"}) + got, _ := ModuleVersions(ctx, ix.Versions, "nats") + switch { + case c.released && (err != nil || len(got) != 1 || len(got[0].Artifacts) != 2 || got[0].Artifacts[0] != (meshcatalog.VersionArtifact{Artifact: "nats:server", Digest: pin})): + t.Fatalf("pinned %q: released %+v (%v); want the server named with its pin", c.from, got, err) + case !c.released && (err == nil || !strings.Contains(err.Error(), "no pinned digest") || len(got) != 1): + t.Fatalf("unpinned %q: released %+v (%v); want a refusal naming the pin", c.from, got, err) + } + } } // Design 47 G §372, A-350: the first fill records, once, each module's highest version the builder gave a number, numbered or