diff --git a/modules/mesh-build-go/buildstep/latepublish.go b/modules/mesh-build-go/buildstep/latepublish.go new file mode 100644 index 000000000..cadb6b7e7 --- /dev/null +++ b/modules/mesh-build-go/buildstep/latepublish.go @@ -0,0 +1,91 @@ +package buildstep + +// **The late publish** (novox/hq ADR 0617 §1, design 51 *The versions*, *Publishing*; CAT-BG2, CAT-BI1): a trunk build +// whose number the builder did not know yet publishes its digest unnumbered; the builder says version.numbered once it +// numbers that fingerprint, and the build module publishes the held digest again under that version. Nothing travels: +// the publish names the one part by its digest, which the holder holds, so it answers held and tags the version (the +// holder asks builder_versions first, ADR 0605 §4). The bytes never stayed on the node, so a holder that no longer holds +// the digest is said and the event offered again. Every instance may hear the same event; the publish is the same call +// under the same transfer name, so a second one is answered held. One place for every build module: mesh-build-go's +// bundles and archives on oci-artifact-registry, mesh-build-image's images on container-image-registry. + +import ( + "context" + "errors" + "fmt" + "io" + "log" + "slices" + "time" + + reg "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/parts" +) + +// NumberedPattern is the event type a build module consumes for the late publish, as its manifest's `consumes` names it. +const NumberedPattern = delivery.BuilderModule + "." + delivery.VersionNumberedEvent + +// errNotHere is a held part's Open: its bytes did not stay on the node. +var errNotHere = errors.New("the built bytes did not stay on the node; the holder must hold the digest") + +// LatePublish publishes a numbered build's held digest under its version on seat, from the builder's version.numbered +// body, when its kind is one of kinds. It answers false, nil for an event that is not the module's to publish: another +// kind, another seat (an event naming none is the artifact seat's, as the builder wrote it before images), or a version +// numbered with no digest. +func LatePublish(ctx context.Context, call SeatCall, body []byte, seat string, kinds ...string) (bool, error) { + n, err := delivery.DecodeVersionNumbered(body) + if err != nil { + return false, err + } + on := n.Seat + if on == "" { + on = ArtifactSeat + } + if n.Digest == "" || on != seat || !slices.Contains(kinds, n.Kind) { + return false, nil + } + in := reg.PublishInput{Name: n.Name, Kind: n.Kind, Version: n.Version, Tag: n.Tag, Digest: n.Digest, + Fingerprint: n.Fingerprint, Commit: n.Commit, Transfer: TransferName(n.Name, "v"+n.Version, n.Digest)} + sent, err := OnSeat(ctx, seat, "late publish of "+n.Name+" "+n.Version+" at "+n.Digest, func() (parts.Sent, error) { + return PublishParts(ctx, call, seat, in, []parts.Part{{Digest: n.Digest, + Open: func() (io.ReadCloser, error) { return nil, errNotHere }}}) + }, nil) + if err != nil { + return false, fmt.Errorf("the late publish of %s@%s (%s): %w", n.Name, n.Version, n.Digest, err) + } + if len(sent.Parts) > 0 { + return false, fmt.Errorf("the late publish of %s@%s sent parts it should not have", n.Name, n.Version) + } + return true, nil +} + +// HearNumbered subscribes late to the builder's version.numbered, asking again until the runtime takes it, and says +// each publish and each failure; an error is the event's, which is offered again. +func HearNumbered(ctx context.Context, late func(ctx context.Context, body []byte) (bool, error)) { + handle := func(e stdio.Envelope) error { + did, err := late(ctx, e.Body) + if err != nil { + log.Printf("%v", err) + return err + } + if did { + log.Printf("published a numbered build's held digest under its version (%s)", e.Headers["x-event-id"]) + } + return nil + } + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + err := stdio.Subscribe(NumberedPattern, handle) + if err == nil { + log.Printf("listening for the builder's %s", NumberedPattern) + return + } + log.Printf("not hearing the builder's %s yet (%v); asking again in %s", NumberedPattern, err, wait) + select { + case <-ctx.Done(): + return + case <-time.After(wait): + } + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/latepublish.go b/modules/mesh-build-go/cmd/mesh-build-go/latepublish.go index 40a44b9b2..66c6d7123 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/latepublish.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/latepublish.go @@ -1,92 +1,27 @@ package main -// The late publish (novox/hq design 51 *The versions*, *Publishing*; CAT-BG2): a trunk build of a bundle or an archive -// publishes its digest unnumbered when the builder has not numbered it yet; the builder says version.numbered once it -// does, and this module publishes the held digest again under that version. Nothing travels: the publish names the one -// part by its digest, which the holder holds, so it answers held and tags the version (the holder asks builder_versions -// first, ADR 0605 §4). The bytes never stayed on the node, so a holder that no longer holds the digest is said and the -// event offered again, which ends in the controller's condition when it keeps failing. Every node's instance may hear -// the same event; the publish is the same call under the same transfer name, so a second one is answered held. +// The late publish (CAT-BG2): a numbered bundle or archive's held digest published under its version, by the build +// modules' one late publish (buildstep/latepublish.go). import ( "context" - "errors" - "fmt" - "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" - "io" - "time" - registry "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry" + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" stdio "git.novox.be/novox/mesh-sdk/go" - "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/parts" ) // NumberedPattern is the event type this module consumes for the late publish, as its manifest's `consumes` names it. -const NumberedPattern = delivery.BuilderModule + "." + delivery.VersionNumberedEvent +const NumberedPattern = buildstep.NumberedPattern -// errNotHere is a held part's Open: its bytes did not stay on the node. -var errNotHere = errors.New("the packed bytes did not stay on the node; the holder must hold the digest") - -// LatePublish publishes a numbered bundle or archive's held digest under its version, from the builder's -// version.numbered body. It answers false, nil for an event that is not this module's to publish: another kind (a Go -// package publishes at its build, an image is mesh-build-image's), another seat, or a version numbered with no digest. +// LatePublish publishes a numbered bundle or archive's held digest under its version on the artifact seat; a Go package +// publishes at its build, an image is mesh-build-image's. func (m *Module) LatePublish(ctx context.Context, body []byte) (bool, error) { - n, err := delivery.DecodeVersionNumbered(body) - if err != nil { - return false, err - } - seat := n.Seat - if seat == "" { - seat = ArtifactSeat - } - if n.Digest == "" || seat != ArtifactSeat || (n.Kind != "bundle" && n.Kind != "archive") { - return false, nil - } call := m.Call if call == nil { call = stdio.Ask } - in := registry.PublishInput{Name: n.Name, Kind: n.Kind, Version: n.Version, Tag: n.Tag, Digest: n.Digest, - Fingerprint: n.Fingerprint, Commit: n.Commit, Transfer: transferName(n.Name, "v"+n.Version, n.Digest)} - sent, err := buildstep.OnSeat(ctx, seat, "late publish of "+n.Name+" "+n.Version+" at "+n.Digest, func() (parts.Sent, error) { - return buildstep.PublishParts(ctx, call, seat, in, []parts.Part{{Digest: n.Digest, - Open: func() (io.ReadCloser, error) { return nil, errNotHere }}}) - }, nil) - if err != nil { - return false, fmt.Errorf("the late publish of %s@%s (%s): %w", n.Name, n.Version, n.Digest, err) - } - if len(sent.Parts) > 0 { - return false, fmt.Errorf("the late publish of %s@%s sent parts it should not have", n.Name, n.Version) - } - return true, nil + return buildstep.LatePublish(ctx, call, body, ArtifactSeat, "bundle", "archive") } -// hearNumbered subscribes to the builder's version.numbered for the late publish, asking again until the runtime -// takes it, and says each publish and each failure. -func (m *Module) hearNumbered(ctx context.Context) { - handle := func(e stdio.Envelope) error { - did, err := m.LatePublish(ctx, e.Body) - if err != nil { - logf("%v", err) - return err - } - if did { - logf("published a numbered build's held digest under its version (%s)", e.Headers["x-event-id"]) - } - return nil - } - for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { - if err := stdio.Subscribe(NumberedPattern, handle); err == nil { - logf("listening for the builder's %s", NumberedPattern) - return - } else { - logf("not hearing the builder's %s yet (%v); asking again in %s", NumberedPattern, err, wait) - } - select { - case <-ctx.Done(): - return - case <-time.After(wait): - } - } -} +// hearNumbered hears the builder's version.numbered for the late publish. +func (m *Module) hearNumbered(ctx context.Context) { buildstep.HearNumbered(ctx, m.LatePublish) } diff --git a/modules/mesh-build-image/README.md b/modules/mesh-build-image/README.md index 63603d894..e0f7dd925 100644 --- a/modules/mesh-build-image/README.md +++ b/modules/mesh-build-image/README.md @@ -40,6 +40,9 @@ The Dockerfile's steps may use the network (ADR 0487 §6); nothing a step can re one): its config, layers and manifest by digest, of which only the parts the holder lacks travel. The key carries the layout's manifest digest and the seat's reference `container-image-registry//@sha256:`; a publish the holder does not take fails `not stored:`. Without `publish` (a pull request) nothing is published. + An image the builder had not numbered yet is published by its digest alone; the module consumes the builder's + `version.numbered` and publishes that held digest under its version, sending nothing (`latepublish.go`, ADR 0617 §1); + a holder that no longer holds the digest fails the event, which is offered again. - `build_image_copy` copies an upstream artifact (`module`, `artifact`, `from`) into the image seat by its `mirror`, by the digest the manifest pins (ADR 0498 §4): the holder copies it and says `published` or `refused` on the seat's events, which the manifest `consumes`; the step ends on that outcome (or fails `not stored:` after 30 minutes without one) and diff --git a/modules/mesh-build-image/cmd/mesh-build-image/compose_test.go b/modules/mesh-build-image/cmd/mesh-build-image/compose_test.go index 66a78eabf..d22fb01b3 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/compose_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/compose_test.go @@ -27,7 +27,8 @@ func TestTheModuleInvokesTheImageSeatAndHoldsNoRegistry(t *testing.T) { "seat:container-image-registry.mirror"}; !slices.Equal(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } - var consumes []string + // The builder's version.numbered for the late publish (ADR 0617 §1), and the seat's outcomes a copy waits on. + consumes := []string{NumberedPattern} for _, e := range OutcomeEvents { consumes = append(consumes, "container-image-registry."+e) } diff --git a/modules/mesh-build-image/cmd/mesh-build-image/latepublish.go b/modules/mesh-build-image/cmd/mesh-build-image/latepublish.go new file mode 100644 index 000000000..2e5a03c8b --- /dev/null +++ b/modules/mesh-build-image/cmd/mesh-build-image/latepublish.go @@ -0,0 +1,23 @@ +package main + +// The late publish (novox/hq ADR 0617 §1, CAT-BI1): an image whose floor the builder did not know yet is built +// unnumbered and published by its digest; once the builder numbers it, its version.numbered publishes the held digest +// under that version through the image seat, sending nothing (buildstep/latepublish.go, the build modules' one). + +import ( + "context" + + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" +) + +// NumberedPattern is the event type this module consumes for the late publish, as its manifest's `consumes` names it. +const NumberedPattern = buildstep.NumberedPattern + +// LatePublish publishes a numbered image's held digest under its version on the image seat; a bundle or an archive is +// mesh-build-go's. +func (m *Module) LatePublish(ctx context.Context, body []byte) (bool, error) { + return buildstep.LatePublish(ctx, m.call(), body, buildstep.ImageSeat, KindImage) +} + +// hearNumbered hears the builder's version.numbered for the late publish. +func (m *Module) hearNumbered(ctx context.Context) { buildstep.HearNumbered(ctx, m.LatePublish) } diff --git a/modules/mesh-build-image/cmd/mesh-build-image/main.go b/modules/mesh-build-image/cmd/mesh-build-image/main.go index 2dfeff6a2..220eb6e94 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/main.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/main.go @@ -73,6 +73,8 @@ func main() { // The image seat's outcomes, which a copy waits on (the manifest's consumes), and the copies' waits in // mirror-waits, where another instance writes an outcome it heard. go hearOutcomes(ctx, m.Outcomes) + // The builder's version.numbered, for the late publish of an image built unnumbered (ADR 0617 §1). + go m.hearNumbered(ctx) go waits.Follow("", m.Outcomes.Changed) } if m.Node != "" { diff --git a/modules/mesh-build-image/cmd/mesh-build-image/publish_test.go b/modules/mesh-build-image/cmd/mesh-build-image/publish_test.go index c1196cd9a..abe0987df 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/publish_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/publish_test.go @@ -122,3 +122,50 @@ func TestATrunkBuildPublishesThroughTheImageSeatOnlyWhatTheHolderLacks(t *testin t.Errorf("a tag that is not one: step = %+v", st) } } + +// ADR 0617 §1: the builder's version.numbered for an image publishes the held digest under its version through the image +// seat, sending nothing; a digest the holder lacks fails the event (offered again); what is not this module's passes. +func TestTheNumberedEventPublishesAHeldImageUnderItsVersion(t *testing.T) { + digest := "sha256:" + strings.Repeat("ab", 32) + numbered := func(kind, seat, digest string) []byte { + raw, err := delivery.EncodeVersionNumbered(delivery.VersionNumbered{Name: "fixture-image/server", Kind: kind, Version: "1.4.0", + Module: "fixture-image", Fingerprint: "fp", Commit: "c0", Digest: digest, Seat: seat}) + if err != nil { + t.Fatal(err) + } + return raw + } + for name, c := range map[string]struct { + body []byte + did, fail bool + }{ + "a held image is numbered": {body: numbered("image", reg.Seat, digest), did: true}, + "a digest the holder lacks fails": {body: numbered("image", reg.Seat, "sha256:"+strings.Repeat("cd", 32)), fail: true}, + "a bundle is mesh-build-go's": {body: numbered("bundle", "oci-artifact-registry", digest)}, + "an event naming no seat is a bundle's": {body: numbered("image", "", digest)}, + "no digest, nothing to publish": {body: numbered("image", reg.Seat, "")}, + } { + t.Run(name, func(t *testing.T) { + m, f, _ := setup(t, same) + f.held[digest] = true + did, err := m.LatePublish(t.Context(), c.body) + if did != c.did || (err != nil) != c.fail { + t.Fatalf("did %v, err %v; want did %v, fail %v", did, err, c.did, c.fail) + } + if len(f.sent) != 0 { + t.Errorf("a late publish sent parts: %v", f.sent) + } + if !c.did { + return + } + if len(f.published) != 1 { + t.Fatalf("published %+v", f.published) + } + p := f.published[0] + if p.Name != "fixture-image/server" || p.Kind != KindImage || p.Version != "1.4.0" || p.Digest != digest || + p.Fingerprint != "fp" || p.Commit != "c0" || len(p.Parts) != 1 || p.Parts[0].Digest != digest { + t.Errorf("published %+v", p) + } + }) + } +} diff --git a/modules/mesh-build-image/module.json b/modules/mesh-build-image/module.json index 406361e06..ebe66b7ee 100644 --- a/modules/mesh-build-image/module.json +++ b/modules/mesh-build-image/module.json @@ -62,6 +62,7 @@ "seat:container-image-registry.mirror" ], "consumes": [ + "mesh-delivery-builder.version.numbered", "container-image-registry.published", "container-image-registry.refused" ],