diff --git a/modules/mesh-delivery-builder/internal/build/r4e_test.go b/modules/mesh-delivery-builder/internal/build/r4e_test.go index ad5b584..03fa95d 100644 --- a/modules/mesh-delivery-builder/internal/build/r4e_test.go +++ b/modules/mesh-delivery-builder/internal/build/r4e_test.go @@ -5,6 +5,7 @@ import ( "strings" "testing" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -105,3 +106,52 @@ func TestADoneKeyWrittenBeforeR4eIsBuiltAgainOnceAsANewRevision(t *testing.T) { t.Errorf("a key whose artifacts all name references was built again (revision %d, was %d): once only", r.st.revs[r.key], rev) } } + +func TestAPinnedUpstreamIsCopiedOnceOnOneImageNodeWithNoFetchAndItsKeyNamesTheReference(t *testing.T) { // B4, ADR 0498 clause 4 + up := catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest} + r := newRig(t, up) + r.b.Health = imageHave(t, "ace", "novox") + v := r.consider(t, "modules/m") + if v.State != delivery.BuildDone || len(v.Artifacts) != 1 { + t.Fatalf("build = %+v, want done with the copy", v) + } + if a := v.Artifacts[0]; a.Kind != "upstream" || a.Digest != digest || a.Reference != "registry.example/m/up@"+digest { + t.Errorf("artifact = %+v, want the upstream's digest and the registry's reference", a) + } + copies := r.bm.callsOf("build_image_copy") + if len(copies) != 1 { + t.Fatalf("%d copies, want one on one node", len(copies)) + } + if args := copies[0].body.(map[string]any); args["module"] != "m" || args["artifact"] != "up" || args["from"] != up.From || len(args) != 3 { + t.Errorf("copy args = %+v, want the module's name, the artifact and the pinned reference", args) + } + if n := len(r.bm.callsOf("build_image_fetch")) + len(r.bm.callsOf("build_image_build")); n != 0 { + t.Errorf("%d fetch or build calls for a copy, want none", n) + } +} + +func TestAnUpstreamBesideAnImageIsCopiedAndTheImageFetchedAndBuiltOnTheSameNode(t *testing.T) { // B4 + r := newRig(t, imageArt, catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest}) + r.b.Health = imageHave(t, "ace") + v := r.consider(t, "modules/m") + if ok, missing := v.Referenced(); !ok { + t.Fatalf("build = %+v, missing references %v", v, missing) + } + if f, b, c := len(r.bm.callsOf("build_image_fetch")), len(r.bm.callsOf("build_image_build")), len(r.bm.callsOf("build_image_copy")); f != 1 || b != 1 || c != 1 { + t.Errorf("fetch %d, build %d, copy %d; want one each", f, b, c) + } +} + +func TestACopyThatFailsNotStoredFailsTheKeyAndIsBuiltAgain(t *testing.T) { // B4, ADR 0498 clause 2 + r := newRig(t, catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest}) + r.b.Health = imageHave(t, "ace") + r.bm.fails["ace/build_image_copy"] = delivery.NotStored + "up: the registry answered 500" + v := r.consider(t, "modules/m") + if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.NotStored) || delivery.FailedUntilChanged(v.Error) { + t.Fatalf("build = %+v, want failed 'not stored:' and built again", v) + } + delete(r.bm.fails, "ace/build_image_copy") + if v = r.consider(t, "modules/m"); v.State != delivery.BuildDone { + t.Errorf("build = %+v, want done on the new revision", v) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 1607a71..64d8889 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -658,6 +658,8 @@ type buildResult struct { } // runLane takes a place on the node, fetches the commit there and builds each of the lane's artifacts, one step at a time. +// An upstream artifact is copied by its pinned reference with no fetch (ADR 0498 clause 4); a lane of copies alone fetches +// nothing. func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m delivery.Module, module string) { release, err := b.Places.Acquire(ctx, l.node) if err != nil { @@ -674,44 +676,49 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del fetchArgs["ref"], fetchArgs["base"] = v.Ref, v.Base } } - raw, err := b.step(ctx, l.module, l.node, "fetch", fetchArgs) - if err != nil { - l.err = err - return - } - var fetched fetchResult - if err := json.Unmarshal(raw, &fetched); err != nil || fetched.Workspace == "" { - l.err = fmt.Errorf("the fetch on %s answered no workspace: %s", l.node, raw) - return - } - // A candidate is built from its head merged onto the trunk. A head that does not merge ends the fetch passed with the tree at - // the trunk, and building that would store the trunk's build under the candidate's fingerprint. _, candidate := fetchArgs["base"] - if candidate { - if len(fetched.Conflicts) > 0 { - l.err = fmt.Errorf("the head does not merge into the trunk on %s: files in conflict: %s", l.node, strings.Join(fetched.Conflicts, ", ")) + var fetched fetchResult + if needsFetch(l.tasks) { + raw, err := b.step(ctx, l.module, l.node, "fetch", fetchArgs) + if err != nil { + l.err = err return } - if fetched.Merge == "" { - l.err = fmt.Errorf("the fetch with a base on %s answered no merge: %s", l.node, raw) + if err := json.Unmarshal(raw, &fetched); err != nil || fetched.Workspace == "" { + l.err = fmt.Errorf("the fetch on %s answered no workspace: %s", l.node, raw) return } + // A candidate is built from its head merged onto the trunk. A head that does not merge ends the fetch passed with the tree at + // the trunk, and building that would store the trunk's build under the candidate's fingerprint. + if candidate { + if len(fetched.Conflicts) > 0 { + l.err = fmt.Errorf("the head does not merge into the trunk on %s: files in conflict: %s", l.node, strings.Join(fetched.Conflicts, ", ")) + return + } + if fetched.Merge == "" { + l.err = fmt.Errorf("the fetch with a base on %s answered no merge: %s", l.node, raw) + return + } + } } for _, t := range l.tasks { - args := map[string]any{"workspace": fetched.Workspace, "module": m.Path, "artifact": t.Artifact} - if !candidate { + tool, args := "build", map[string]any{"workspace": fetched.Workspace, "module": m.Path, "artifact": t.Artifact} + if t.Kind == catalogue.ArtifactUpstream { + // The copy runs no code of a change: the module's name (the reference's path), the artifact and its pin. + tool, args = "copy", map[string]any{"module": module, "artifact": t.Artifact, "from": t.From} + } else if !candidate { // Only a trunk key's build pushes an image or publishes a package; a candidate's never does (ADR 0487 clause 12, // ADR 0498 clauses 5 and 6). A bundle or an archive is pushed either way (clause 3). args["publish"] = true } - raw, err := b.step(ctx, l.module, l.node, "build", args) + raw, err := b.step(ctx, l.module, l.node, tool, args) if err != nil { l.err = err return } var built buildResult if err := json.Unmarshal(raw, &built); err != nil || built.Digest == "" { - l.err = fmt.Errorf("the build of %s of %s on %s answered no digest: %s", t.Artifact, module, l.node, raw) + l.err = fmt.Errorf("the %s of %s of %s on %s answered no digest: %s", tool, t.Artifact, module, l.node, raw) return } l.digests[t.Artifact] = built.Digest @@ -720,3 +727,13 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del } } } + +// needsFetch is whether any of the tasks builds from the module's source: every one but an upstream copy. +func needsFetch(tasks []Task) bool { + for _, t := range tasks { + if t.Kind != catalogue.ArtifactUpstream { + return true + } + } + return false +} diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 5a4395b..739bdb3 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -69,8 +69,10 @@ func (b *buildModules) CallOn(node, key string, body any) (json.RawMessage, erro if v, ok := b.digests[node+"/"+fmt.Sprint(args["artifact"])]; ok { d = v } - if tool == "build_image_build" && args["publish"] == true { - step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"registry.example/m/img@` + d + `"}`) + if tool == "build_image_copy" { // mesh-build-image's copy answers the digest and where it copied to (ADR 0498 clause 4) + step.Result = json.RawMessage(`{"artifact":"` + fmt.Sprint(args["artifact"]) + `","digest":"` + d + `","reference":"registry.example/` + fmt.Sprint(args["module"]) + `/` + fmt.Sprint(args["artifact"]) + `@` + d + `","pushed":true}`) + } else if tool == "build_image_build" && args["publish"] == true { + step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"registry.example/m/` + fmt.Sprint(args["artifact"]) + `@` + d + `"}`) } else if tool != "build_image_build" && !b.noRef[node+"/"+fmt.Sprint(args["artifact"])] { // A bundle or an archive is pushed to the artifact store and its reference answered (ADR 0498 clause 1). step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"artifact-store://m/` + fmt.Sprint(args["artifact"]) + `/blobs/` + d + `"}`) diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 28d9afa..3fd061d 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -90,8 +90,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } - if want := []string{"mesh-build-go.build_go_build", "mesh-build-go.build_go_fetch", "mesh-build-image.build_image_build", "mesh-build-image.build_image_fetch", "mesh-build-node.build_node_build", "mesh-build-node.build_node_fetch"}; !reflect.DeepEqual(sorted(m.Invokes), want) { - t.Errorf("invokes = %v, want only the build modules' fetch and build (go, image, node)", m.Invokes) + if want := []string{"mesh-build-go.build_go_build", "mesh-build-go.build_go_fetch", "mesh-build-image.build_image_build", "mesh-build-image.build_image_copy", "mesh-build-image.build_image_fetch", "mesh-build-node.build_node_build", "mesh-build-node.build_node_fetch"}; !reflect.DeepEqual(sorted(m.Invokes), want) { + t.Errorf("invokes = %v, want only the build modules' fetch and build (go, image, node) and the image module's copy (ADR 0498 clause 4)", m.Invokes) } // The builder is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it starts. if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported"}; !reflect.DeepEqual(sorted(m.Consumes), want) { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 6d36190..460ae22 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -25,6 +25,7 @@ "mesh-build-go.build_go_build", "mesh-build-go.build_go_fetch", "mesh-build-image.build_image_build", + "mesh-build-image.build_image_copy", "mesh-build-image.build_image_fetch", "mesh-build-node.build_node_build", "mesh-build-node.build_node_fetch"