novox/mesh-delivery
19
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Copy a pinned upstream image with mesh-build-image's copy step, with no fetch, so its key names the registry's reference (issue 1587, B4, ADR 0498 §4)

Browse Source
This commit is contained in:
jschoubben
2026-10-22 17:59:19 +02:00
parent c26f909152
commit 081f32d6c9
5 changed files with 95 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-delivery-builder/internal/build/r4e_test.go
+50
View File
@@ -5,6 +5,7 @@ import (
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -105,3 +106,52 @@ func TestADoneKeyWrittenBeforeR4eIsBuiltAgainOnceAsANewRevision(t *testing.T) {
t.Errorf("a key whose artifacts all name references was built again (revision %d, was %d): once only", r.st.revs[r.key], rev)
}
}
func TestAPinnedUpstreamIsCopiedOnceOnOneImageNodeWithNoFetchAndItsKeyNamesTheReference(t *testing.T) { // B4, ADR 0498 clause 4
up := catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest}
r := newRig(t, up)
r.b.Health = imageHave(t, "ace", "novox")
v := r.consider(t, "modules/m")
if v.State != delivery.BuildDone || len(v.Artifacts) != 1 {
t.Fatalf("build = %+v, want done with the copy", v)
}
if a := v.Artifacts[0]; a.Kind != "upstream" || a.Digest != digest || a.Reference != "registry.example/m/up@"+digest {
t.Errorf("artifact = %+v, want the upstream's digest and the registry's reference", a)
}
copies := r.bm.callsOf("build_image_copy")
if len(copies) != 1 {
t.Fatalf("%d copies, want one on one node", len(copies))
}
if args := copies[0].body.(map[string]any); args["module"] != "m" || args["artifact"] != "up" || args["from"] != up.From || len(args) != 3 {
t.Errorf("copy args = %+v, want the module's name, the artifact and the pinned reference", args)
}
if n := len(r.bm.callsOf("build_image_fetch")) + len(r.bm.callsOf("build_image_build")); n != 0 {
t.Errorf("%d fetch or build calls for a copy, want none", n)
}
}
func TestAnUpstreamBesideAnImageIsCopiedAndTheImageFetchedAndBuiltOnTheSameNode(t *testing.T) { // B4
r := newRig(t, imageArt, catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest})
r.b.Health = imageHave(t, "ace")
v := r.consider(t, "modules/m")
if ok, missing := v.Referenced(); !ok {
t.Fatalf("build = %+v, missing references %v", v, missing)
}
if f, b, c := len(r.bm.callsOf("build_image_fetch")), len(r.bm.callsOf("build_image_build")), len(r.bm.callsOf("build_image_copy")); f != 1 || b != 1 || c != 1 {
t.Errorf("fetch %d, build %d, copy %d; want one each", f, b, c)
}
}
func TestACopyThatFailsNotStoredFailsTheKeyAndIsBuiltAgain(t *testing.T) { // B4, ADR 0498 clause 2
r := newRig(t, catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest})
r.b.Health = imageHave(t, "ace")
r.bm.fails["ace/build_image_copy"] = delivery.NotStored + "up: the registry answered 500"
v := r.consider(t, "modules/m")
if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.NotStored) || delivery.FailedUntilChanged(v.Error) {
t.Fatalf("build = %+v, want failed 'not stored:' and built again", v)
}
delete(r.bm.fails, "ace/build_image_copy")
if v = r.consider(t, "modules/m"); v.State != delivery.BuildDone {
t.Errorf("build = %+v, want done on the new revision", v)
}
}
modules/mesh-delivery-builder/internal/build/run.go
+38 -21
View File
@@ -658,6 +658,8 @@ type buildResult struct {
}
// runLane takes a place on the node, fetches the commit there and builds each of the lane's artifacts, one step at a time.
// An upstream artifact is copied by its pinned reference with no fetch (ADR 0498 clause 4); a lane of copies alone fetches
// nothing.
func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m delivery.Module, module string) {
release, err := b.Places.Acquire(ctx, l.node)
if err != nil {
@@ -674,44 +676,49 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del
fetchArgs["ref"], fetchArgs["base"] = v.Ref, v.Base
}
}
raw, err := b.step(ctx, l.module, l.node, "fetch", fetchArgs)
if err != nil {
l.err = err
return
}
var fetched fetchResult
if err := json.Unmarshal(raw, &fetched); err != nil || fetched.Workspace == "" {
l.err = fmt.Errorf("the fetch on %s answered no workspace: %s", l.node, raw)
return
}
// A candidate is built from its head merged onto the trunk. A head that does not merge ends the fetch passed with the tree at
// the trunk, and building that would store the trunk's build under the candidate's fingerprint.
_, candidate := fetchArgs["base"]
if candidate {
if len(fetched.Conflicts) > 0 {
l.err = fmt.Errorf("the head does not merge into the trunk on %s: files in conflict: %s", l.node, strings.Join(fetched.Conflicts, ", "))
var fetched fetchResult
if needsFetch(l.tasks) {
raw, err := b.step(ctx, l.module, l.node, "fetch", fetchArgs)
if err != nil {
l.err = err
return
}
if fetched.Merge == "" {
l.err = fmt.Errorf("the fetch with a base on %s answered no merge: %s", l.node, raw)
if err := json.Unmarshal(raw, &fetched); err != nil || fetched.Workspace == "" {
l.err = fmt.Errorf("the fetch on %s answered no workspace: %s", l.node, raw)
return
}
// A candidate is built from its head merged onto the trunk. A head that does not merge ends the fetch passed with the tree at
// the trunk, and building that would store the trunk's build under the candidate's fingerprint.
if candidate {
if len(fetched.Conflicts) > 0 {
l.err = fmt.Errorf("the head does not merge into the trunk on %s: files in conflict: %s", l.node, strings.Join(fetched.Conflicts, ", "))
return
}
if fetched.Merge == "" {
l.err = fmt.Errorf("the fetch with a base on %s answered no merge: %s", l.node, raw)
return
}
}
}
for _, t := range l.tasks {
args := map[string]any{"workspace": fetched.Workspace, "module": m.Path, "artifact": t.Artifact}
if !candidate {
tool, args := "build", map[string]any{"workspace": fetched.Workspace, "module": m.Path, "artifact": t.Artifact}
if t.Kind == catalogue.ArtifactUpstream {
// The copy runs no code of a change: the module's name (the reference's path), the artifact and its pin.
tool, args = "copy", map[string]any{"module": module, "artifact": t.Artifact, "from": t.From}
} else if !candidate {
// Only a trunk key's build pushes an image or publishes a package; a candidate's never does (ADR 0487 clause 12,
// ADR 0498 clauses 5 and 6). A bundle or an archive is pushed either way (clause 3).
args["publish"] = true
}
raw, err := b.step(ctx, l.module, l.node, "build", args)
raw, err := b.step(ctx, l.module, l.node, tool, args)
if err != nil {
l.err = err
return
}
var built buildResult
if err := json.Unmarshal(raw, &built); err != nil || built.Digest == "" {
l.err = fmt.Errorf("the build of %s of %s on %s answered no digest: %s", t.Artifact, module, l.node, raw)
l.err = fmt.Errorf("the %s of %s of %s on %s answered no digest: %s", tool, t.Artifact, module, l.node, raw)
return
}
l.digests[t.Artifact] = built.Digest
@@ -720,3 +727,13 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del
}
}
}
// needsFetch is whether any of the tasks builds from the module's source: every one but an upstream copy.
func needsFetch(tasks []Task) bool {
for _, t := range tasks {
if t.Kind != catalogue.ArtifactUpstream {
return true
}
}
return false
}
modules/mesh-delivery-builder/internal/build/run_test.go
+4 -2
View File
@@ -69,8 +69,10 @@ func (b *buildModules) CallOn(node, key string, body any) (json.RawMessage, erro
if v, ok := b.digests[node+"/"+fmt.Sprint(args["artifact"])]; ok {
d = v
}
if tool == "build_image_build" && args["publish"] == true {
step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"registry.example/m/img@` + d + `"}`)
if tool == "build_image_copy" { // mesh-build-image's copy answers the digest and where it copied to (ADR 0498 clause 4)
step.Result = json.RawMessage(`{"artifact":"` + fmt.Sprint(args["artifact"]) + `","digest":"` + d + `","reference":"registry.example/` + fmt.Sprint(args["module"]) + `/` + fmt.Sprint(args["artifact"]) + `@` + d + `","pushed":true}`)
} else if tool == "build_image_build" && args["publish"] == true {
step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"registry.example/m/` + fmt.Sprint(args["artifact"]) + `@` + d + `"}`)
} else if tool != "build_image_build" && !b.noRef[node+"/"+fmt.Sprint(args["artifact"])] {
// A bundle or an archive is pushed to the artifact store and its reference answered (ADR 0498 clause 1).
step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"artifact-store://m/` + fmt.Sprint(args["artifact"]) + `/blobs/` + d + `"}`)
modules/mesh-delivery-builder/manifest_test.go
+2 -2
View File
@@ -90,8 +90,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
"mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) {
t.Errorf("reads = %v, want %v", m.Reads, want)
}
if want := []string{"mesh-build-go.build_go_build", "mesh-build-go.build_go_fetch", "mesh-build-image.build_image_build", "mesh-build-image.build_image_fetch", "mesh-build-node.build_node_build", "mesh-build-node.build_node_fetch"}; !reflect.DeepEqual(sorted(m.Invokes), want) {
t.Errorf("invokes = %v, want only the build modules' fetch and build (go, image, node)", m.Invokes)
if want := []string{"mesh-build-go.build_go_build", "mesh-build-go.build_go_fetch", "mesh-build-image.build_image_build", "mesh-build-image.build_image_copy", "mesh-build-image.build_image_fetch", "mesh-build-node.build_node_build", "mesh-build-node.build_node_fetch"}; !reflect.DeepEqual(sorted(m.Invokes), want) {
t.Errorf("invokes = %v, want only the build modules' fetch and build (go, image, node) and the image module's copy (ADR 0498 clause 4)", m.Invokes)
}
// The builder is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it starts.
if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
modules/mesh-delivery-builder/module.json
+1
View File
@@ -25,6 +25,7 @@
"mesh-build-go.build_go_build",
"mesh-build-go.build_go_fetch",
"mesh-build-image.build_image_build",
"mesh-build-image.build_image_copy",
"mesh-build-image.build_image_fetch",
"mesh-build-node.build_node_build",
"mesh-build-node.build_node_fetch"