mesh-delivery/modules/mesh-delivery-builder/internal/build/r4e_test.go

package build

import (
	"context"
	"encoding/json"
	"strings"
	"testing"

	"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
	"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
	"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
	"git.novox.be/novox/mesh-delivery/sdk/buildstep"
	"git.novox.be/novox/mesh-sdk/go/catalogue"
	"git.novox.be/novox/mesh-sdk/go/keys"
)

// R4e: references in the key (novox/hq ADR 0498), upstream copies, packages published from the trunk, the package wait
// (ADR 0500 clause 6) and one node per fingerprint (ADR 0478 clause 4).

func TestACandidatesBundleKeyNamesItsReferenceAndTheTrunkBuildsItNoMore(t *testing.T) { // B1, ADR 0498 clause 3 and its check 3
	r := newRig(t, goBundle, archiveA)
	trunkModule(t, r)
	r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
	r.b.Wait()
	v := valueOf(t, r.st, r.key)
	if v.State != builds.BuildDone {
		t.Fatalf("build = %+v, want done", v)
	}
	if ok, missing := v.Referenced(); !ok {
		t.Fatalf("a candidate's bundle and archive name no reference: %v (%+v)", missing, v.Artifacts)
	}
	rev := r.st.revs[r.key]
	r.bm.mu.Lock()
	r.bm.calls = nil
	r.bm.mu.Unlock()
	r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // the merge: the trunk now names the same fingerprint
	r.b.Wait()
	if n := len(r.bm.calls); n != 0 || r.st.revs[r.key] != rev {
		t.Errorf("the trunk built its candidate's fingerprint again: %d calls, revision %d (was %d)", n, r.st.revs[r.key], rev)
	}
}

func TestABundleWhosePushAnsweredNoReferenceFailsNotStoredAndIsBuiltAgain(t *testing.T) { // B1, ADR 0498 clauses 1 and 2
	r := newRig(t, goBundle)
	r.bm.noRef["ace/tool"] = true
	v := r.consider(t, "modules/m")
	if v.State != builds.BuildFailed || !strings.HasPrefix(v.Error, builds.NotStored) || !strings.Contains(v.Error, "tool") {
		t.Fatalf("build = %+v, want failed opening %q and naming the artifact", v, builds.NotStored)
	}
	delete(r.bm.noRef, "ace/tool")
	if v = r.consider(t, "modules/m"); v.State != builds.BuildDone {
		t.Errorf("build = %+v, want a not-stored key built again as a new revision", v)
	}
}

func TestAPushRefusedByTheStoreFailsTheKeyOpeningNotStored(t *testing.T) { // B1, ADR 0498 clause 2
	r := newRig(t, goBundle)
	r.bm.fails["ace/build_go_build"] = builds.NotStored + "tool: the artifact store answered 507 insufficient storage"
	v := r.consider(t, "modules/m")
	if v.State != builds.BuildFailed || !strings.HasPrefix(v.Error, builds.NotStored) || !strings.Contains(v.Error, "507") || !strings.Contains(v.Error, "ace") {
		t.Fatalf("build = %+v, want failed opening %q with the store's words and the node", v, builds.NotStored)
	}
}

func TestOnlyATrunkKeysBuildsAreAskedToPublish(t *testing.T) { // B1, the lead's interface (a): publish only for a trunk key
	r := newRig(t, goBundle)
	r.consider(t, "modules/m")
	for _, c := range r.bm.callsOf("build_go_build") {
		if c.body.(map[string]any)["publish"] != true {
			t.Errorf("trunk build args = %+v, want publish", c.body)
		}
	}
	c := newRig(t, goBundle)
	trunkModule(t, c)
	c.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
	c.b.Wait()
	for _, call := range c.bm.callsOf("build_go_build") {
		if _, ok := call.body.(map[string]any)["publish"]; ok {
			t.Errorf("candidate build args = %+v, want no publish", call.body)
		}
	}
}

// putBuild writes a builder key's value as the store would hold it.
func putBuild(t *testing.T, st *memStore, key string, b builds.BuilderBuild) uint64 {
	t.Helper()
	data, err := builds.EncodeBuilderBuild(b)
	if err != nil {
		t.Fatal(err)
	}
	st.mu.Lock()
	defer st.mu.Unlock()
	return st.put(key, data)
}

func TestADoneKeyWrittenBeforeR4eIsBuiltAgainOnceAsANewRevision(t *testing.T) { // B2, ADR 0498 clause 8
	r := newRig(t, goBundle, archiveA)
	rev := putBuild(t, r.st, r.key, builds.BuilderBuild{State: builds.BuildDone, By: "novox.builder", Manifest: manifestD, Step: "built mesh-build-go on ace and novox",
		Artifacts: []builds.BuiltArtifact{{Name: "files", Kind: "archive", Digest: digest}, {Name: "tool", Kind: "bundle", Digest: digest}}})
	v := r.consider(t, "modules/m")
	if v.State != builds.BuildDone || r.st.revs[r.key] <= rev {
		t.Fatalf("build = %+v at revision %d (was %d), want built again as a new revision", v, r.st.revs[r.key], rev)
	}
	if ok, missing := v.Referenced(); !ok {
		t.Fatalf("the new revision names no reference for %v", missing)
	}
	rev = r.st.revs[r.key]
	r.b.tried = map[string]bool{} // a builder started anew looks again
	r.consider(t, "modules/m")
	if r.st.revs[r.key] != rev {
		t.Errorf("a key whose artifacts all name references was built again (revision %d, was %d): once only", r.st.revs[r.key], rev)
	}
}

func TestAPinnedUpstreamIsCopiedOnceOnOneImageNodeWithNoFetchAndItsKeyNamesTheReference(t *testing.T) { // B4, ADR 0498 clause 4
	up := catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest}
	r := newRig(t, up)
	r.puller = "ace"
	v := r.consider(t, "modules/m")
	if v.State != builds.BuildDone || len(v.Artifacts) != 1 {
		t.Fatalf("build = %+v, want done with the copy", v)
	}
	if a := v.Artifacts[0]; a.Kind != "upstream" || a.Digest != digest || a.Reference != "registry.example/m/up@"+digest {
		t.Errorf("artifact = %+v, want the upstream's digest and the registry's reference", a)
	}
	copies := r.bm.callsOf("build_image_copy")
	if len(copies) != 1 {
		t.Fatalf("%d copies, want one on one node", len(copies))
	}
	if args := copies[0].body.(map[string]any); args["module"] != "m" || args["artifact"] != "up" || args["from"] != up.From || len(args) != 3 {
		t.Errorf("copy args = %+v, want the module's name, the artifact and the pinned reference", args)
	}
	if n := len(r.bm.callsOf("build_image_fetch")) + len(r.bm.callsOf("build_image_build")); n != 0 {
		t.Errorf("%d fetch or build calls for a copy, want none", n)
	}
}

func TestAnUpstreamBesideAnImageIsCopiedAndTheImageFetchedAndBuiltOnTheSameNode(t *testing.T) { // B4
	r := newRig(t, imageArt, catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest})
	r.puller = "ace"
	v := r.consider(t, "modules/m")
	if ok, missing := v.Referenced(); !ok {
		t.Fatalf("build = %+v, missing references %v", v, missing)
	}
	if f, b, c := len(r.bm.callsOf("build_image_fetch")), len(r.bm.callsOf("build_image_build")), len(r.bm.callsOf("build_image_copy")); f != 1 || b != 1 || c != 1 {
		t.Errorf("fetch %d, build %d, copy %d; want one each", f, b, c)
	}
}

func TestACopyThatFailsNotStoredFailsTheKeyAndIsBuiltAgain(t *testing.T) { // B4, ADR 0498 clause 2
	r := newRig(t, catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest})
	r.puller = "ace"
	r.bm.fails["ace/build_image_copy"] = builds.NotStored + "up: the registry answered 500"
	v := r.consider(t, "modules/m")
	if v.State != builds.BuildFailed || !strings.HasPrefix(v.Error, builds.NotStored) || builds.FailedUntilChanged(v.Error) {
		t.Fatalf("build = %+v, want failed 'not stored:' and built again", v)
	}
	delete(r.bm.fails, "ace/build_image_copy")
	if v = r.consider(t, "modules/m"); v.State != builds.BuildDone {
		t.Errorf("build = %+v, want done on the new revision", v)
	}
}

var tsPackage = catalogue.Artifact{Name: "lib", Kind: "package", Language: "typescript"}

func TestATypeScriptPackageIsBuiltByTheNodeModuleAndPublishedOnlyForTheTrunk(t *testing.T) { // B5, ADR 0498 clause 5, ADR 0500 clause 4
	c := newRig(t, tsPackage)
	c.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
	trunkModule(t, c)
	c.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
	c.b.Wait()
	v := valueOf(t, c.st, c.key)
	if v.State != builds.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "package" || v.Artifacts[0].Reference != "" {
		t.Fatalf("candidate build = %+v, want done with the packed digest and no reference", v)
	}
	for _, call := range c.bm.callsOf("build_node_build") {
		if _, ok := call.body.(map[string]any)["publish"]; ok {
			t.Errorf("a candidate's package build was asked to publish: %+v", call.body)
		}
	}

	r := newRig(t, tsPackage)
	r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
	v = r.consider(t, "modules/m")
	if v.State != builds.BuildDone || v.Artifacts[0].Reference != "@novox/mesh-sdk@0.1.15" {
		t.Fatalf("trunk build = %+v, want done naming the published version", v)
	}
	if n := len(r.bm.callsOf("build_node_build")); n == 0 {
		t.Error("the package was not built by the Node build module")
	}
}

func TestAVersionTakenFailsTheKeyUntilTheFingerprintChanges(t *testing.T) { // B5, ADR 0498 clause 5
	r := newRig(t, tsPackage)
	r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
	r.bm.fails["ace/build_node_build"] = builds.VersionTaken + "@novox/mesh-sdk@0.1.15 is published with another packed file"
	r.bm.fails["novox/build_node_build"] = r.bm.fails["ace/build_node_build"]
	v := r.consider(t, "modules/m")
	if v.State != builds.BuildFailed || !strings.HasPrefix(v.Error, builds.VersionTaken) || !strings.Contains(v.Error, "0.1.15") {
		t.Fatalf("build = %+v, want failed opening %q naming the version", v, builds.VersionTaken)
	}
	writes := r.st.writes
	r.consider(t, "modules/m")
	if r.st.writes != writes {
		t.Errorf("a version-taken key was built again (%d writes, was %d)", r.st.writes, writes)
	}
}

func TestACandidatesPackageIsPublishedOnceWhenTheTrunkNamesItsFingerprint(t *testing.T) { // B6, ADR 0498 clause 6
	r := newRig(t, tsPackage, tsBundle)
	r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
	trunkModule(t, r)
	r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
	r.b.Wait()
	if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone {
		t.Fatalf("candidate build = %+v", v)
	}
	rev := r.st.revs[r.key]
	r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // the merge names the candidate's fingerprint
	r.b.Wait()
	v := valueOf(t, r.st, r.key)
	if r.st.revs[r.key] <= rev {
		t.Fatalf("the trunk did not build the unpublished package again (revision %d, was %d)", r.st.revs[r.key], rev)
	}
	if ok, missing := v.Referenced(); !ok {
		t.Fatalf("build = %+v, missing references %v", v, missing)
	}
	rev = r.st.revs[r.key]
	r.b.tried = map[string]bool{}
	r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
	r.b.Wait()
	if r.st.revs[r.key] != rev {
		t.Error("a published package was built again: once only")
	}
}

func TestAPublishThatFailsVersionTakenOnTheTrunkKeepsThePrefixFirst(t *testing.T) { // B6, ADR 0498 clauses 5 and 6
	r := newRig(t, tsPackage)
	r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
	trunkModule(t, r)
	r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
	r.b.Wait()
	for _, n := range []string{"ace", "novox"} {
		r.bm.fails[n+"/build_node_build"] = builds.VersionTaken + "@novox/mesh-sdk@0.1.15 is published with another packed file"
	}
	r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
	r.b.Wait()
	if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.HasPrefix(v.Error, builds.VersionTaken) {
		t.Errorf("build = %+v, want failed opening %q", v, builds.VersionTaken)
	}
}

// sdkModule adds to the rig a module "sdk" whose manifest (digest b) holds the TypeScript package lib, published as ref.
func sdkModule(t *testing.T, r *rig, ref string) []byte {
	t.Helper()
	r.mf.byDigest[mdigest('b')] = catalogue.Manifest{Module: "sdk", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{tsPackage}}}
	r.bm.packages["lib"] = ref
	data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-sdk", Commit: "8a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp2: "src2:" + hex64('5'),
		Manifest: mdigest('b'), Path: ".", Toolchains: CatalogToolchains(agreedToolchains)})
	if err != nil {
		t.Fatal(err)
	}
	return data
}

func TestAModuleWhoseLockNamesAnUnpublishedPackageWaitsQueuedAndIsBuiltOncePublished(t *testing.T) { // B7, ADR 0500 clause 6
	const ref = "@novox/mesh-sdk@0.1.15"
	r := newRig(t, tsBundle)
	for _, n := range []string{"ace", "novox"} {
		r.bm.fails[n+"/build_node_fetch"] = waitsFor + ref + ": the lock names a version the registry does not hold"
	}
	v := r.consider(t, "modules/m")
	if v.State != builds.BuildQueued || v.Waits != ref || v.Error != "" {
		t.Fatalf("build = %+v, want queued, waiting for %s, not failed", v, ref)
	}
	delete(r.bm.fails, "ace/build_node_fetch")
	delete(r.bm.fails, "novox/build_node_fetch")
	writes := r.st.writes
	r.b.tried = map[string]bool{}
	r.consider(t, "modules/m")
	if r.st.writes != writes {
		t.Fatalf("a key waiting for an unpublished package was claimed (%d writes, was %d)", r.st.writes, writes)
	}
	r.b.Module(context.Background(), "sdk", sdkModule(t, r, ref)) // the package's trunk key is built and published
	r.b.Wait()
	if v = valueOf(t, r.st, r.key); v.State != builds.BuildDone || v.Waits != "" {
		t.Errorf("build = %+v, want built once the package it waits for is published", v)
	}
}

func TestAKeyWaitingForAPackageAlreadyPublishedIsBuiltWhenTheBuilderStarts(t *testing.T) { // B7: the scan at start
	const ref = "@novox/mesh-sdk@0.1.15"
	r := newRig(t, tsBundle)
	putBuild(t, r.st, r.key, builds.BuilderBuild{State: builds.BuildQueued, By: "novox.builder", Manifest: manifestD, Waits: ref, Step: "waits"})
	sdkKey, _ := builds.BuilderBuildKey("sdk", "src2:"+hex64('5'))
	putBuild(t, r.st, sdkKey, builds.BuilderBuild{State: builds.BuildDone, By: "novox.builder", Manifest: mdigest('b'), Step: "built",
		Artifacts: []builds.BuiltArtifact{{Name: "lib", Kind: "package", Digest: digest, Reference: ref}}})
	r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
	if err := r.b.Sweep(context.Background()); err != nil {
		t.Fatal(err)
	}
	r.b.Wait()
	if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone {
		t.Errorf("build = %+v, want built: the package it waits for was published before the builder started", v)
	}
}

func TestTheFetchOfAModuleNamingItsPathsCarriesItsDirectoryAndThosePaths(t *testing.T) { // the lead's interface (c), ADR 0499 clause 4
	r := newRig(t, tsPackage)
	r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
	paths := map[string]string{"src": hex40('1'), "package.json": hex40('2')}
	// The named tree the indexer computes from those paths (its own, ADR 0593 §2), given as data: the builder reads it,
	// never computes it.
	const tree = "named:" + "5b5e0f3b6f3c1f2a9d8e7c6b5a4938271605f4e3d2c1b0a9f8e7d6c5b4a39281"
	data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-sdk", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp2: fp1, Manifest: manifestD, Path: ".",
		Toolchains: CatalogToolchains(agreedToolchains), Inputs2: &meshcatalog.ModulesByModuleValueInputs2{Tree: tree, Paths: paths}})
	if err != nil {
		t.Fatal(err)
	}
	r.b.Module(context.Background(), "m", data)
	r.b.Wait()
	f := r.bm.callsOf("build_node_fetch")
	if len(f) == 0 {
		t.Fatal("no fetch")
	}
	req, err := buildstep.ParseFetchRequest(f[0].body.(map[string]any))
	if err != nil || req.Dir != "." || strings.Join(req.Paths, ",") != "package.json,src" || req.Commit != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" {
		t.Errorf("fetch request = %+v, %v; want dir . and the kept paths, sorted", req, err)
	}
	r2 := newRig(t, goBundle)
	r2.consider(t, "modules/m")
	if a := r2.bm.callsOf("build_go_fetch")[0].body.(map[string]any); a["dir"] != nil || a["paths"] != nil {
		t.Errorf("fetch args = %+v, want no dir or paths for a module naming none", a)
	}
}

func hex40(c byte) string { return strings.Repeat(string(c), 40) }

func TestOneNodeBuildsEachFingerprint(t *testing.T) { // B9, ADR 0478 clause 4: "From R4e one node builds each fingerprint"
	r := newRig(t, goBundle, archiveA, tsBundle) // ace and novox both qualify for both build modules
	v := r.consider(t, "modules/m")
	if v.State != builds.BuildDone || strings.Contains(v.Step, " and ") || strings.HasPrefix(v.Step, "one node (") {
		t.Fatalf("build = %+v, want done on one node of each build module, unmarked", v)
	}
	if f, b := len(r.bm.callsOf("build_go_fetch")), len(r.bm.callsOf("build_go_build")); f != 1 || b != 2 {
		t.Errorf("go fetches %d, builds %d; want one fetch and one build of each artifact, on one node", f, b)
	}
	if f := len(r.bm.callsOf("build_node_fetch")); f != 1 {
		t.Errorf("node fetches %d, want one", f)
	}
}

func TestAKeyDoneOnOneNodeIsNeverBuiltAgainForASecondNode(t *testing.T) { // B9: the second-node retake is gone
	r := newRig(t, goBundle)
	putBuild(t, r.st, r.key, builds.BuilderBuild{State: builds.BuildDone, By: "novox.builder", Manifest: manifestD,
		Step:      "one node (mesh-build-go): built mesh-build-go on one node, novox",
		Artifacts: []builds.BuiltArtifact{{Name: "tool", Kind: "bundle", Digest: digest, Reference: "artifact-store://m/tool/blobs/" + digest}}})
	writes := r.st.writes
	r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
	r.b.NodesChanged(context.Background())
	r.b.Wait()
	if r.st.writes != writes || len(r.bm.calls) != 0 {
		t.Errorf("a key done on one node was built again (%d writes, was %d; %d calls)", r.st.writes, writes, len(r.bm.calls))
	}
}

func TestACandidatesFetchNamesTheNamedPathsTheIndexerGaveIt(t *testing.T) { // ADR 0499 clause 4, mesh-sdk CandidateModule.Paths
	r := newRig(t, tsPackage)
	r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
	trunkModule(t, r)
	pulls := []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-catalog", Number: 7, Head: headX, Modules: []string{"m"},
		Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"m": {Fp2: fp1, Manifest: mdigest('a'), Paths: map[string]string{"src": hex40('1'), "package.json": hex40('2')}}}}}
	hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: pulls[0].Repository, Number: int(pulls[0].Number), Head: pulls[0].Head}})
	data, err := json.Marshal(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat/x", Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)})
	if err != nil {
		t.Fatal(err)
	}
	r.b.Candidate(context.Background(), "feat%2Fx", data)
	r.b.Wait()
	f := r.bm.callsOf("build_node_fetch")
	if len(f) == 0 {
		t.Fatal("no fetch")
	}
	req, err := buildstep.ParseFetchRequest(f[0].body.(map[string]any))
	// The base is the trunk head the candidate was previewed onto, never the module value's commit (an older one).
	if err != nil || req.Dir != "modules/m" || strings.Join(req.Paths, ",") != "package.json,src" || req.Base != trunkHead {
		t.Errorf("fetch request = %+v, %v; want the trunk head %.8s as the base, the module's directory and its named paths", req, err, trunkHead)
	}
}

// 1886/AC5: the builder tells who answered a reference: the build module whose step named one, and none for a step that
// answered a digest alone (a build module from before R4e).
func TestTheBuilderTellsWhichBuildModuleAnsweredAReference(t *testing.T) {
	r := newRig(t, goBundle)
	var answered []string
	r.b.Answered = func(module string) { answered = append(answered, module) }
	r.bm.noRef["ace/tool"] = true
	r.consider(t, "modules/m")
	if len(answered) != 0 {
		t.Fatalf("a step that answered no reference was told as one: %v", answered)
	}
	delete(r.bm.noRef, "ace/tool")
	r.consider(t, "modules/m")
	if len(answered) != 1 || answered[0] != "mesh-build-go" {
		t.Fatalf("the build module that answered a reference: %v, want mesh-build-go", answered)
	}
}