mesh-delivery/modules/mesh-delivery-builder/internal/build/round4_test.go

package build

import (
	"context"
	"strings"
	"testing"

	"git.novox.be/novox/mesh-sdk/go/catalogue"
	"git.novox.be/novox/mesh-sdk/go/delivery"
)

func haveValue(t *testing.T, phase delivery.Phase, health delivery.Health) []byte {
	t.Helper()
	b, err := delivery.EncodeHave(delivery.Have{Phase: phase, Health: health})
	if err != nil {
		t.Fatal(err)
	}
	return b
}

// 1. A module of an archive only: the real indexer gives an archive no language (inputs.go languagesOf), so its
// fingerprint names no toolchain, and the builder builds it on the build module's nodes with no toolchain agreed.
func TestAnArchiveOnlyModuleIsBuiltAlthoughItsFingerprintNamesNoToolchain(t *testing.T) { // review round 4, 1
	r := newRig(t, archiveA)
	v := func() delivery.BuilderBuild {
		r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", nil))
		r.b.Wait()
		if _, ok := r.st.vals[r.key]; !ok {
			t.Fatal("no build key was written for the archive-only module")
		}
		return valueOf(t, r.st, r.key)
	}()
	if v.State != delivery.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "archive" || len(v.Toolchains) != 0 {
		t.Fatalf("build = %+v, want done with the archive and no toolchain named", v)
	}
	if n := len(r.bm.callsOf("build_go_build")); n != 2 {
		t.Errorf("%d go builds, want one on each of two nodes", n)
	}
}

func TestAnArchiveOnlyModuleNeedsNoAgreedToolchainButNeedsNodesThatQualify(t *testing.T) { // review round 4, 1
	r := newRig(t, archiveA)
	r.tools.Put(GoBuildModule, "ace.go", tcValue(t, "go", "go1.26.0"), false) // the nodes disagree: a bundle would wait
	r.b.Health = &HaveSet{}                                                   // no build module is healthy anywhere
	r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", nil))
	r.b.Wait()
	if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.Contains(v.Error, "healthy") {
		t.Errorf("build = %+v, want failed: no node's build module is healthy", v)
	}
	have := &HaveSet{}
	have.Put("novox.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false)
	r.b.Health = have
	r.b.Reconsider(context.Background())
	r.b.Wait()
	v := valueOf(t, r.st, r.key)
	if v.State != delivery.BuildDone || !strings.HasPrefix(v.Step, OneNodePrefix) {
		t.Errorf("build = %+v, want done on novox, the one node that qualifies, marked as one node", v)
	}
}

func TestABundleAndAnArchiveOfOneBuildModuleStillNeedTheAgreedToolchain(t *testing.T) { // review round 4, 1
	p, err := PlanOf(catalogue.Manifest{Module: "m", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle, archiveA}}})
	if err != nil || !p.NeedsToolchain(GoBuildModule) {
		t.Errorf("a plan with a Go bundle: %v, %v", p.NeedsToolchain(GoBuildModule), err)
	}
	p, _ = PlanOf(catalogue.Manifest{Module: "m", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{archiveA}}})
	if p.NeedsToolchain(GoBuildModule) {
		t.Error("a plan of an archive needs a toolchain")
	}
}

// 2. The digests-differ failure opens with its prefix, and nothing builds it again.
func TestDifferingDigestsFailTheKeyWithThePrefixAndNothingBuildsItAgain(t *testing.T) { // review round 4, 2
	r := newRig(t, goBundle)
	r.bm.digests["ace/tool"] = "sha256:" + hex64('9')
	v := r.consider(t, "modules/m")
	if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, "digests differ:") {
		t.Fatalf("build = %+v, want a failure opening with exactly %q", v, "digests differ:")
	}
	calls, writes := len(r.bm.calls), r.st.writes
	r.bm.digests = map[string]string{} // the nodes would now agree
	r.b.Reconsider(context.Background())
	r.b.Wait()
	r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
	r.b.Wait()
	if len(r.bm.calls) != calls || r.st.writes != writes {
		t.Errorf("a failed-for-digests key was built again: %d calls (was %d), %d writes (was %d)", len(r.bm.calls), calls, r.st.writes, writes)
	}
}

func TestClaimLeavesAKeyFailedForDifferingDigestsAlone(t *testing.T) { // review round 4, 2
	st := newMem()
	seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, Error: DigestsDifferPrefix + " the nodes built tool to different digests: ace x; novox y"})
	writes := st.writes
	got, err := (Claimer{Store: st, By: "novox.a"}).Claim(context.Background(), "m", fp1, "", ClaimOptions{TakeOver: true})
	if err != nil || got != nil || st.writes != writes {
		t.Errorf("claim = %v, %v, %d writes (was %d), want none and no write", got, err, st.writes, writes)
	}
}

// 3. A done key built on one node is built again on a second node once one qualifies.
func TestADoneKeyOfOneNodeIsBuiltAgainOnASecondNodeAndTheDigestsCompared(t *testing.T) { // review round 4, 3
	r := newRig(t, goBundle)
	r.pres.Put("ace", presenceValue(t, false), false) // only novox qualifies
	v := r.consider(t, "modules/m")
	if v.State != delivery.BuildDone || !strings.HasPrefix(v.Step, OneNodePrefix+GoBuildModule+"): ") || !strings.Contains(v.Step, "on one node, novox") {
		t.Fatalf("build = %+v, want done on one node, marked", v)
	}
	if got := OneNodeModules(v); len(got) != 1 || got[0] != GoBuildModule {
		t.Errorf("OneNodeModules = %v", got)
	}
	rev, calls := r.st.revs[r.key], len(r.bm.callsOf("build_go_build"))
	// Nothing changes while one node qualifies: looking again builds nothing.
	r.b.Reconsider(context.Background())
	r.b.Wait()
	if r.st.revs[r.key] != rev || len(r.bm.callsOf("build_go_build")) != calls {
		t.Fatal("a one-node key was built again with no second node")
	}
	// ace comes online: the key is built on ace and novox, as a new revision.
	r.pres.Put("ace", presenceValue(t, true), false)
	r.b.Reconsider(context.Background())
	r.b.Wait()
	v = valueOf(t, r.st, r.key)
	if v.State != delivery.BuildDone || strings.HasPrefix(v.Step, OneNodePrefix) || !strings.Contains(v.Step, "ace and novox") {
		t.Errorf("build = %+v, want done on ace and novox", v)
	}
	if r.st.revs[r.key] <= rev {
		t.Errorf("revision %d, was %d: not a new revision", r.st.revs[r.key], rev)
	}
	builds := r.bm.callsOf("build_go_build")
	if len(builds) != calls+2 || builds[len(builds)-2].node == builds[len(builds)-1].node {
		t.Errorf("%d builds (was %d), want one on each of two nodes", len(builds), calls)
	}
}

func TestASecondNodeThatDisagreesFailsTheDoneKeyWithTheDigestsPrefix(t *testing.T) { // review round 4, 3
	r := newRig(t, goBundle)
	r.pres.Put("ace", presenceValue(t, false), false)
	r.consider(t, "modules/m")
	r.bm.digests["ace/tool"] = "sha256:" + hex64('9')
	r.pres.Put("ace", presenceValue(t, true), false)
	r.b.Reconsider(context.Background())
	r.b.Wait()
	if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, DigestsDifferPrefix) {
		t.Errorf("build = %+v, want failed with digests differ", v)
	}
}

func TestASecondBuildThatFailsForAnotherReasonLeavesTheDoneKeyStanding(t *testing.T) { // review round 4, 3
	r := newRig(t, goBundle)
	r.pres.Put("ace", presenceValue(t, false), false)
	first := r.consider(t, "modules/m")
	r.bm.fails["ace/build_go_fetch"] = "ace is out of disk"
	r.pres.Put("ace", presenceValue(t, true), false)
	r.b.Reconsider(context.Background())
	r.b.Wait()
	if v := valueOf(t, r.st, r.key); !delivery.SameBuilderBuild(v, first) {
		t.Errorf("build = %+v, want the first done build to stand: %+v", v, first)
	}
}

// 4. A step never waits on a full node while another qualifying node has room.
func TestChoosesMadeAtOnceSeeEachOthersReservationsSoNoLaneWaitsWhileANodeHasRoom(t *testing.T) { // review round 4, 4
	p := &Places{Capacity: map[string]int{"ace": 2, "g14": 1, "novox": 1, "shanks": 3}}
	entries := tcs("go", "go1.27.1", "ace", "g14", "novox", "shanks")
	total := 2 + 1 + 1 + 3
	var waiting int
	for i := 1; i <= 4; i++ { // four modules choose before any lane has started
		room := 0
		for _, n := range []string{"ace", "g14", "novox", "shanks"} {
			room += p.CapacityOf(n) - p.Running()[n] - p.Reserved()[n]
		}
		nodes, _, err := ChooseNodes("go", entries, p, nil, 2)
		if err != nil || len(nodes) != 2 {
			t.Fatalf("choice %d: %v, %v", i, nodes, err)
		}
		if room >= 2 {
			for _, n := range nodes {
				if p.Reserved()[n] > p.CapacityOf(n) {
					t.Errorf("choice %d: %v put a lane on full %s while %d places were free", i, nodes, n, room)
				}
			}
		} else {
			waiting += 2 - room
		}
	}
	if want := 4*2 - total; waiting != want {
		t.Errorf("%d lanes wait, want %d: the 8 lanes less the %d places", waiting, want, total)
	}
	loads := p.Reserved()
	for n, c := range loads {
		if c > p.CapacityOf(n) && n != "shanks" && n != "ace" && n != "g14" && n != "novox" {
			t.Errorf("unknown node %s", n)
		}
	}
	if loads["shanks"] != 3 || loads["ace"] != 2 || loads["g14"]+loads["novox"] != 3 {
		t.Errorf("reservations = %v, want shanks 3, ace 2 and the single-place nodes taking the rest", loads)
	}
}

func TestACancelledReservationFreesTheChoiceAndAnAcquireTakesItsReservation(t *testing.T) { // review round 4, 4
	p := &Places{}
	nodes, _, _ := ChooseNodes("go", tcs("go", "go1.27.1", "ace"), p, nil, 1)
	if len(nodes) != 1 || p.Reserved()["ace"] != 1 {
		t.Fatalf("nodes %v, reserved %v", nodes, p.Reserved())
	}
	release, err := p.Acquire(context.Background(), "ace")
	if err != nil || p.Reserved()["ace"] != 0 || p.Running()["ace"] != 1 {
		t.Fatalf("acquire: %v, reserved %v, running %v", err, p.Reserved(), p.Running())
	}
	release()
	ChooseNodes("go", tcs("go", "go1.27.1", "ace"), p, nil, 1)
	p.Cancel("ace")
	if len(p.Reserved()) != 0 {
		t.Errorf("reserved after Cancel = %v", p.Reserved())
	}
}

// 5. A node whose build module is not healthy is not chosen.
func TestHaveSetKnowsOnlyBuildModulesRunningAndHealthy(t *testing.T) { // review round 4, 5
	var s HaveSet
	if s.Healthy("ace", GoBuildModule) {
		t.Error("a module with no have is healthy")
	}
	if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) || !s.Healthy("ace", GoBuildModule) {
		t.Error("a running, healthy module is not")
	}
	if s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) {
		t.Error("the same state counted as a change")
	}
	if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthFailed), false) || s.Healthy("ace", GoBuildModule) {
		t.Error("a module whose health failed is healthy")
	}
	s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false)
	if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseFailed, delivery.HealthOK), false) || s.Healthy("ace", GoBuildModule) {
		t.Error("a module whose phase failed is healthy")
	}
	s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false)
	if !s.Put("ace.mesh-build-go", nil, true) || s.Healthy("ace", GoBuildModule) {
		t.Error("a deleted have is healthy")
	}
	if s.Put("ace.node", []byte(`{}`), false) || s.Put("ace.mesh-build-go", []byte(`not json`), false) {
		t.Error("a node's key or a non-have counted")
	}
}

func TestANodeWhoseBuildModuleIsNotHealthyIsNotChosen(t *testing.T) { // review round 4, 5
	r := newRig(t, goBundle)
	have := &HaveSet{}
	have.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthFailed), false) // go is not the recipe's
	have.Put("novox.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false)
	r.b.Health = have
	v := r.consider(t, "modules/m")
	if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on one node, novox") {
		t.Fatalf("build = %+v, want done on novox alone", v)
	}
	for _, c := range r.bm.calls {
		if c.node == "ace" {
			t.Errorf("a step was called on ace, whose build module is unhealthy: %+v", c)
		}
	}
	have.Put("novox.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthFailed), false)
	r.st.vals = map[string][]byte{} // a fresh key
	r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
	r.b.Wait()
	if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.Contains(v.Error, "healthy") {
		t.Errorf("build = %+v, want failed: no node has a healthy build module", v)
	}
}

// 6. A key found building is taken over, whoever holds it.
func TestABuildLeftBuildingByAnyBuilderIsTakenOverAtStart(t *testing.T) { // review round 4, 6; 1607/AC3, ADR 0478 clauses 5, 6
	st := newMem()
	seed(t, st, delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.a"})
	if got, err := (Claimer{Store: st, By: "novox.a"}).Claim(context.Background(), "m", fp1, "", ClaimOptions{}); err != nil || got == nil {
		t.Errorf("own building key: claim = %v, %v, want it taken again", got, err)
	}
	if got, err := (Claimer{Store: st, By: "ace.b"}).Claim(context.Background(), "m", fp1, "", ClaimOptions{TakeOver: true}); err != nil || got == nil || got.Build.By != "novox.a" {
		t.Errorf("another node's building key at start: claim = %v, %v, want it taken over", got, err)
	}
	if got, _ := (Claimer{Store: st, By: "ace.b"}).Claim(context.Background(), "m", fp1, "", ClaimOptions{}); got != nil {
		t.Errorf("another builder's building key after the start was taken: a running builder's key is its own")
	}
}

func TestABuilderFindingAKeyBuildingByAnotherNodeBuildsItOnStart(t *testing.T) { // review round 4, 6
	r := newRig(t, goBundle)
	seed(t, r.st, delivery.BuilderBuild{State: delivery.BuildBuilding, By: "ace.builder", Manifest: manifestD})
	v := r.consider(t, "modules/m")
	if v.State != delivery.BuildDone || v.By != "novox.builder" {
		t.Errorf("build = %+v, want it taken over and done by novox.builder", v)
	}
}